From bae4e4487f86af126d62dc62db06b36d5a557bf9 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 20 Oct 2024 02:10:55 +0200 Subject: [PATCH 01/30] add support for invidious companion --- config/config.example.yml | 16 ++++++++++ src/invidious/config.cr | 3 ++ src/invidious/videos/parser.cr | 39 +++++++++++++----------- src/invidious/yt_backend/youtube_api.cr | 40 +++++++++++++++++++------ 4 files changed, 71 insertions(+), 27 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index b04e0a30..ddce3e41 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -54,6 +54,22 @@ db: ## #signature_server: +## +## Path to the Invidious companion. +## An external program for loading the video streams from YouTube servers. +## +## When this setting is commented out, Invidious companion is not used. +## +## When this setting is configured and "external_port" is used then +## you need to configure Invidious companion routes into your reverse proxy. +## If "external_port" is not configured then Invidious will proxy the requests +## to Invidious companion. +## +## Accepted values: "http(s)://:" +## Default: +## +#invidious_companion: + ######################################### # diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 453256b5..d60ef940 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -160,6 +160,9 @@ class Config # poToken for passing bot attestation property po_token : String? = nil + # Invidious companion + property invidious_companion : String? = nil + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 5ca4bdb2..61bd014f 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -108,27 +108,30 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - new_player_response = nil + if CONFIG.invidious_companion.nil? + new_player_response = nil - # Don't use Android test suite client if po_token is passed because po_token doesn't - # work for Android test suite client. - if reason.nil? && CONFIG.po_token.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Replace player response and reset reason - if !new_player_response.nil? - # Preserve captions & storyboard data before replacement - new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? - new_player_response["captions"] = player_response["captions"] if player_response["captions"]? + # Replace player response and reset reason + if !new_player_response.nil? + # Preserve captions & storyboard data before replacement + new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? - player_response = new_player_response - params.delete("reason") + player_response = new_player_response + params.delete("reason") + end + end end {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index ec080d8c..c64a4e57 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -615,12 +615,19 @@ module YoutubeAPI headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", "x-youtube-client-name" => client_config.name_proto, "x-youtube-client-version" => client_config.version, } + if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player" + headers["Authorization"] = "Bearer " + CONFIG.hmac_key + end + + if !CONFIG.invidious_companion + headers["Accept-Encoding"] = "gzip, deflate" + end + if user_agent = client_config.user_agent headers["User-Agent"] = user_agent end @@ -634,16 +641,31 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") + invidious_companion_url = CONFIG.invidious_companion + # Send the POST request - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - if response.status_code != 200 - raise InfoException.new("Error: non 200 status code. Youtube API returned \ - status code #{response.status_code}. See \ - https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") - end - self._decompress(response.body_io, response.headers["Content-Encoding"]?) + if invidious_companion_url && endpoint == "/youtubei/v1/player" + begin + body = make_client(URI.parse(invidious_companion_url), + &.post(endpoint, headers: headers, body: data.to_json).body) + rescue + raise InfoException.new("Unable to communicate with Invidious companion.") end + else + body = YT_POOL.client() do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See \ + https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") + end + self._decompress(response.body_io, response.headers["Content-Encoding"]?) + end + end + end + + if body.nil? && invidious_companion_url + raise InfoException.new("Unable to communicate with Invidious companion.") end # Convert result to Hash From 3629bdfd46671fff7d892767e22dde0fd1c3a10a Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:51:00 +0200 Subject: [PATCH 02/30] redirect latest_version and dash manifest to invidious companion --- config/config.example.yml | 3 ++- src/invidious/config.cr | 2 +- src/invidious/routes/api/manifest.cr | 4 ++++ src/invidious/routes/video_playback.cr | 3 +++ src/invidious/routes/watch.cr | 7 +++++++ src/invidious/videos.cr | 4 ++++ src/invidious/videos/parser.cr | 15 ++++++++++----- src/invidious/views/components/player.ecr | 9 +++++++-- src/invidious/yt_backend/youtube_api.cr | 22 ++++++++++++++-------- 9 files changed, 52 insertions(+), 17 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index ddce3e41..af2e9612 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -68,7 +68,8 @@ db: ## Accepted values: "http(s)://:" ## Default: ## -#invidious_companion: +# invidious_companion: +# - http://127.0.0.1:8282 ######################################### diff --git a/src/invidious/config.cr b/src/invidious/config.cr index d60ef940..fbb29560 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -161,7 +161,7 @@ class Config property po_token : String? = nil # Invidious companion - property invidious_companion : String? = nil + property invidious_companion : Array(String)? = nil # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 6c4225e5..f99c5d43 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -20,6 +20,10 @@ module Invidious::Routes::API::Manifest haltf env, status_code: 403 end + if local && CONFIG.invidious_companion + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + end + if dashmpd = video.dash_manifest_url response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index a8f9f665..932e5745 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -297,6 +297,9 @@ module Invidious::Routes::VideoPlayback end if local + if (CONFIG.invidious_companion) + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + end url = URI.parse(url).request_target.not_nil! url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 1f384546..95f47895 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -192,6 +192,13 @@ module Invidious::Routes::Watch captions: video.captions ) + if (CONFIG.invidious_companion && env.params.query["local"] == true) + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s) + end + templated "watch" end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 962f87bd..6fb47cb3 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -192,6 +192,10 @@ struct Video } end + def invidious_companion : Hash(String, JSON::Any) + info["invidiousCompanion"].try &.as_h + end + # Macros defining getters/setters for various types of data private macro getset_string(name) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 61bd014f..10fc842c 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -149,6 +149,10 @@ def extract_video_info(video_id : String) params["streamingData"] = streaming_data end + if CONFIG.invidious_companion + params["invidiousCompanion"] = player_response["invidiousCompanion"] + end + # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -457,11 +461,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Music section "music" => JSON.parse(music_list.to_json), # Author infos - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), + "invidiousCompanion" => JSON::Any.new(subs_text), } return params diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 523f6bbf..514d70d1 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,6 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -34,8 +35,11 @@ <% end %> <% end %> <% else %> - <% if params.quality == "dash" %> - + <% if params.quality == "dash" + src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion) + %> + <% end %> <% @@ -44,6 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index c64a4e57..6edc2624 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -641,15 +641,21 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") - invidious_companion_url = CONFIG.invidious_companion + invidious_companion_urls = CONFIG.invidious_companion # Send the POST request - if invidious_companion_url && endpoint == "/youtubei/v1/player" + if invidious_companion_urls && endpoint == "/youtubei/v1/player" + puts "invidious companion section" + puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] begin - body = make_client(URI.parse(invidious_companion_url), - &.post(endpoint, headers: headers, body: data.to_json).body) - rescue - raise InfoException.new("Unable to communicate with Invidious companion.") + response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), + &.post(endpoint, headers: headers, body: data.to_json)) + body = response.body + if (response.status_code != 200) + raise Exception.new("status code: " + response.status_code.to_s + " and body: " + body) + end + rescue ex + raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end else body = YT_POOL.client() do |client| @@ -664,8 +670,8 @@ module YoutubeAPI end end - if body.nil? && invidious_companion_url - raise InfoException.new("Unable to communicate with Invidious companion.") + if body.nil? && CONFIG.invidious_companion + raise InfoException.new("Error while communicating with Invidious companion: no response data.") end # Convert result to Hash From 8dbe1f6f33da0ea9bace421bb84d55fa5f5e961b Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:53:08 +0200 Subject: [PATCH 03/30] fix Shadowing outer local variable `response` --- src/invidious/yt_backend/youtube_api.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 6edc2624..83510403 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -648,11 +648,11 @@ module YoutubeAPI puts "invidious companion section" puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] begin - response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), + invidious_companion_response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), &.post(endpoint, headers: headers, body: data.to_json)) - body = response.body - if (response.status_code != 200) - raise Exception.new("status code: " + response.status_code.to_s + " and body: " + body) + body = invidious_companion_response.body + if (invidious_companion_response.status_code != 200) + raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body) end rescue ex raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) From c29c878b0552935066b6a7c458d5d7d66fb5fc98 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Mon, 21 Oct 2024 01:20:16 +0200 Subject: [PATCH 04/30] fixing condition for Content-Security-Policy --- src/invidious/routes/watch.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 95f47895..c1228f64 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -192,7 +192,7 @@ module Invidious::Routes::Watch captions: video.captions ) - if (CONFIG.invidious_companion && env.params.query["local"] == true) + if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash")) env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) From 02c5def9082869aa17ec531e4f1a85d055c3148b Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:30:58 +0100 Subject: [PATCH 05/30] throw error if inv_sig_helper and invidious_companion used same time --- src/invidious.cr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/invidious.cr b/src/invidious.cr index 12ffad33..2bab2186 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -161,6 +161,12 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} +# invidious_companion and signature_server can't work together +if CONFIG.signature_server && CONFIG.invidious_companion + puts "You can not run inv_sig_helper and invidious_companion at the same time." + exit(1) +end + # Misc DECRYPT_FUNCTION = From c95dc9a55b3adf69b00bb3cd98e5c05487ae38c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:31:21 +0100 Subject: [PATCH 06/30] Use sample instead of Random.rand Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious/yt_backend/youtube_api.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 83510403..af48fe95 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -648,7 +648,7 @@ module YoutubeAPI puts "invidious companion section" puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] begin - invidious_companion_response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), + invidious_companion_response = make_client(URI.parse(invidious_companion_urls.sample), &.post(endpoint, headers: headers, body: data.to_json)) body = invidious_companion_response.body if (invidious_companion_response.status_code != 200) From 84bd6a831d55cb08cd276714c0e2f530e90acfa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:31:45 +0100 Subject: [PATCH 07/30] Remove debug puts functions Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious/yt_backend/youtube_api.cr | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index af48fe95..93da03e3 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -645,8 +645,6 @@ module YoutubeAPI # Send the POST request if invidious_companion_urls && endpoint == "/youtubei/v1/player" - puts "invidious companion section" - puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] begin invidious_companion_response = make_client(URI.parse(invidious_companion_urls.sample), &.post(endpoint, headers: headers, body: data.to_json)) From c262d7072fbee75c2f460aafd661c60cae3537c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:50:59 +0100 Subject: [PATCH 08/30] modify the description for config.example.yaml about invidious companion --- config/config.example.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index af2e9612..4c0816e5 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -55,15 +55,14 @@ db: #signature_server: ## -## Path to the Invidious companion. -## An external program for loading the video streams from YouTube servers. +## Invidious companion is an external program +## for loading the video streams from YouTube servers. ## ## When this setting is commented out, Invidious companion is not used. ## -## When this setting is configured and "external_port" is used then -## you need to configure Invidious companion routes into your reverse proxy. -## If "external_port" is not configured then Invidious will proxy the requests +## When this setting is configured, then Invidious will proxy the requests ## to Invidious companion. +## Or randomly choose one if multiple Invidious companion are configured. ## ## Accepted values: "http(s)://:" ## Default: From 401cba0f2fc308c998b92317b1c1b8c4bae35d62 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 8 Nov 2024 21:05:17 +0100 Subject: [PATCH 09/30] move config checks for invidious companion --- config/config.example.yml | 12 +++++++++++- src/invidious.cr | 6 ------ src/invidious/config.cr | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 4c0816e5..4c0bd0f0 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -67,9 +67,19 @@ db: ## Accepted values: "http(s)://:" ## Default: ## -# invidious_companion: +#invidious_companion: # - http://127.0.0.1:8282 +## +## API key for Invidious companion +## +## Needed when invidious_companion is configured +## +## Accepted values: "http(s)://:" +## Default: +## + +#invidious_companion_key: "CHANGE_ME!!" ######################################### # diff --git a/src/invidious.cr b/src/invidious.cr index 2bab2186..12ffad33 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -161,12 +161,6 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} -# invidious_companion and signature_server can't work together -if CONFIG.signature_server && CONFIG.invidious_companion - puts "You can not run inv_sig_helper and invidious_companion at the same time." - exit(1) -end - # Misc DECRYPT_FUNCTION = diff --git a/src/invidious/config.cr b/src/invidious/config.cr index fbb29560..0e0f0d9d 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -163,6 +163,9 @@ class Config # Invidious companion property invidious_companion : Array(String)? = nil + # Invidious companion API key + property invidious_companion_key : String? = nil + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new @@ -243,6 +246,21 @@ class Config end {% end %} + if CONFIG.invidious_companion + # invidious_companion and signature_server can't work together + if CONFIG.signature_server + puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." + exit(1) + end + if !CONFIG.invidious_companion_key + puts "Config: Please configure a key if you are using invidious companion." + exit(1) + elsif CONFIG.invidious_companion_key == "CHANGE_ME!!" + puts "Config: The value of 'invidious_companion_key' needs to be changed!!" + exit(1) + end + end + # HMAC_key is mandatory # See: https://github.com/iv-org/invidious/issues/3854 if config.hmac_key.empty? From 39e5370d35e3cb876daedd8dc11f8dfce9e8f8e8 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 16 Nov 2024 22:33:28 +0100 Subject: [PATCH 10/30] separate invidious_companion logic + better config.yaml config --- src/invidious/config.cr | 29 ++++++- src/invidious/routes/api/manifest.cr | 8 +- src/invidious/routes/embed.cr | 7 ++ src/invidious/routes/video_playback.cr | 8 +- src/invidious/routes/watch.cr | 6 +- src/invidious/videos.cr | 6 +- src/invidious/videos/parser.cr | 19 ++--- src/invidious/views/components/player.ecr | 6 +- src/invidious/yt_backend/youtube_api.cr | 95 ++++++++++++++--------- 9 files changed, 119 insertions(+), 65 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 0e0f0d9d..81836eb3 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -74,6 +74,28 @@ end class Config include YAML::Serializable + module URIArrayConverter + def self.to_yaml(values : Array(URI), yaml : YAML::Nodes::Builder) + yaml.sequence do + values.each { |v| yaml.scalar v.to_s } + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(URI) + if node.is_a?(YAML::Nodes::Sequence) + node.map do |child| + unless child.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{child.class}" + end + + URI.parse(child.value) + end + else + node.raise "Expected sequence, not #{node.class}" + end + end + end + # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 # Time interval between two executions of the job that crawls channel videos (subscriptions update). @@ -161,10 +183,11 @@ class Config property po_token : String? = nil # Invidious companion - property invidious_companion : Array(String)? = nil + @[YAML::Field(converter: Config::URIArrayConverter)] + property invidious_companion : Array(URI) = [] of URI # Invidious companion API key - property invidious_companion_key : String? = nil + property invidious_companion_key : String = "" # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] @@ -246,7 +269,7 @@ class Config end {% end %} - if CONFIG.invidious_companion + if !CONFIG.invidious_companion.empty? # invidious_companion and signature_server can't work together if CONFIG.signature_server puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index f99c5d43..e5a2118f 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -1,6 +1,10 @@ module Invidious::Routes::API::Manifest # /api/manifest/dash/id/:id def self.get_dash_video_id(env) + if !CONFIG.invidious_companion.empty? + return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") + end + env.response.headers.add("Access-Control-Allow-Origin", "*") env.response.content_type = "application/dash+xml" @@ -20,10 +24,6 @@ module Invidious::Routes::API::Manifest haltf env, status_code: 403 end - if local && CONFIG.invidious_companion - return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" - end - if dashmpd = video.dash_manifest_url response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 00f24159..f89c50fc 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -203,6 +203,13 @@ module Invidious::Routes::Embed return env.redirect url end + if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash")) + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + end + rendered "embed" end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 932e5745..44580443 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -256,6 +256,10 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) + if !CONFIG.invidious_companion.empty? && CONFIG.disabled?("downloads") + return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") + end + id = env.params.query["id"]? itag = env.params.query["itag"]?.try &.to_i? @@ -297,8 +301,8 @@ module Invidious::Routes::VideoPlayback end if local - if (CONFIG.invidious_companion) - return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + if (!CONFIG.invidious_companion.empty?) + return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" end url = URI.parse(url).request_target.not_nil! url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index c1228f64..d69d4c2c 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -192,11 +192,11 @@ module Invidious::Routes::Watch captions: video.captions ) - if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash")) + if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash")) env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) - .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s) + .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) end templated "watch" diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 6fb47cb3..70e6f401 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -15,7 +15,7 @@ struct Video # NOTE: don't forget to bump this number if any change is made to # the `params` structure in videos/parser.cr!!! # - SCHEMA_VERSION = 2 + SCHEMA_VERSION = 3 property id : String @@ -192,8 +192,8 @@ struct Video } end - def invidious_companion : Hash(String, JSON::Any) - info["invidiousCompanion"].try &.as_h + def invidious_companion : Hash(String, JSON::Any)? + info["invidiousCompanion"]?.try &.as_h end # Macros defining getters/setters for various types of data diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 10fc842c..c48e4243 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -108,7 +108,7 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - if CONFIG.invidious_companion.nil? + if !CONFIG.invidious_companion.empty? new_player_response = nil # Don't use Android test suite client if po_token is passed because po_token doesn't @@ -134,7 +134,7 @@ def extract_video_info(video_id : String) end end - {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f| params[f] = player_response[f] if player_response[f]? end @@ -149,10 +149,6 @@ def extract_video_info(video_id : String) params["streamingData"] = streaming_data end - if CONFIG.invidious_companion - params["invidiousCompanion"] = player_response["invidiousCompanion"] - end - # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -461,12 +457,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Music section "music" => JSON.parse(music_list.to_json), # Author infos - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), - "invidiousCompanion" => JSON::Any.new(subs_text), + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), } return params diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 514d70d1..2ed5d30c 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,7 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -37,7 +37,7 @@ <% else %> <% if params.quality == "dash" src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) %> <% end %> @@ -48,7 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 93da03e3..9c007b07 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -500,7 +500,11 @@ module YoutubeAPI data["params"] = params end - return self._post_json("/youtubei/v1/player", data, client_config) + if !CONFIG.invidious_companion.empty? + return self._post_invidious_companion("/youtubei/v1/player", data) + else + return self._post_json("/youtubei/v1/player", data, client_config) + end end #################################################################### @@ -615,19 +619,12 @@ module YoutubeAPI headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", "x-youtube-client-name" => client_config.name_proto, "x-youtube-client-version" => client_config.version, } - if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player" - headers["Authorization"] = "Bearer " + CONFIG.hmac_key - end - - if !CONFIG.invidious_companion - headers["Accept-Encoding"] = "gzip, deflate" - end - if user_agent = client_config.user_agent headers["User-Agent"] = user_agent end @@ -641,35 +638,16 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") - invidious_companion_urls = CONFIG.invidious_companion - # Send the POST request - if invidious_companion_urls && endpoint == "/youtubei/v1/player" - begin - invidious_companion_response = make_client(URI.parse(invidious_companion_urls.sample), - &.post(endpoint, headers: headers, body: data.to_json)) - body = invidious_companion_response.body - if (invidious_companion_response.status_code != 200) - raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body) + body = YT_POOL.client() do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See \ + https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") end - rescue ex - raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) + self._decompress(response.body_io, response.headers["Content-Encoding"]?) end - else - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - if response.status_code != 200 - raise InfoException.new("Error: non 200 status code. Youtube API returned \ - status code #{response.status_code}. See \ - https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") - end - self._decompress(response.body_io, response.headers["Content-Encoding"]?) - end - end - end - - if body.nil? && CONFIG.invidious_companion - raise InfoException.new("Error while communicating with Invidious companion: no response data.") end # Convert result to Hash @@ -692,6 +670,53 @@ module YoutubeAPI return initial_data end + #################################################################### + # _post_invidious_companion(endpoint, data) + # + # Internal function that does the actual request to Invidious companion + # and handles errors. + # + # The requested data is an endpoint (URL without the domain part) + # and the data as a Hash object. + # + def _post_invidious_companion( + endpoint : String, + data : Hash + ) : Hash(String, JSON::Any) + headers = HTTP::Headers{ + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip", + "Authorization" => "Bearer " + CONFIG.invidious_companion_key, + } + + # Logging + LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"") + LOGGER.trace("Invidious companion: POST data: #{data}") + + # Send the POST request + + begin + response = make_client(CONFIG.invidious_companion.sample, + &.post(endpoint, headers: headers, body: data.to_json)) + body = self._decompress(response.body_io, response.headers["Content-Encoding"]?) + if (response.status_code != 200) + raise Exception.new("Error while communicating with Invidious companion: \ + status code: " + response.status_code.to_s + " and body: " + body) + end + rescue ex + raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) + end + + if body.nil? + raise InfoException.new("Error while communicating with Invidious companion: no response data.") + end + + # Convert result to Hash + initial_data = JSON.parse(body).as_h + + return initial_data + end + #################################################################### # _decompress(body_io, headers) # From 02c904cc3ed1cc8ab2f47869a9f8d49359e05416 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 16 Nov 2024 22:38:00 +0100 Subject: [PATCH 11/30] fixing "end" misplacement --- src/invidious/videos/parser.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index c48e4243..596a2d39 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -131,7 +131,6 @@ def extract_video_info(video_id : String) player_response = new_player_response params.delete("reason") end - end end {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f| From 233f952157b4155c77c5e8245ff1fa1bb7b3aa7f Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 16 Nov 2024 23:00:48 +0100 Subject: [PATCH 12/30] fix linting + use .empty? --- src/invidious/config.cr | 2 +- src/invidious/videos/parser.cr | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 81836eb3..5807f6c3 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -275,7 +275,7 @@ class Config puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." exit(1) end - if !CONFIG.invidious_companion_key + if CONFIG.invidious_companion_key.empty? puts "Config: Please configure a key if you are using invidious companion." exit(1) elsif CONFIG.invidious_companion_key == "CHANGE_ME!!" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 596a2d39..c89fde1e 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -128,9 +128,9 @@ def extract_video_info(video_id : String) new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? new_player_response["captions"] = player_response["captions"] if player_response["captions"]? - player_response = new_player_response - params.delete("reason") - end + player_response = new_player_response + params.delete("reason") + end end {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f| From 4d7ee90a7489f8c81fcee7abd5fe3c4b97aa1000 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:26:35 +0100 Subject: [PATCH 13/30] crystal handle decompression already by itself --- src/invidious/config.cr | 8 ++++---- src/invidious/yt_backend/youtube_api.cr | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 5807f6c3..39d791c9 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -269,16 +269,16 @@ class Config end {% end %} - if !CONFIG.invidious_companion.empty? + if !config.invidious_companion.empty? # invidious_companion and signature_server can't work together - if CONFIG.signature_server + if config.signature_server puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." exit(1) end - if CONFIG.invidious_companion_key.empty? + if config.invidious_companion_key.empty? puts "Config: Please configure a key if you are using invidious companion." exit(1) - elsif CONFIG.invidious_companion_key == "CHANGE_ME!!" + elsif config.invidious_companion_key == "CHANGE_ME!!" puts "Config: The value of 'invidious_companion_key' needs to be changed!!" exit(1) end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 9c007b07..f5297c4c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -685,7 +685,6 @@ module YoutubeAPI ) : Hash(String, JSON::Any) headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip", "Authorization" => "Bearer " + CONFIG.invidious_companion_key, } @@ -698,7 +697,7 @@ module YoutubeAPI begin response = make_client(CONFIG.invidious_companion.sample, &.post(endpoint, headers: headers, body: data.to_json)) - body = self._decompress(response.body_io, response.headers["Content-Encoding"]?) + body = response.body if (response.status_code != 200) raise Exception.new("Error while communicating with Invidious companion: \ status code: " + response.status_code.to_s + " and body: " + body) From d95df8710dc6ffd46ad3ef460ea826b59e69fbf4 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun, 17 Nov 2024 19:18:29 +0100 Subject: [PATCH 14/30] fix download function when invidious companion used --- src/invidious/routes/video_playback.cr | 5 +---- src/invidious/routes/watch.cr | 11 +++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 44580443..64afc0da 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -256,7 +256,7 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) - if !CONFIG.invidious_companion.empty? && CONFIG.disabled?("downloads") + if !CONFIG.invidious_companion.empty? return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") end @@ -301,9 +301,6 @@ module Invidious::Routes::VideoPlayback end if local - if (!CONFIG.invidious_companion.empty?) - return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" - end url = URI.parse(url).request_target.not_nil! url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index d69d4c2c..c5546fe8 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -322,13 +322,20 @@ module Invidious::Routes::Watch return Invidious::Routes::API::V1::Videos.captions(env) elsif itag = download_widget["itag"]?.try &.as_i + itag = itag.to_s + # URL params specific to /latest_version env.params.query["id"] = video_id - env.params.query["itag"] = itag.to_s + env.params.query["itag"] = itag env.params.query["title"] = filename env.params.query["local"] = "true" - return Invidious::Routes::VideoPlayback.latest_version(env) + if (!CONFIG.invidious_companion.empty?) + video = get_video(video_id) + return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}/latest_version?id=#{video_id}&itag=#{itag}&local=true" + else + return Invidious::Routes::VideoPlayback.latest_version(env) + end else return error_template(400, "Invalid label or itag") end From 96ed65f0aedde1a89ddb8fd82f87fe56c5c61c1d Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:22:23 +0100 Subject: [PATCH 15/30] fix linting --- src/invidious/yt_backend/youtube_api.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index f5297c4c..48c65252 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -684,8 +684,8 @@ module YoutubeAPI data : Hash ) : Hash(String, JSON::Any) headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Authorization" => "Bearer " + CONFIG.invidious_companion_key, + "Content-Type" => "application/json; charset=UTF-8", + "Authorization" => "Bearer " + CONFIG.invidious_companion_key, } # Logging From 50f47a5eb0f237081ab2975cba47f6e6e00b990c Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:30:37 +0100 Subject: [PATCH 16/30] invidious companion always used so always add CSP and redirect latest_version --- src/invidious/routes/embed.cr | 2 +- src/invidious/routes/watch.cr | 2 +- src/invidious/views/components/player.ecr | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index f89c50fc..4819ac22 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -203,7 +203,7 @@ module Invidious::Routes::Embed return env.redirect url end - if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash")) + if (!CONFIG.invidious_companion.empty?) env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index c5546fe8..8df1191f 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -192,7 +192,7 @@ module Invidious::Routes::Watch captions: video.captions ) - if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash")) + if (!CONFIG.invidious_companion.empty?) env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 2ed5d30c..757ce823 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,7 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -48,7 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) From d7d3aedd0645c772b5efbc549a6c5e833d712b18 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun, 8 Dec 2024 22:21:10 +0100 Subject: [PATCH 17/30] apply all the suggestions + rework invidious_companion parameter --- config/config.example.yml | 19 ++++++++++++---- src/invidious/config.cr | 27 ++++++----------------- src/invidious/routes/api/manifest.cr | 9 ++++---- src/invidious/routes/embed.cr | 6 ++--- src/invidious/routes/video_playback.cr | 3 ++- src/invidious/routes/watch.cr | 12 +++++----- src/invidious/videos.cr | 2 +- src/invidious/views/components/player.ecr | 6 ++--- src/invidious/yt_backend/youtube_api.cr | 15 ++++++------- 9 files changed, 48 insertions(+), 51 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 4c0bd0f0..593ca192 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -59,16 +59,27 @@ db: ## for loading the video streams from YouTube servers. ## ## When this setting is commented out, Invidious companion is not used. +## Otherwise, Invidious will proxy the requests to Invidious companion. +## +## Note: multiple URL can be configured. In this case, invidious will +## randomly pick one every time video data needs to be retrieved. This +## URL is then kept in the video metadata cache to allow video playback +## to work. Once said cache has expired, requesting that video's data +## again will cause a new companion URL to be picked. ## -## When this setting is configured, then Invidious will proxy the requests -## to Invidious companion. -## Or randomly choose one if multiple Invidious companion are configured. +## The parameter private_url needs to be configured for the internal +## communication between the companion and Invidious. +## And public_url is the public URL from which companion is listening +## to the requests from the user(s). +## Both parameter can have identical URL when Invidious is hosted in +## an internal network or at home. ## ## Accepted values: "http(s)://:" ## Default: ## #invidious_companion: -# - http://127.0.0.1:8282 +# - private_url: "http://localhost:8282" +# public_url: "http://localhost:8282" ## ## API key for Invidious companion diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 39d791c9..5e7eb006 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -74,26 +74,14 @@ end class Config include YAML::Serializable - module URIArrayConverter - def self.to_yaml(values : Array(URI), yaml : YAML::Nodes::Builder) - yaml.sequence do - values.each { |v| yaml.scalar v.to_s } - end - end + class CompanionConfig + include YAML::Serializable - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(URI) - if node.is_a?(YAML::Nodes::Sequence) - node.map do |child| - unless child.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{child.class}" - end + @[YAML::Field(converter: Preferences::URIConverter)] + property private_url : URI = URI.parse("") - URI.parse(child.value) - end - else - node.raise "Expected sequence, not #{node.class}" - end - end + @[YAML::Field(converter: Preferences::URIConverter)] + property public_url : URI = URI.parse("") end # Number of threads to use for crawling videos from channels (for updating subscriptions) @@ -183,8 +171,7 @@ class Config property po_token : String? = nil # Invidious companion - @[YAML::Field(converter: Config::URIArrayConverter)] - property invidious_companion : Array(URI) = [] of URI + property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig # Invidious companion API key property invidious_companion_key : String = "" diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index e5a2118f..17c3bcb6 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -1,10 +1,6 @@ module Invidious::Routes::API::Manifest # /api/manifest/dash/id/:id def self.get_dash_video_id(env) - if !CONFIG.invidious_companion.empty? - return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") - end - env.response.headers.add("Access-Control-Allow-Origin", "*") env.response.content_type = "application/dash+xml" @@ -12,6 +8,11 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? + if !CONFIG.invidious_companion.empty? + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url.to_s}/api/manifest/dash/id/#{id}?#{env.params.query}" + end + # Since some implementations create playlists based on resolution regardless of different codecs, # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 4819ac22..49aeade5 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -203,11 +203,11 @@ module Invidious::Routes::Embed return env.redirect url end - if (!CONFIG.invidious_companion.empty?) + if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) - .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + .gsub("media-src", "media-src #{companion_base_url}") + .gsub("connect-src", "connect-src #{companion_base_url}") end rendered "embed" diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 64afc0da..b04868df 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -257,7 +257,8 @@ module Invidious::Routes::VideoPlayback # so we have a mechanism here to redirect to the latest version def self.latest_version(env) if !CONFIG.invidious_companion.empty? - return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url.to_s}/latest_version?#{env.params.query}" end id = env.params.query["id"]? diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 8df1191f..9a498265 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -192,11 +192,11 @@ module Invidious::Routes::Watch captions: video.captions ) - if (!CONFIG.invidious_companion.empty?) + if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) - .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + .gsub("media-src", "media-src #{companion_base_url}") + .gsub("connect-src", "connect-src #{companion_base_url}") end templated "watch" @@ -321,18 +321,16 @@ module Invidious::Routes::Watch env.params.query["label"] = URI.decode_www_form(label.as_s) return Invidious::Routes::API::V1::Videos.captions(env) - elsif itag = download_widget["itag"]?.try &.as_i - itag = itag.to_s + elsif itag = download_widget["itag"]?.try &.as_i.to_s # URL params specific to /latest_version env.params.query["id"] = video_id - env.params.query["itag"] = itag env.params.query["title"] = filename env.params.query["local"] = "true" if (!CONFIG.invidious_companion.empty?) video = get_video(video_id) - return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}/latest_version?id=#{video_id}&itag=#{itag}&local=true" + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}/latest_version?#{env.params.query}" else return Invidious::Routes::VideoPlayback.latest_version(env) end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 70e6f401..95abd709 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -193,7 +193,7 @@ struct Video end def invidious_companion : Hash(String, JSON::Any)? - info["invidiousCompanion"]?.try &.as_h + info["invidiousCompanion"]?.try &.as_h || {} of String => JSON::Any end # Macros defining getters/setters for various types of data diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 757ce823..cc7ffeaa 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,7 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -37,7 +37,7 @@ <% else %> <% if params.quality == "dash" src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" - src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) %> <% end %> @@ -48,7 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 48c65252..6b382820 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -685,7 +685,7 @@ module YoutubeAPI ) : Hash(String, JSON::Any) headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", - "Authorization" => "Bearer " + CONFIG.invidious_companion_key, + "Authorization" => "Bearer #{CONFIG.invidious_companion_key}", } # Logging @@ -695,21 +695,20 @@ module YoutubeAPI # Send the POST request begin - response = make_client(CONFIG.invidious_companion.sample, + invidious_companion = CONFIG.invidious_companion.sample + response = make_client(invidious_companion.private_url, &.post(endpoint, headers: headers, body: data.to_json)) body = response.body if (response.status_code != 200) - raise Exception.new("Error while communicating with Invidious companion: \ - status code: " + response.status_code.to_s + " and body: " + body) + raise Exception.new( + "Error while communicating with Invidious companion: \ + status code: #{response.status_code} and body: #{body.dump}" + ) end rescue ex raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end - if body.nil? - raise InfoException.new("Error while communicating with Invidious companion: no response data.") - end - # Convert result to Hash initial_data = JSON.parse(body).as_h From d7fe9114541389581a4cc191e16ad8c92d22d268 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun, 8 Dec 2024 22:22:08 +0100 Subject: [PATCH 18/30] format watch.cr --- src/invidious/routes/watch.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 9a498265..0b9d0e50 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -322,7 +322,6 @@ module Invidious::Routes::Watch return Invidious::Routes::API::V1::Videos.captions(env) elsif itag = download_widget["itag"]?.try &.as_i.to_s - # URL params specific to /latest_version env.params.query["id"] = video_id env.params.query["title"] = filename From b4910f4e5675cc9539c7dbe33d15b4b27725fe1a Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun, 8 Dec 2024 22:24:57 +0100 Subject: [PATCH 19/30] fix ameba Redundant use of `Object#to_s` in interpolation --- src/invidious/routes/api/manifest.cr | 2 +- src/invidious/routes/video_playback.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 17c3bcb6..d37a95aa 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -10,7 +10,7 @@ module Invidious::Routes::API::Manifest if !CONFIG.invidious_companion.empty? invidious_companion = CONFIG.invidious_companion.sample - return env.redirect "#{invidious_companion.public_url.to_s}/api/manifest/dash/id/#{id}?#{env.params.query}" + return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" end # Since some implementations create playlists based on resolution regardless of different codecs, diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index b04868df..c33d010b 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -258,7 +258,7 @@ module Invidious::Routes::VideoPlayback def self.latest_version(env) if !CONFIG.invidious_companion.empty? invidious_companion = CONFIG.invidious_companion.sample - return env.redirect "#{invidious_companion.public_url.to_s}/latest_version?#{env.params.query}" + return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" end id = env.params.query["id"]? From 847d4b55fdea3a01858984c8defba6c8e55e5687 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:08:57 +0100 Subject: [PATCH 20/30] add ability for invidious companion to check request from invidious --- config/config.example.yml | 1 + src/invidious/config.cr | 3 +++ src/invidious/helpers/utils.cr | 19 +++++++++++++++++++ src/invidious/views/components/player.ecr | 9 ++++++--- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 593ca192..9adea976 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -83,6 +83,7 @@ db: ## ## API key for Invidious companion +## The size of the key needs to be more or equal to 16. ## ## Needed when invidious_companion is configured ## diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 5e7eb006..da9cef08 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -268,6 +268,9 @@ class Config elsif config.invidious_companion_key == "CHANGE_ME!!" puts "Config: The value of 'invidious_companion_key' needs to be changed!!" exit(1) + elsif config.invidious_companion_key.size < 16 + puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 or more." + exit(1) end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 4d9bb28d..85462eb8 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -383,3 +383,22 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String) end return text end + +def encrypt_ecb_without_salt(data, key) + cipher = OpenSSL::Cipher.new("aes-128-ecb") + cipher.encrypt + cipher.key = key + + io = IO::Memory.new + io.write(cipher.update(data)) + io.write(cipher.final) + io.rewind + + return io +end + +def invidious_companion_encrypt(data) + timestamp = Time.utc.to_unix + encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key) + return Base64.urlsafe_encode(encrypted_data) +end diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index cc7ffeaa..5789e801 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,7 +22,8 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) + src_url = video.invidious_companion["baseUrl"].as_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (!CONFIG.invidious_companion.empty?) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -37,7 +38,8 @@ <% else %> <% if params.quality == "dash" src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) + src_url = video.invidious_companion["baseUrl"].as_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (!CONFIG.invidious_companion.empty?) %> <% end %> @@ -48,7 +50,8 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) + src_url = video.invidious_companion["baseUrl"].as_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (!CONFIG.invidious_companion.empty?) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) From 74bf68404dcb6f04ac981f1dd90d7fb806b35373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Tue, 24 Dec 2024 16:18:58 +0000 Subject: [PATCH 21/30] Better document private_url and public_url --- config/config.example.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/config.example.yml b/config/config.example.yml index 9adea976..c17a73fd 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -71,8 +71,14 @@ db: ## communication between the companion and Invidious. ## And public_url is the public URL from which companion is listening ## to the requests from the user(s). +## +## If you are using a reverse proxy then you will probably need to +## configure the public_url to be the same as the domain used for Invidious. +## Also apply when used from an external IP address (without a domain). +## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282 +## ## Both parameter can have identical URL when Invidious is hosted in -## an internal network or at home. +## an internal network or at home or locally (localhost). ## ## Accepted values: "http(s)://:" ## Default: From 5beb2a852807342162d94a5c5b55e269ba3d2099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Tue, 24 Dec 2024 17:43:54 +0000 Subject: [PATCH 22/30] Better doc for invidious_companion_key --- config/config.example.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index c17a73fd..8484c6be 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -88,15 +88,18 @@ db: # public_url: "http://localhost:8282" ## -## API key for Invidious companion +## API key for Invidious companion, used for securing the communication +## between Invidious and Invidious companion. ## The size of the key needs to be more or equal to 16. ## -## Needed when invidious_companion is configured +## Note: This parameter is mandatory when Invidious companion is enabled +## and should be a random string. +## Such random string can be generated on linux with the following +## command: `pwgen 16 1` ## -## Accepted values: "http(s)://:" +## Accepted values: a string ## Default: ## - #invidious_companion_key: "CHANGE_ME!!" ######################################### From da5f95909ed263254a1571836ff2b560a57c2b63 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 30 Dec 2024 21:52:07 +0100 Subject: [PATCH 23/30] !empty? to present? --- src/invidious/config.cr | 5 ++--- src/invidious/routes/api/manifest.cr | 2 +- src/invidious/routes/video_playback.cr | 2 +- src/invidious/routes/watch.cr | 2 +- src/invidious/videos/parser.cr | 2 +- src/invidious/views/components/player.ecr | 6 +++--- src/invidious/yt_backend/youtube_api.cr | 4 ++-- 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index da9cef08..2a16a76f 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -256,13 +256,12 @@ class Config end {% end %} - if !config.invidious_companion.empty? + if config.invidious_companion.present? # invidious_companion and signature_server can't work together if config.signature_server puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." exit(1) - end - if config.invidious_companion_key.empty? + elsif config.invidious_companion_key.empty? puts "Config: Please configure a key if you are using invidious companion." exit(1) elsif config.invidious_companion_key == "CHANGE_ME!!" diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index d37a95aa..c27caad7 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? - if !CONFIG.invidious_companion.empty? + if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index c33d010b..b1c788c2 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -256,7 +256,7 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) - if !CONFIG.invidious_companion.empty? + if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 0b9d0e50..50b5a25d 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -327,7 +327,7 @@ module Invidious::Routes::Watch env.params.query["title"] = filename env.params.query["local"] = "true" - if (!CONFIG.invidious_companion.empty?) + if (CONFIG.invidious_companion.present?) video = get_video(video_id) return env.redirect "#{video.invidious_companion["baseUrl"].as_s}/latest_version?#{env.params.query}" else diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index c89fde1e..afbd9e00 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -108,7 +108,7 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - if !CONFIG.invidious_companion.empty? + if CONFIG.invidious_companion.present? new_player_response = nil # Don't use Android test suite client if po_token is passed because po_token doesn't diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 5789e801..32badc10 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -23,7 +23,7 @@ src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local src_url = video.invidious_companion["baseUrl"].as_s + src_url + - "&check=#{invidious_companion_encrypt(video.id)}" if (!CONFIG.invidious_companion.empty?) + "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -39,7 +39,7 @@ <% if params.quality == "dash" src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" src_url = video.invidious_companion["baseUrl"].as_s + src_url + - "&check=#{invidious_companion_encrypt(video.id)}" if (!CONFIG.invidious_companion.empty?) + "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) %> <% end %> @@ -51,7 +51,7 @@ src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local src_url = video.invidious_companion["baseUrl"].as_s + src_url + - "&check=#{invidious_companion_encrypt(video.id)}" if (!CONFIG.invidious_companion.empty?) + "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 6b382820..daff6c9c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -500,7 +500,7 @@ module YoutubeAPI data["params"] = params end - if !CONFIG.invidious_companion.empty? + if CONFIG.invidious_companion.present? return self._post_invidious_companion("/youtubei/v1/player", data) else return self._post_json("/youtubei/v1/player", data, client_config) @@ -696,7 +696,7 @@ module YoutubeAPI begin invidious_companion = CONFIG.invidious_companion.sample - response = make_client(invidious_companion.private_url, + response = make_client(invidious_companion.private_url, use_http_proxy: false &.post(endpoint, headers: headers, body: data.to_json)) body = response.body if (response.status_code != 200) From 020de0857543b00b5e99a00977ee5923aa52a6d1 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 30 Dec 2024 21:52:34 +0100 Subject: [PATCH 24/30] skip proxy for invidious companion --- src/invidious/yt_backend/connection_pool.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index c4a73aa7..060e5e2b 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -61,9 +61,9 @@ def add_yt_headers(request) end end -def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true) client = HTTP::Client.new(url) - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy # Force the usage of a specific configured IP Family if force_resolve @@ -78,8 +78,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you return client end -def make_client(url : URI, region = nil, force_resolve : Bool = false, &) - client = make_client(url, region, force_resolve: force_resolve) +def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &) + client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy) begin yield client ensure From 5e6cee62d08c42c72eaaaf88a0160c9755fe96f1 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:19:45 +0100 Subject: [PATCH 25/30] fixing format --- src/invidious/yt_backend/youtube_api.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index daff6c9c..1a907acf 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -696,8 +696,8 @@ module YoutubeAPI begin invidious_companion = CONFIG.invidious_companion.sample - response = make_client(invidious_companion.private_url, use_http_proxy: false - &.post(endpoint, headers: headers, body: data.to_json)) + response = make_client(invidious_companion.private_url, use_http_proxy: false &.post(endpoint, headers: headers, body: data.to_json) + ) body = response.body if (response.status_code != 200) raise Exception.new( From 568081b6bf89b6e1a1c10906f69d23719bfddd64 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:25:13 +0100 Subject: [PATCH 26/30] missing , --- src/invidious/yt_backend/youtube_api.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 1a907acf..58e100d8 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -696,8 +696,8 @@ module YoutubeAPI begin invidious_companion = CONFIG.invidious_companion.sample - response = make_client(invidious_companion.private_url, use_http_proxy: false &.post(endpoint, headers: headers, body: data.to_json) - ) + response = make_client(invidious_companion.private_url, use_http_proxy: false, + &.post(endpoint, headers: headers, body: data.to_json)) body = response.body if (response.status_code != 200) raise Exception.new( From e57f292f2f8c05dcdf01fced0cc9e6d1fb53e2d6 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun, 2 Mar 2025 23:30:42 +0100 Subject: [PATCH 27/30] add companion pooling http --- src/invidious.cr | 4 +++ src/invidious/yt_backend/connection_pool.cr | 39 +++++++++++++++++++++ src/invidious/yt_backend/youtube_api.cr | 6 ++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 12ffad33..d3300ece 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -97,6 +97,10 @@ YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) +COMPANION_POOL = CompanionConnectionPool.new( + capacity: CONFIG.pool_size +) + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 060e5e2b..c41cfd0a 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -46,6 +46,45 @@ struct YoutubeConnectionPool end end +struct CompanionConnectionPool + property pool : DB::Pool(HTTP::Client) + + def initialize(capacity = 5, timeout = 5.0) + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + @pool = DB::Pool(HTTP::Client).new(options) do + companion = CONFIG.invidious_companion.sample + next make_client(companion.private_url, force_resolve: true) + end + end + + def client(&) + conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + + begin + response = yield conn + rescue ex + conn.close + + companion = CONFIG.invidious_companion.sample + conn = make_client(companion.private_url, force_resolve: true) + + response = yield conn + ensure + pool.release(conn) + end + + response + end +end + def add_yt_headers(request) request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 58e100d8..b40092a1 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -681,7 +681,7 @@ module YoutubeAPI # def _post_invidious_companion( endpoint : String, - data : Hash + data : Hash, ) : Hash(String, JSON::Any) headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", @@ -695,9 +695,7 @@ module YoutubeAPI # Send the POST request begin - invidious_companion = CONFIG.invidious_companion.sample - response = make_client(invidious_companion.private_url, use_http_proxy: false, - &.post(endpoint, headers: headers, body: data.to_json)) + response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json) body = response.body if (response.status_code != 200) raise Exception.new( From 48ddd7ea8624a663f0f85102506ede12a1d8c91b Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 8 Mar 2025 20:58:34 +0100 Subject: [PATCH 28/30] fix: don't use http proxy when sending requests to companion --- src/invidious/yt_backend/connection_pool.cr | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index c41cfd0a..0daed46c 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -59,14 +59,12 @@ struct CompanionConnectionPool @pool = DB::Pool(HTTP::Client).new(options) do companion = CONFIG.invidious_companion.sample - next make_client(companion.private_url, force_resolve: true) + next make_client(companion.private_url, use_http_proxy: false) end end def client(&) conn = pool.checkout - # Proxy needs to be reinstated every time we get a client from the pool - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy begin response = yield conn @@ -74,7 +72,7 @@ struct CompanionConnectionPool conn.close companion = CONFIG.invidious_companion.sample - conn = make_client(companion.private_url, force_resolve: true) + conn = make_client(companion.private_url, use_http_proxy: false) response = yield conn ensure From 790eba42fbfece5793dd21bfd88ac647f9375a78 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 8 Mar 2025 23:35:54 +0100 Subject: [PATCH 29/30] fix: logic where we want to have the invidious logic if companion is not used --- src/invidious/videos/parser.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index afbd9e00..2fafe780 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -108,7 +108,7 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - if CONFIG.invidious_companion.present? + if !CONFIG.invidious_companion.present? new_player_response = nil # Don't use Android test suite client if po_token is passed because po_token doesn't From acd5e0cae539fff0873109afe010395166947f81 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 8 Mar 2025 23:38:05 +0100 Subject: [PATCH 30/30] chore: remove baseurl usage from invidious companion --- src/invidious/routes/embed.cr | 7 ++++--- src/invidious/routes/watch.cr | 10 ++++++---- src/invidious/videos.cr | 4 ---- src/invidious/videos/parser.cr | 2 +- src/invidious/views/components/player.ecr | 12 ++++++------ 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 49aeade5..bdbb2d89 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -203,11 +203,12 @@ module Invidious::Routes::Embed return env.redirect url end - if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{companion_base_url}") - .gsub("connect-src", "connect-src #{companion_base_url}") + .gsub("media-src", "media-src #{invidious_companion.public_url}") + .gsub("connect-src", "connect-src #{invidious_companion.public_url}") end rendered "embed" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 50b5a25d..ab588ad6 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -192,11 +192,12 @@ module Invidious::Routes::Watch captions: video.captions ) - if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{companion_base_url}") - .gsub("connect-src", "connect-src #{companion_base_url}") + .gsub("media-src", "media-src #{invidious_companion.public_url}") + .gsub("connect-src", "connect-src #{invidious_companion.public_url}") end templated "watch" @@ -329,7 +330,8 @@ module Invidious::Routes::Watch if (CONFIG.invidious_companion.present?) video = get_video(video_id) - return env.redirect "#{video.invidious_companion["baseUrl"].as_s}/latest_version?#{env.params.query}" + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" else return Invidious::Routes::VideoPlayback.latest_version(env) end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 95abd709..348a0a66 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -192,10 +192,6 @@ struct Video } end - def invidious_companion : Hash(String, JSON::Any)? - info["invidiousCompanion"]?.try &.as_h || {} of String => JSON::Any - end - # Macros defining getters/setters for various types of data private macro getset_string(name) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 2fafe780..26d74f37 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -133,7 +133,7 @@ def extract_video_info(video_id : String) end end - {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| params[f] = player_response[f] if player_response[f]? end diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 32badc10..af352102 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,8 +22,8 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url + - "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) + src_url = invidious_companion.public_url.to_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -38,8 +38,8 @@ <% else %> <% if params.quality == "dash" src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" - src_url = video.invidious_companion["baseUrl"].as_s + src_url + - "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) + src_url = invidious_companion.public_url.to_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion) %> <% end %> @@ -50,8 +50,8 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url + - "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) + src_url = invidious_companion.public_url.to_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s)