From db3c57d49f35ee03a634ddd41dc0c0f19dfb4f5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89milien=20=28perso=29?=
<4016501+unixfox@users.noreply.github.com>
Date: Thu, 2 Nov 2023 23:35:50 +0100
Subject: [PATCH 1/4] force resolve for everything (#4225)
---
src/invidious/yt_backend/connection_pool.cr | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index e9eb726c..03ff0ee4 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -37,7 +37,7 @@ struct YoutubeConnectionPool
conn.close
conn = HTTP::Client.new(url)
- conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
+ conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
response = yield conn
@@ -52,7 +52,7 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
conn = HTTP::Client.new(url)
- conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
+ conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
conn
@@ -62,7 +62,7 @@ end
def make_client(url : URI, region = nil)
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
- client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
+ client.family = CONFIG.force_resolve
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
From 438467f69a20007cace4f300d03138b7d2d7e79a Mon Sep 17 00:00:00 2001
From: syeopite <70992037+syeopite@users.noreply.github.com>
Date: Sat, 4 Nov 2023 13:52:30 +0000
Subject: [PATCH 2/4] Add playback success rate to `/api/v1/stats` (#4085)
* Add stats-based /videoplayback blockage status
* Count when YouTube returns wrong video as failure
* Cast playback stats hash type prior to return
* Bump stats refresh timer to 10 minutes
---
src/invidious/helpers/helpers.cr | 17 +++++++++++++++++
src/invidious/jobs/statistics_refresh_job.cr | 12 +++++++++++-
src/invidious/routes/api/v1/misc.cr | 16 ++++++++++++++++
src/invidious/routes/video_playback.cr | 5 +++++
src/invidious/videos/parser.cr | 5 +++++
5 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 23ff0da9..6dc9860e 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -208,3 +208,20 @@ def proxy_file(response, env)
IO.copy response.body_io, env.response
end
end
+
+# Fetch the playback requests tracker from the statistics endpoint.
+#
+# Creates a new tracker when unavailable.
+def get_playback_statistic
+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
+ tracker = {
+ "totalRequests" => 0_i64,
+ "successfulRequests" => 0_i64,
+ "ratio" => 0_f64,
+ }
+
+ Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
+ end
+
+ return tracker.as(Hash(String, Int64 | Float64))
+end
diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr
index a113bd77..72d1ce88 100644
--- a/src/invidious/jobs/statistics_refresh_job.cr
+++ b/src/invidious/jobs/statistics_refresh_job.cr
@@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
+
+ #
+ # "totalRequests" => 0_i64,
+ # "successfulRequests" => 0_i64
+ # "ratio" => 0_i64
+ #
+ "playback" => {} of String => Int64 | Float64,
}
private getter db : DB::Database
@@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
loop do
refresh_stats
- sleep 1.minute
+ sleep 10.minute
Fiber.yield
end
end
@@ -56,5 +63,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
+
+ # Reset playback requests tracker
+ STATISTICS["playback"] = {} of String => Int64 | Float64
end
end
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 8a92e160..b42ecd1a 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
+ # Calculate playback success rate
+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
+ tracker = tracker.as(Hash(String, Int64 | Float64))
+
+ if !tracker.empty?
+ total_requests = tracker["totalRequests"]
+ success_count = tracker["successfulRequests"]
+
+ if total_requests.zero?
+ tracker["ratio"] = 1_i64
+ else
+ tracker["ratio"] = (success_count / (total_requests)).round(2)
+ end
+ end
+ end
+
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 9641e01a..1d5aa914 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
+ playback_statistics = get_playback_statistic()
+ playback_statistics["totalRequests"] += 1
+
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
+ else
+ playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index 551ce2cb..77520dbe 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
# Line to be reverted if one day we solve the video not available issue.
+
+ # Although technically not a call to /videoplayback the fact that YouTube is returning the
+ # wrong video means that we should count it as a failure.
+ get_playback_statistic()["totalRequests"] += 1
+
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."),
From c31908a011f5ac460de7c816f8861a6ec3cb6692 Mon Sep 17 00:00:00 2001
From: syeopite
Date: Tue, 7 Nov 2023 22:19:56 -0800
Subject: [PATCH 3/4] Update params protobuf for channel about tab
---
src/invidious/channels/about.cr | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 0054f8f2..894670dd 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -18,8 +18,8 @@ record AboutChannel,
def get_about_info(ucid, locale) : AboutChannel
begin
- # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
- initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==")
+ # "EgVhYm91dPIGBAoCEgA=" is the base64-encoded protobuf object {"2:0:string":"about","110:1:embedded":{"1:0:embedded":{"2:0:string":""}}}
+ initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dPIGBAoCEgA=")
rescue
raise InfoException.new("Could not get channel info.")
end
From ab4c0a1d3c5af5c1ba50364c04e1a996dbabc152 Mon Sep 17 00:00:00 2001
From: syeopite
Date: Tue, 7 Nov 2023 23:45:44 -0800
Subject: [PATCH 4/4] Fetch channel info from home tab instead of about
---
src/invidious/channels/about.cr | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 894670dd..8b60a728 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -18,8 +18,8 @@ record AboutChannel,
def get_about_info(ucid, locale) : AboutChannel
begin
- # "EgVhYm91dPIGBAoCEgA=" is the base64-encoded protobuf object {"2:0:string":"about","110:1:embedded":{"1:0:embedded":{"2:0:string":""}}}
- initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dPIGBAoCEgA=")
+ # Fetch channel information from channel home page
+ initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end