From 8087e64dfe8353bd7ee224b1985fcb1d8a47122c Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:22:59 +0000 Subject: [PATCH 1/7] Bump postgre version in dev compose (#4203) Fix postgre ver diff between dev and prod compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6a854475..d879919a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,7 @@ services: - invidious-db invidious-db: - image: docker.io/library/postgres:13 + image: docker.io/library/postgres:14 restart: unless-stopped volumes: - postgresdata:/var/lib/postgresql/data From db3c57d49f35ee03a634ddd41dc0c0f19dfb4f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Thu, 2 Nov 2023 23:35:50 +0100 Subject: [PATCH 2/7] force resolve for everything (#4225) --- src/invidious/yt_backend/connection_pool.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index e9eb726c..03ff0ee4 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -37,7 +37,7 @@ struct YoutubeConnectionPool conn.close conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn @@ -52,7 +52,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET + conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn @@ -62,7 +62,7 @@ end def make_client(url : URI, region = nil) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.family = CONFIG.force_resolve client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds From 438467f69a20007cace4f300d03138b7d2d7e79a Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Sat, 4 Nov 2023 13:52:30 +0000 Subject: [PATCH 3/7] Add playback success rate to `/api/v1/stats` (#4085) * Add stats-based /videoplayback blockage status * Count when YouTube returns wrong video as failure * Cast playback stats hash type prior to return * Bump stats refresh timer to 10 minutes --- src/invidious/helpers/helpers.cr | 17 +++++++++++++++++ src/invidious/jobs/statistics_refresh_job.cr | 12 +++++++++++- src/invidious/routes/api/v1/misc.cr | 16 ++++++++++++++++ src/invidious/routes/video_playback.cr | 5 +++++ src/invidious/videos/parser.cr | 5 +++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 23ff0da9..6dc9860e 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -208,3 +208,20 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end + +# Fetch the playback requests tracker from the statistics endpoint. +# +# Creates a new tracker when unavailable. +def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end + + return tracker.as(Hash(String, Int64 | Float64)) +end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index a113bd77..72d1ce88 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => 0_i64, }, + + # + # "totalRequests" => 0_i64, + # "successfulRequests" => 0_i64 + # "ratio" => 0_i64 + # + "playback" => {} of String => Int64 | Float64, } private getter db : DB::Database @@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob loop do refresh_stats - sleep 1.minute + sleep 10.minute Fiber.yield end end @@ -56,5 +63,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } + + # Reset playback requests tracker + STATISTICS["playback"] = {} of String => Int64 | Float64 end end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 8a92e160..b42ecd1a 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc if !CONFIG.statistics_enabled return {"software" => SOFTWARE}.to_json else + # Calculate playback success rate + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?) + tracker = tracker.as(Hash(String, Int64 | Float64)) + + if !tracker.empty? + total_requests = tracker["totalRequests"] + success_count = tracker["successfulRequests"] + + if total_requests.zero? + tracker["ratio"] = 1_i64 + else + tracker["ratio"] = (success_count / (total_requests)).round(2) + end + end + end + return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 9641e01a..1d5aa914 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback # Remove the Range header added previously. headers.delete("Range") if range_header.nil? + playback_statistics = get_playback_statistic() + playback_statistics["totalRequests"] += 1 + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code + else + playback_statistics["successfulRequests"] += 1 end if url.includes? "&file=seg.ts" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 551ce2cb..77520dbe 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 # Line to be reverted if one day we solve the video not available issue. + + # Although technically not a call to /videoplayback the fact that YouTube is returning the + # wrong video means that we should count it as a failure. + get_playback_statistic()["totalRequests"] += 1 + return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."), From c31908a011f5ac460de7c816f8861a6ec3cb6692 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 7 Nov 2023 22:19:56 -0800 Subject: [PATCH 4/7] Update params protobuf for channel about tab --- src/invidious/channels/about.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 0054f8f2..894670dd 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -18,8 +18,8 @@ record AboutChannel, def get_about_info(ucid, locale) : AboutChannel begin - # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} - initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") + # "EgVhYm91dPIGBAoCEgA=" is the base64-encoded protobuf object {"2:0:string":"about","110:1:embedded":{"1:0:embedded":{"2:0:string":""}}} + initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dPIGBAoCEgA=") rescue raise InfoException.new("Could not get channel info.") end From ab4c0a1d3c5af5c1ba50364c04e1a996dbabc152 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 7 Nov 2023 23:45:44 -0800 Subject: [PATCH 5/7] Fetch channel info from home tab instead of about --- src/invidious/channels/about.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 894670dd..8b60a728 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -18,8 +18,8 @@ record AboutChannel, def get_about_info(ucid, locale) : AboutChannel begin - # "EgVhYm91dPIGBAoCEgA=" is the base64-encoded protobuf object {"2:0:string":"about","110:1:embedded":{"1:0:embedded":{"2:0:string":""}}} - initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dPIGBAoCEgA=") + # Fetch channel information from channel home page + initdata = YoutubeAPI.browse(browse_id: ucid, params: "") rescue raise InfoException.new("Could not get channel info.") end From 9ce9c543992243737516750bf08f5d073e899715 Mon Sep 17 00:00:00 2001 From: Timothy Redaelli Date: Tue, 21 Nov 2023 20:47:07 +0100 Subject: [PATCH 6/7] Install rsvg-convert on docker images (#4230) (#4282) In #4103 alpine was updated from 3.16 to 3.18, but in 3.17 librsvg packages was splitted and rsvg-convert is on his own package. --- docker/Dockerfile | 2 +- docker/Dockerfile.arm64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c9644ca6..ace096bf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \ fi FROM alpine:3.18 -RUN apk add --no-cache librsvg ttf-opensans tini +RUN apk add --no-cache rsvg-convert ttf-opensans tini WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index d9a4eeaf..602f3ab2 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \ fi FROM alpine:3.18 -RUN apk add --no-cache librsvg ttf-opensans tini +RUN apk add --no-cache rsvg-convert ttf-opensans tini WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious From d76fed585003728ea759767d8e5e24fe918118f5 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Fri, 1 Dec 2023 23:11:45 +0500 Subject: [PATCH 7/7] ci: update github actions (#4209) * ci: update actions/checkout to v4 * ci: remove patch version from crystal-lang/install-crystal latest patch will be automatically installed which is 1.8.1 https://github.com/crystal-lang/install-crystal * ci: update docker/setup-buildx-action to v3 * ci: update docker/build-push-action to v5 * ci: update docker/setup-qemu-action to v3 * ci: update docker/login-action to v3 * ci: actions/stale to v8 https://github.com/actions/stale/blob/main/CHANGELOG.md * Revert "ci: remove patch version from crystal-lang/install-crystal" This reverts commit 8c39ce846fa0a7057d66dc25b853c40b2366faf3. --- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/container-release.yml | 12 ++++++------ .github/workflows/stale.yml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ca0dc96..7fb7ffa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: stable: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build Docker run: docker-compose build --build-arg release=0 @@ -103,18 +103,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build Docker ARM64 image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile.arm64 diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index c2756fcc..8b3202d7 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Crystal uses: crystal-lang/install-crystal@v1.8.0 @@ -38,15 +38,15 @@ jobs: fi - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} @@ -54,7 +54,7 @@ jobs: - name: Build and push Docker AMD64 image for Push Event if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile @@ -67,7 +67,7 @@ jobs: - name: Build and push Docker ARM64 image for Push Event if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile.arm64 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a7e218a2..b25199e3 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v5 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 365