Merge branch 'iv-org:master' into main

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
             stable: false
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -87,7 +87,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build Docker
         run: docker-compose build --build-arg release=0
@@ -103,18 +103,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build Docker ARM64 image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker/Dockerfile.arm64

.github/workflows/container-release.yml

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Crystal
         uses: crystal-lang/install-crystal@v1.8.0
@@ -38,15 +38,15 @@ jobs:
           fi
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
@@ -54,7 +54,7 @@ jobs:
 
       - name: Build and push Docker AMD64 image for Push Event
         if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker/Dockerfile
@@ -67,7 +67,7 @@ jobs:
 
       - name: Build and push Docker ARM64 image for Push Event
         if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: docker/Dockerfile.arm64

.github/workflows/stale.yml

@@ -10,7 +10,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v5
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 365

docker-compose.yml

@@ -40,7 +40,7 @@ services:
       - invidious-db
 
   invidious-db:
-    image: docker.io/library/postgres:13
+    image: docker.io/library/postgres:14
     restart: unless-stopped
     volumes:
       - postgresdata:/var/lib/postgresql/data

docker/Dockerfile

@@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \
     fi
 
 FROM alpine:3.18
-RUN apk add --no-cache librsvg ttf-opensans tini
+RUN apk add --no-cache rsvg-convert ttf-opensans tini
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

docker/Dockerfile.arm64

@@ -33,7 +33,7 @@ RUN if [[ "${release}" == 1 ]] ; then \
     fi
 
 FROM alpine:3.18
-RUN apk add --no-cache librsvg ttf-opensans tini
+RUN apk add --no-cache rsvg-convert ttf-opensans tini
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

src/invidious/channels/about.cr

@@ -18,8 +18,8 @@ record AboutChannel,
 
 def get_about_info(ucid, locale) : AboutChannel
   begin
-    # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
-    initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==")
+    # Fetch channel information from channel home page
+    initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
   rescue
     raise InfoException.new("Could not get channel info.")
   end

src/invidious/helpers/helpers.cr

@@ -208,3 +208,20 @@ def proxy_file(response, env)
     IO.copy response.body_io, env.response
   end
 end
+
+# Fetch the playback requests tracker from the statistics endpoint.
+#
+# Creates a new tracker when unavailable.
+def get_playback_statistic
+  if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
+    tracker = {
+      "totalRequests"      => 0_i64,
+      "successfulRequests" => 0_i64,
+      "ratio"              => 0_f64,
+    }
+
+    Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
+  end
+
+  return tracker.as(Hash(String, Int64 | Float64))
+end

src/invidious/jobs/statistics_refresh_job.cr

@@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
       "updatedAt"              => Time.utc.to_unix,
       "lastChannelRefreshedAt" => 0_i64,
     },
+
+    #
+    #    "totalRequests" => 0_i64,
+    #    "successfulRequests" => 0_i64
+    #    "ratio"   => 0_i64
+    #
+    "playback" => {} of String => Int64 | Float64,
   }
 
   private getter db : DB::Database
@@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
 
     loop do
       refresh_stats
-      sleep 1.minute
+      sleep 10.minute
       Fiber.yield
     end
   end
@@ -56,5 +63,8 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
       "updatedAt"              => Time.utc.to_unix,
       "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
     }
+
+    # Reset playback requests tracker
+    STATISTICS["playback"] = {} of String => Int64 | Float64
   end
 end

src/invidious/routes/api/v1/misc.cr

@@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc
     if !CONFIG.statistics_enabled
       return {"software" => SOFTWARE}.to_json
     else
+      # Calculate playback success rate
+      if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
+        tracker = tracker.as(Hash(String, Int64 | Float64))
+
+        if !tracker.empty?
+          total_requests = tracker["totalRequests"]
+          success_count = tracker["successfulRequests"]
+
+          if total_requests.zero?
+            tracker["ratio"] = 1_i64
+          else
+            tracker["ratio"] = (success_count / (total_requests)).round(2)
+          end
+        end
+      end
+
       return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
     end
   end

src/invidious/routes/video_playback.cr

@@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback
     # Remove the Range header added previously.
     headers.delete("Range") if range_header.nil?
 
+    playback_statistics = get_playback_statistic()
+    playback_statistics["totalRequests"] += 1
+
     if response.status_code >= 400
       env.response.content_type = "text/plain"
       haltf env, response.status_code
+    else
+      playback_statistics["successfulRequests"] += 1
     end
 
     if url.includes? "&file=seg.ts"

src/invidious/videos/parser.cr

@@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
     # YouTube may return a different video player response than expected.
     # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
     # Line to be reverted if one day we solve the video not available issue.
+
+    # Although technically not a call to /videoplayback the fact that YouTube is returning the
+    # wrong video means that we should count it as a failure.
+    get_playback_statistic()["totalRequests"] += 1
+
     return {
       "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
       "reason"  => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"),

src/invidious/yt_backend/connection_pool.cr

@@ -37,7 +37,7 @@ struct YoutubeConnectionPool
         conn.close
         conn = HTTP::Client.new(url)
 
-        conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
+        conn.family = CONFIG.force_resolve
         conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
         conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
         response = yield conn
@@ -52,7 +52,7 @@ struct YoutubeConnectionPool
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
       conn = HTTP::Client.new(url)
-      conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
+      conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
       conn
@@ -62,7 +62,7 @@ end
 
 def make_client(url : URI, region = nil)
   client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
-  client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
+  client.family = CONFIG.force_resolve
   client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds