Merge 6a88c098f0 into 53e8a5d62d

* Update to latest version WEB_CREATOR

* fix comment about using web embed as a fallback

.github/CODEOWNERS

@@ -6,7 +6,7 @@ docker/ @unixfox
 kubernetes/ @unixfox
 
 README.md @thefrenchghosty
-config/config.example.yml @thefrenchghosty @SamantazFox @unixfox
+config/config.example.yml @SamantazFox @unixfox
 
 scripts/ @syeopite
 shards.lock @syeopite

src/invidious/jobs/statistics_refresh_job.cr

@@ -27,6 +27,12 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
     "playback" => {} of String => Int64 | Float64,
   }
 
+  # Latches the playback success stats from before statistics gets refreshed
+  # Used to ensure that the object won't get reset back to an empty object
+  LATCHED_PLAYBACK_STATS = {
+    "playback" => {} of String => Int64 | Float64,
+  }
+
   private getter db : DB::Database
 
   def initialize(@db, @software_config : Hash(String, String))
@@ -65,6 +71,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
     }
 
     # Reset playback requests tracker
+    LATCHED_PLAYBACK_STATS["playback"] = STATISTICS["playback"].as(Hash(String, Int64 | Float64))
     STATISTICS["playback"] = {} of String => Int64 | Float64
   end
 end

src/invidious/routes/api/v1/misc.cr

@@ -19,6 +19,8 @@ module Invidious::Routes::API::V1::Misc
           else
             tracker["ratio"] = (success_count / (total_requests)).round(2)
           end
+        else
+          return (Invidious::Jobs::StatisticsRefreshJob::STATISTICS.merge Invidious::Jobs::StatisticsRefreshJob::LATCHED_PLAYBACK_STATS).to_json
         end
       end
 

src/invidious/videos/parser.cr

@@ -53,6 +53,10 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  if CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
@@ -89,6 +93,14 @@ def extract_video_info(video_id : String)
     }
   else
     reason = nil
+
+    # Although technically not a call to /videoplayback, because we are counting requests
+    # in which YouTube returned a "this content is not available" video as a failure,
+    # we should also count requests that returned the correct video as a success
+    # in order to ensure a correct and accurate ratio
+    playback_stats = get_playback_statistic()
+    playback_stats["totalRequests"] += 1
+    playback_stats["successfulRequests"] += 1
   end
 
   # Don't fetch the next endpoint if the video is unavailable.
@@ -102,6 +114,13 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
+  # Second try in case WEB_CREATOR doesn't work with po_token.
+  # Only trigger if reason found and po_token configured.
+  if reason && CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
   if reason.nil? && CONFIG.po_token.nil?
@@ -114,10 +133,9 @@ def extract_video_info(video_id : String)
   end
 
   # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  # Only trigger if reason found or didn't work wth Android client.
+  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
+  if reason && CONFIG.po_token.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -185,10 +203,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
-  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+  if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+    microformat = {} of String => JSON::Any
+  end
 
   raise BrokenTubeException.new("videoDetails") if !video_details
-  raise BrokenTubeException.new("microformat") if !microformat
 
   # Basic video infos
 
@@ -225,7 +244,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
-  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  family_friendly = microformat["isFamilySafe"]?.try &.as_bool
   is_listed = video_details["isCrawlable"]?.try &.as_bool
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 

src/invidious/yt_backend/youtube_api.cr

@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -80,6 +81,14 @@ module YoutubeAPI
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20240918.03.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -291,8 +300,9 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
+      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+        "embedUrl" => "https://www.google.com/",
       } of String => String | Int64
     end