Merge 2d1437f6ff into 53e8a5d62d

* Update to latest version WEB_CREATOR

* fix comment about using web embed as a fallback

.github/CODEOWNERS

@@ -6,7 +6,7 @@ docker/ @unixfox
 kubernetes/ @unixfox
 
 README.md @thefrenchghosty
-config/config.example.yml @thefrenchghosty @SamantazFox @unixfox
+config/config.example.yml @SamantazFox @unixfox
 
 scripts/ @syeopite
 shards.lock @syeopite

src/invidious/user/imports.cr

@@ -290,42 +290,39 @@ struct Invidious::User
     end
 
     def from_newpipe(user : User, body : String) : Bool
-      io = IO::Memory.new(body)
+      Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
+        entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
+        return false if entry.nil?
+        entry.open do |file_io|
+          # Ensure max size of 4MB
+          io_sized = IO::Sized.new(file_io, 0x400000)
 
-      Compress::Zip::File.open(io) do |file|
+          begin
-        file.entries.each do |entry|
+            temp = File.tempfile(".db") do |tempfile|
-          entry.open do |file_io|
+              begin
-            # Ensure max size of 4MB
+                File.write(tempfile.path, io_sized.gets_to_end)
-            io_sized = IO::Sized.new(file_io, 0x400000)
+              rescue
+                return false
+              end
 
-            next if entry.filename != "newpipe.db"
+              DB.open("sqlite3://" + tempfile.path) do |db|
+                user.watched += db.query_all("SELECT url FROM streams", as: String)
+                  .map(&.lchop("https://www.youtube.com/watch?v="))
 
-            tempfile = File.tempfile(".db")
+                user.watched.uniq!
+                Invidious::Database::Users.update_watch_history(user)
 
-            begin
+                user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
-              File.write(tempfile.path, io_sized.gets_to_end)
+                  .map(&.lchop("https://www.youtube.com/channel/"))
-            rescue
+
-              return false
+                user.subscriptions.uniq!
+                user.subscriptions = get_batch_channels(user.subscriptions)
+
+                Invidious::Database::Users.update_subscriptions(user)
+              end
             end
-
+          ensure
-            db = DB.open("sqlite3://" + tempfile.path)
+            temp.delete if !temp.nil?
-
-            user.watched += db.query_all("SELECT url FROM streams", as: String)
-              .map(&.lchop("https://www.youtube.com/watch?v="))
-
-            user.watched.uniq!
-            Invidious::Database::Users.update_watch_history(user)
-
-            user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
-              .map(&.lchop("https://www.youtube.com/channel/"))
-
-            user.subscriptions.uniq!
-            user.subscriptions = get_batch_channels(user.subscriptions)
-
-            Invidious::Database::Users.update_subscriptions(user)
-
-            db.close
-            tempfile.delete
           end
         end
       end

src/invidious/videos/parser.cr

@@ -53,6 +53,10 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  if CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
@@ -102,6 +106,13 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
+  # Second try in case WEB_CREATOR doesn't work with po_token.
+  # Only trigger if reason found and po_token configured.
+  if reason && CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
   if reason.nil? && CONFIG.po_token.nil?
@@ -114,10 +125,9 @@ def extract_video_info(video_id : String)
   end
 
   # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
+  # Only trigger if reason found or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
+  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
-  # if the IP address is not blocked.
+  if reason && CONFIG.po_token.nil?
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -185,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
-  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+  if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+    microformat = {} of String => JSON::Any
+  end
 
   raise BrokenTubeException.new("videoDetails") if !video_details
-  raise BrokenTubeException.new("microformat") if !microformat
 
   # Basic video infos
 
@@ -225,7 +236,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
-  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  family_friendly = microformat["isFamilySafe"]?.try &.as_bool
   is_listed = video_details["isCrawlable"]?.try &.as_bool
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 

src/invidious/yt_backend/youtube_api.cr

@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -80,6 +81,14 @@ module YoutubeAPI
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20240918.03.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -291,8 +300,9 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
+      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+        "embedUrl" => "https://www.google.com/",
       } of String => String | Int64
     end