Merge 1e925e97d87ac1a80650bdf02deb44aeca215698 into edcc1554827aa3d0f22c39765ec03d5b5cd01421

config/sql/channel_continuations.sql

@@ -0,0 +1,23 @@
+-- Table: public.channel_continuations
+
+-- DROP TABLE public.channel_continuations;
+
+CREATE TABLE IF NOT EXISTS public.channel_continuations
+(
+  id text NOT NULL,
+  page integer NOT NULL,
+  sort_by text NOT NULL,
+  continuation text,
+  CONSTRAINT channel_continuations_id_page_sort_by_key UNIQUE (id, page, sort_by)
+);
+
+GRANT ALL ON TABLE public.channel_continuations TO default_user;
+
+-- Index: public.channel_continuations_id_idx
+
+-- DROP INDEX public.channel_continuations_id_idx;
+
+CREATE INDEX IF NOT EXISTS channel_continuations_id_idx
+  ON public.channel_continuations
+  USING btree
+  (id COLLATE pg_catalog."default");

docker/init-invidious-db.sh

@@ -3,6 +3,7 @@ set -eou pipefail
 
 psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channels.sql
 psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/videos.sql
+psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channel_continuations.sql
 psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channel_videos.sql
 psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/users.sql
 psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/session_ids.sql

src/invidious.cr

@@ -115,6 +115,7 @@ if CONFIG.check_tables
   check_enum(PG_DB, "privacy", PlaylistPrivacy)
 
   check_table(PG_DB, "channels", InvidiousChannel)
+  check_table(PG_DB, "channel_continuations", ChannelContinuation)
   check_table(PG_DB, "channel_videos", ChannelVideo)
   check_table(PG_DB, "playlists", InvidiousPlaylist)
   check_table(PG_DB, "playlist_videos", PlaylistVideo)

src/invidious/channels/channels.cr

@@ -8,6 +8,23 @@ struct InvidiousChannel
   property subscribed : Time?
 end
 
+struct ChannelContinuation
+  include DB::Serializable
+
+  property id : String
+  property page : Int32
+  property sort_by : String
+  property continuation : String
+
+  def to_tuple
+    {% begin %}
+      {
+        {{*@type.instance_vars.map(&.name)}}
+      }
+    {% end %}
+  end
+end
+
 struct ChannelVideo
   include DB::Serializable
 

src/invidious/channels/videos.cr

@@ -58,10 +58,79 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
 end
 
 def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
-  continuation = produce_channel_videos_continuation(ucid, page,
-    auto_generated: auto_generated, sort_by: sort_by, v2: true)
+  continuation = ""
+  initial_data = Hash(String, JSON::Any).new
 
-  return YoutubeAPI.browse(continuation)
+  # Manually generating the continuation works correctly for both 'newest' and 'popular' sort modes,
+  # and for page 1 when sorting by 'oldest'. So only fallback to using the db if not in either of these states.
+  if sort_by != "oldest" || page == 1
+    continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true)
+  elsif channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, page, sort_by, as: ChannelContinuation)
+    continuation = channel_continuation.continuation
+  else
+    # This branch should not be needed in normal operation (navigating via the previous/next page buttons).
+    # This is just here as a fallback in case someone requests, for example, page 3 without previously requesting page 2.
+
+    # Iterate backwards from the wanted page to page 2 to find a stored continuation.
+    start = 1
+    ((page - 1)..2).each do |i|
+      if channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, i, sort_by, as: ChannelContinuation)
+        start = i
+        continuation = channel_continuation.continuation
+        break
+      end
+    end
+
+    # If a continuation hasn't been found after getting to page 2, manually create the continuation for page 1.
+    if start == 1
+      continuation = produce_channel_videos_continuation(ucid, 1, auto_generated: auto_generated, sort_by: sort_by, v2: true)
+    end
+
+    # Iterate from the found/created continuation until we have the continuation for the wanted page or there are no more pages.
+    # Store the returned continuation each time so that it can be found in the db next time the current page is wanted.
+    (start..(page - 1)).each do |i|
+      initial_data = YoutubeAPI.browse(continuation)
+      continuation = fetch_continuation_token(initial_data)
+
+      break if continuation.nil? || continuation.empty?
+
+      channel_continuation = ChannelContinuation.new({
+        id:           ucid,
+        page:         i,
+        sort_by:      sort_by,
+        continuation: continuation,
+      })
+      PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \
+        ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple)
+    end
+  end
+
+  # If we reached the channel's last page in the else loop above return an empty hash.
+  if continuation.nil? || continuation.empty?
+    initial_data.clear
+  else
+    # Get the wanted page and store the returned continuation for the next page,
+    # if there is one, so that it can be used the next time this function is called requesting that page.
+    initial_data = YoutubeAPI.browse(continuation)
+
+    # Only get the continuation and store it if the sort mode is 'oldest'.
+    if sort_by == "oldest"
+      continuation = fetch_continuation_token(initial_data)
+
+      if !continuation.nil? && !continuation.empty?
+        channel_continuation = ChannelContinuation.new({
+          id:           ucid,
+          page:         page + 1,
+          sort_by:      sort_by,
+          continuation: continuation,
+        })
+        PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \
+         ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple)
+      end
+    end
+  end
+
+  return initial_data
 end
 
 def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")

src/invidious/yt_backend/extractors_utils.cr

@@ -55,13 +55,20 @@ def fetch_continuation_token(items : Array(JSON::Any))
 end
 
 def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
+  continuation = ""
   # Fetches the continuation token from initial data
   if initial_data["onResponseReceivedActions"]?
     continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
-  else
+  elsif initial_data["contents"]?
     tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
     continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
+  elsif initial_data["continuationContents"]["gridContinuation"]["continuations"]?
+    continuation = initial_data["continuationContents"]["gridContinuation"]["continuations"][0]["nextContinuationData"]["continuation"].as_s
   end
 
-  return fetch_continuation_token(continuation_items.as_a)
+  if !continuation_items.nil?
+    continuation = fetch_continuation_token(continuation_items.as_a)
+  end
+
+  return continuation
 end