From 99bf5197812b470f17e66aec2ca6eccc55ca6e15 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 1 Dec 2022 20:09:31 +0100
Subject: [PATCH 01/17] shards: Bump protodec to v0.1.5
---
shard.lock | 2 +-
shard.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/shard.lock b/shard.lock
index cdce1160..235e4c25 100644
--- a/shard.lock
+++ b/shard.lock
@@ -34,7 +34,7 @@ shards:
protodec:
git: https://github.com/iv-org/protodec.git
- version: 0.1.4
+ version: 0.1.5
radix:
git: https://github.com/luislavena/radix.git
diff --git a/shard.yml b/shard.yml
index 9c9b0d37..7ee0bb2a 100644
--- a/shard.yml
+++ b/shard.yml
@@ -24,7 +24,7 @@ dependencies:
version: ~> 0.6.1
protodec:
github: iv-org/protodec
- version: ~> 0.1.4
+ version: ~> 0.1.5
lsquic:
github: iv-org/lsquic.cr
version: ~> 2.18.1-2
From fbcce57ce29b05c234c0c31b5f179d861e143260 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 11 Nov 2022 01:31:32 +0100
Subject: [PATCH 02/17] channel: use extractor utils to parse tabs (+ code
cleaning)
---
src/invidious/channels/about.cr | 54 ++++++++++++++++++---------------
1 file changed, 30 insertions(+), 24 deletions(-)
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 4c442959..bb9bd8c7 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -100,34 +100,40 @@ def get_about_info(ucid, locale) : AboutChannel
total_views = 0_i64
joined = Time.unix(0)
- tabs = [] of String
+ tab_names = [] of String
- tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
- if !tabs_json.nil?
- # Retrieve information from the tabs array. The index we are looking for varies between channels.
- tabs_json.each do |node|
- # Try to find the about section which is located in only one of the tabs.
- channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
- .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
- .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
+ if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
+ # Get the name of the tabs available on this channel
+ tab_names = tabs_json.as_a
+ .compact_map(&.dig?("tabRenderer", "title").try &.as_s.downcase)
- if !channel_about_meta.nil?
- total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+ # Get the currently active tab ("About")
+ about_tab = extract_selected_tab(tabs_json)
- # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
- joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
- .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+ # Try to find the about metadata section
+ channel_about_meta = about_tab.dig?(
+ "content",
+ "sectionListRenderer", "contents", 0,
+ "itemSectionRenderer", "contents", 0,
+ "channelAboutFullMetadataRenderer"
+ )
- # Normal Auto-generated channels
- # https://support.google.com/youtube/answer/2579942
- # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
- if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
- (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
- auto_generated = true
- end
- end
+ if !channel_about_meta.nil?
+ total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+
+ # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
+ joined = extract_text(channel_about_meta["joinedDateText"]?)
+ .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+
+ # Normal Auto-generated channels
+ # https://support.google.com/youtube/answer/2579942
+ # For auto-generated channels, channel_about_meta only has
+ # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
+ auto_generated = (
+ (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
+ extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube"
+ )
end
- tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase)
end
sub_count = initdata
@@ -148,7 +154,7 @@ def get_about_info(ucid, locale) : AboutChannel
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
- tabs: tabs,
+ tabs: tab_names,
verified: author_verified || false,
)
end
From 9588fcb5d1dd90e8591ed53a342727a0df6923c4 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 3 Dec 2022 22:39:24 +0100
Subject: [PATCH 03/17] frontend: remove paging on channel videos
---
src/invidious/routes/channels.cr | 5 +----
src/invidious/views/channel.ecr | 18 +++---------------
2 files changed, 4 insertions(+), 19 deletions(-)
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index c6e02cbd..f26f29f5 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -12,9 +12,6 @@ module Invidious::Routes::Channels
end
locale, user, subscriptions, continuation, ucid, channel = data
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
-
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
@@ -35,7 +32,7 @@ module Invidious::Routes::Channels
sort_options = {"newest", "oldest", "popular"}
sort_by ||= "newest"
- count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ count, items = get_60_videos(channel.ucid, channel.author, 1, channel.auto_generated, sort_by)
end
templated "channel"
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index dea86abe..878587d4 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -90,7 +90,7 @@
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
- <a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>">
+ <a href="/channel/<%= ucid %>?sort_by=<%= sort %>">
<%= translate(locale, sort) %>
</a>
<% end %>
@@ -111,19 +111,7 @@
</div>
<div class="pure-g h-box">
- <div class="pure-u-1 pure-u-lg-1-5">
- <% if page > 1 %>
- <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Previous page") %>
- </a>
- <% end %>
- </div>
+ <div class="pure-u-1 pure-u-lg-1-5"></div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
- <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
- <% if count == 60 %>
- <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Next page") %>
- </a>
- <% end %>
- </div>
+ <div class="pure-u-1 pure-u-lg-1-5"></div>
</div>
From bdc51cd20fd2df99c2fe5ddc281aada86000a783 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 10 Nov 2022 23:32:51 +0100
Subject: [PATCH 04/17] extractors: separate 'extract' and 'parse' logic
---
src/invidious/channels/playlists.cr | 2 +-
src/invidious/search/processors.cr | 2 +-
src/invidious/yt_backend/extractors.cr | 54 +++++++++++++++-----------
3 files changed, 33 insertions(+), 25 deletions(-)
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index d5628f6a..e6c0a1d5 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -8,7 +8,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
- extract_item(item, author, ucid).try { |t| items << t }
+ parse_item(item, author, ucid).try { |t| items << t }
}
continuation = continuation_items.as_a.last["continuationItemRenderer"]?
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index d1409c06..683a4a7e 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -37,7 +37,7 @@ module Invidious::Search
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
- extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
+ parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
return items
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index edc722cf..a4b20d04 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -20,6 +20,8 @@ private ITEM_PARSERS = {
Parsers::ReelItemRendererParser,
}
+private alias InitialData = Hash(String, JSON::Any)
+
record AuthorFallback, name : String, id : String
# Namespace for logic relating to parsing InnerTube data into various datastructs.
@@ -348,7 +350,7 @@ private module Parsers
raw_contents = content_container["items"]?.try &.as_a
if !raw_contents.nil?
raw_contents.each do |item|
- result = extract_item(item)
+ result = parse_item(item)
if !result.nil?
contents << result
end
@@ -510,7 +512,7 @@ private module Extractors
# }]
#
module YouTubeTabs
- def self.process(initial_data : Hash(String, JSON::Any))
+ def self.process(initial_data : InitialData)
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end
@@ -575,7 +577,7 @@ private module Extractors
# }
#
module SearchResults
- def self.process(initial_data : Hash(String, JSON::Any))
+ def self.process(initial_data : InitialData)
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end
@@ -608,8 +610,8 @@ private module Extractors
# The way they are structured is too varied to be accurately written down here.
# However, they all eventually lead to an array of parsable items after traversing
# through the JSON structure.
- module Continuation
- def self.process(initial_data : Hash(String, JSON::Any))
+ module ContinuationContent
+ def self.process(initial_data : InitialData)
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
@@ -691,8 +693,7 @@ end
# Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
-def extract_item(item : JSON::Any, author_fallback : String? = "",
- author_id_fallback : String? = "")
+def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
# We "allow" nil values but secretly use empty strings instead. This is to save us the
# hassle of modifying every author_fallback and author_id_fallback arg usage
# which is more often than not nil.
@@ -702,24 +703,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
# Each parser automatically validates the data given to see if the data is
# applicable to itself. If not nil is returned and the next parser is attempted.
ITEM_PARSERS.each do |parser|
- LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
+ LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
if result = parser.process(item, author_fallback)
- LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
-
+ LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
return result
else
- LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
+ LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
end
end
end
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem.
-def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
- author_id_fallback : String? = nil) : Array(SearchItem)
- items = [] of SearchItem
-
+#
+# This function yields the container so that items can be parsed separately.
+#
+def extract_items(initial_data : InitialData, &block)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@@ -727,24 +727,32 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
unpackaged_data = initial_data
end
- # This is identical to the parser cycling of extract_item().
+ # This is identical to the parser cycling of parse_item().
ITEM_CONTAINER_EXTRACTOR.each do |extractor|
LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
if container = extractor.process(unpackaged_data)
LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
# Extract items in container
- container.each do |item|
- if parsed_result = extract_item(item, author_fallback, author_id_fallback)
- items << parsed_result
- end
- end
-
- break
+ container.each { |item| yield item }
else
LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
end
end
+end
+
+# Wrapper using the block function above
+def extract_items(
+ initial_data : InitialData,
+ author_fallback : String? = nil,
+ author_id_fallback : String? = nil
+) : Array(SearchItem)
+ items = [] of SearchItem
+
+ extract_items(initial_data) do |item|
+ parsed = parse_item(item, author_fallback, author_id_fallback)
+ items << parsed if !parsed.nil?
+ end
return items
end
From ce7db8d2cb87111af15de2de9faf12aae38283bb Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 5 Nov 2022 18:56:35 +0100
Subject: [PATCH 05/17] extractors: Add continuation token parser
---
spec/invidious/hashtag_spec.cr | 4 +-
src/invidious/channels/playlists.cr | 16 +-----
src/invidious/hashtag.cr | 3 +-
src/invidious/helpers/serialized_yt_data.cr | 7 +++
src/invidious/search/processors.cr | 14 ++---
src/invidious/yt_backend/extractors.cr | 54 +++++++++++++++-----
src/invidious/yt_backend/extractors_utils.cr | 27 ++--------
7 files changed, 63 insertions(+), 62 deletions(-)
diff --git a/spec/invidious/hashtag_spec.cr b/spec/invidious/hashtag_spec.cr
index 77676878..266ec57b 100644
--- a/spec/invidious/hashtag_spec.cr
+++ b/spec/invidious/hashtag_spec.cr
@@ -4,7 +4,7 @@ Spectator.describe Invidious::Hashtag do
it "parses richItemRenderer containers (test 1)" do
# Enable mock
test_content = load_mock("hashtag/martingarrix_page1")
- videos = extract_items(test_content)
+ videos, _ = extract_items(test_content)
expect(typeof(videos)).to eq(Array(SearchItem))
expect(videos.size).to eq(60)
@@ -57,7 +57,7 @@ Spectator.describe Invidious::Hashtag do
it "parses richItemRenderer containers (test 2)" do
# Enable mock
test_content = load_mock("hashtag/martingarrix_page2")
- videos = extract_items(test_content)
+ videos, _ = extract_items(test_content)
expect(typeof(videos)).to eq(Array(SearchItem))
expect(videos.size).to eq(60)
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index e6c0a1d5..0d46499a 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -1,18 +1,7 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
response_json = YoutubeAPI.browse(continuation)
- continuation_items = response_json["onResponseReceivedActions"]?
- .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
- return [] of SearchItem, nil if !continuation_items
-
- items = [] of SearchItem
- continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
- parse_item(item, author, ucid).try { |t| items << t }
- }
-
- continuation = continuation_items.as_a.last["continuationItemRenderer"]?
- .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
+ items, continuation = extract_items(response_json, author, ucid)
else
url = "/channel/#{ucid}/playlists?flow=list&view=1"
@@ -30,8 +19,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
initial_data = extract_initial_data(response.body)
return [] of SearchItem, nil if !initial_data
- items = extract_items(initial_data, author, ucid)
- continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?
+ items, continuation = extract_items(initial_data, author, ucid)
end
return items, continuation
diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr
index afe31a36..bc329205 100644
--- a/src/invidious/hashtag.cr
+++ b/src/invidious/hashtag.cr
@@ -8,7 +8,8 @@ module Invidious::Hashtag
client_config = YoutubeAPI::ClientConfig.new(region: region)
response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
- return extract_items(response)
+ items, _ = extract_items(response)
+ return items
end
def generate_continuation(hashtag : String, cursor : Int)
diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr
index c52e2a0d..635f0984 100644
--- a/src/invidious/helpers/serialized_yt_data.cr
+++ b/src/invidious/helpers/serialized_yt_data.cr
@@ -265,4 +265,11 @@ class Category
end
end
+struct Continuation
+ getter token
+
+ def initialize(@token : String)
+ end
+end
+
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 683a4a7e..7e909590 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -9,7 +9,8 @@ module Invidious::Search
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
- return extract_items(initial_data)
+ items, _ = extract_items(initial_data)
+ return items
end
# Search a youtube channel
@@ -30,16 +31,7 @@ module Invidious::Search
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
- continuation_items = response_json["onResponseReceivedActions"]?
- .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
- return [] of SearchItem if !continuation_items
-
- items = [] of SearchItem
- continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
- parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
- end
-
+ items, _ = extract_items(response_json, "", ucid)
return items
end
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index a4b20d04..baf52118 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data"
private ITEM_CONTAINER_EXTRACTOR = {
Extractors::YouTubeTabs,
Extractors::SearchResults,
- Extractors::Continuation,
+ Extractors::ContinuationContent,
}
private ITEM_PARSERS = {
@@ -18,6 +18,7 @@ private ITEM_PARSERS = {
Parsers::CategoryRendererParser,
Parsers::RichItemRendererParser,
Parsers::ReelItemRendererParser,
+ Parsers::ContinuationItemRendererParser,
}
private alias InitialData = Hash(String, JSON::Any)
@@ -347,14 +348,9 @@ private module Parsers
content_container = item_contents["contents"]
end
- raw_contents = content_container["items"]?.try &.as_a
- if !raw_contents.nil?
- raw_contents.each do |item|
- result = parse_item(item)
- if !result.nil?
- contents << result
- end
- end
+ content_container["items"]?.try &.as_a.each do |item|
+ result = parse_item(item, author_fallback.name, author_fallback.id)
+ contents << result if result.is_a?(SearchItem)
end
Category.new({
@@ -477,6 +473,35 @@ private module Parsers
return {{@type.name}}
end
end
+
+ # Parses an InnerTube continuationItemRenderer into a Continuation.
+ # Returns nil when the given object isn't a continuationItemRenderer.
+ #
+ # continuationItemRenderer contains various metadata ued to load more
+ # content (i.e when the user scrolls down). The interesting bit is the
+ # protobuf object known as the "continutation token". Previously, those
+ # were generated from sratch, but recent (as of 11/2022) Youtube changes
+ # are forcing us to extract them from replies.
+ #
+ module ContinuationItemRendererParser
+ def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+ if item_contents = item["continuationItemRenderer"]?
+ return self.parse(item_contents)
+ end
+ end
+
+ private def self.parse(item_contents)
+ token = item_contents
+ .dig?("continuationEndpoint", "continuationCommand", "token")
+ .try &.as_s
+
+ return Continuation.new(token) if token
+ end
+
+ def self.parser_name
+ return {{@type.name}}
+ end
+ end
end
# The following are the extractors for extracting an array of items from
@@ -746,13 +771,18 @@ def extract_items(
initial_data : InitialData,
author_fallback : String? = nil,
author_id_fallback : String? = nil
-) : Array(SearchItem)
+) : {Array(SearchItem), String?}
items = [] of SearchItem
+ continuation = nil
extract_items(initial_data) do |item|
parsed = parse_item(item, author_fallback, author_id_fallback)
- items << parsed if !parsed.nil?
+
+ case parsed
+ when .is_a?(Continuation) then continuation = parsed.token
+ when .is_a?(SearchItem) then items << parsed
+ end
end
- return items
+ return items, continuation
end
diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr
index f8245160..0cb3c079 100644
--- a/src/invidious/yt_backend/extractors_utils.cr
+++ b/src/invidious/yt_backend/extractors_utils.cr
@@ -68,10 +68,10 @@ rescue ex
return false
end
-def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
- extracted = extract_items(initial_data, author_fallback, author_id_fallback)
+def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo)
+ extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback)
- target = [] of SearchItem
+ target = [] of (SearchItem | Continuation)
extracted.each do |i|
if i.is_a?(Category)
i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str
target << i
end
end
- return target.select(SearchVideo).map(&.as(SearchVideo))
+
+ return target.select(SearchVideo)
end
def extract_selected_tab(tabs)
# Extract the selected tab from the array of tabs Youtube returns
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"]
end
-
-def fetch_continuation_token(items : Array(JSON::Any))
- # Fetches the continuation token from an array of items
- return items.last["continuationItemRenderer"]?
- .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
-end
-
-def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
- # Fetches the continuation token from initial data
- if initial_data["onResponseReceivedActions"]?
- continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
- else
- tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
- continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
- end
-
- return fetch_continuation_token(continuation_items.as_a)
-end
From 8e8ca4fcc5cfcb7bebc3f29440d6abc1de770513 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 12 Nov 2022 00:04:27 +0100
Subject: [PATCH 06/17] Prepare to create a 'Channel' module
---
src/invidious.cr | 9 ++++++++-
src/invidious/jobs/notification_job.cr | 4 ++--
src/invidious/jobs/refresh_channels_job.cr | 2 +-
src/invidious/jobs/refresh_feeds_job.cr | 2 +-
src/invidious/jobs/subscribe_to_feeds_job.cr | 2 +-
5 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/src/invidious.cr b/src/invidious.cr
index 2874cc71..5064f0b8 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -48,6 +48,13 @@ require "./invidious/search/*"
require "./invidious/routes/**"
require "./invidious/jobs/**"
+# Declare the base namespace for invidious
+module Invidious
+end
+
+# Simple alias to make code easier to read
+alias IV = Invidious
+
CONFIG = Config.load
HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
@@ -172,7 +179,7 @@ if CONFIG.popular_enabled
Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
end
-CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
+CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
index 2f525e08..b445107b 100644
--- a/src/invidious/jobs/notification_job.cr
+++ b/src/invidious/jobs/notification_job.cr
@@ -1,12 +1,12 @@
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
- private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+ private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
def initialize(@connection_channel, @pg_url)
end
def begin
- connections = [] of Channel(PQ::Notification)
+ connections = [] of ::Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
index 92681408..80812a63 100644
--- a/src/invidious/jobs/refresh_channels_job.cr
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
backoff = 2.minutes
loop do
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
index 4b52c959..4f8130df 100644
--- a/src/invidious/jobs/refresh_feeds_job.cr
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
index a431a48a..8584fb9c 100644
--- a/src/invidious/jobs/subscribe_to_feeds_job.cr
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
From c5ee2bfc0f5e485f91e53dedc879312c3e729be8 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 11 Nov 2022 00:44:24 +0100
Subject: [PATCH 07/17] channel: use YT API to fetch playlist items
---
src/invidious/channels/playlists.cr | 40 +++++++++++++++--------------
1 file changed, 21 insertions(+), 19 deletions(-)
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index 0d46499a..8fdac3a7 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -1,28 +1,30 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
- response_json = YoutubeAPI.browse(continuation)
- items, continuation = extract_items(response_json, author, ucid)
+ initial_data = YoutubeAPI.browse(continuation)
else
- url = "/channel/#{ucid}/playlists?flow=list&view=1"
+ params =
+ case sort_by
+ when "last", "last_added"
+ # Equivalent to "&sort=lad"
+ # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
+ "EglwbGF5bGlzdHMYBCABMAE%3D"
+ when "oldest", "oldest_created"
+ # formerly "&sort=da"
+ # Not available anymore :c or maybe ??
+ # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
+ "EglwbGF5bGlzdHMYAiABMAE%3D"
+ # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
+ # "EglwbGF5bGlzdHMYASABMAE%3D"
+ when "newest", "newest_created"
+ # Formerly "&sort=dd"
+ # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
+ "EglwbGF5bGlzdHMYAyABMAE%3D"
+ end
- case sort_by
- when "last", "last_added"
- #
- when "oldest", "oldest_created"
- url += "&sort=da"
- when "newest", "newest_created"
- url += "&sort=dd"
- else nil # Ignore
- end
-
- response = YT_POOL.client &.get(url)
- initial_data = extract_initial_data(response.body)
- return [] of SearchItem, nil if !initial_data
-
- items, continuation = extract_items(initial_data, author, ucid)
+ initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
- return items, continuation
+ return extract_items(initial_data, ucid, author)
end
# ## NOTE: DEPRECATED
From 2903e896ecf2404bf932438a33432125a6ad1fca Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 11 Nov 2022 20:26:34 +0100
Subject: [PATCH 08/17] channel: use YT API + extractors to fetch videos
---
src/invidious/channels/channels.cr | 65 ++++++++---------
src/invidious/channels/playlists.cr | 2 +-
src/invidious/channels/videos.cr | 94 ++++++++++++++++++-------
src/invidious/routes/api/v1/channels.cr | 66 +++++++----------
src/invidious/routes/channels.cr | 4 +-
5 files changed, 127 insertions(+), 104 deletions(-)
diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr
index e3d3d9ee..27369f12 100644
--- a/src/invidious/channels/channels.cr
+++ b/src/invidious/channels/channels.cr
@@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
- page = 1
+ channel = InvidiousChannel.new({
+ id: ucid,
+ author: author,
+ updated: Time.utc,
+ deleted: false,
+ subscribed: nil,
+ })
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
- initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
- videos = extract_videos(initial_data, author, ucid)
+ videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//feed/entry").each do |entry|
@@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
views ||= 0_i64
- channel_video = videos.select { |video| video.id == video_id }[0]?
+ channel_video = videos
+ .select(SearchVideo)
+ .select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
@@ -235,30 +242,25 @@ def fetch_channel(ucid, pull_all_videos : Bool)
end
if pull_all_videos
- page += 1
-
- ids = [] of String
-
loop do
- initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
- videos = extract_videos(initial_data, author, ucid)
+ # Keep fetching videos using the continuation token retrieved earlier
+ videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
- count = videos.size
- videos = videos.map { |video| ChannelVideo.new({
- id: video.id,
- title: video.title,
- published: video.published,
- updated: Time.utc,
- ucid: video.ucid,
- author: video.author,
- length_seconds: video.length_seconds,
- live_now: video.live_now,
- premiere_timestamp: video.premiere_timestamp,
- views: video.views,
- }) }
-
- videos.each do |video|
- ids << video.id
+ count = 0
+ videos.select(SearchVideo).each do |video|
+ count += 1
+ video = ChannelVideo.new({
+ id: video.id,
+ title: video.title,
+ published: video.published,
+ updated: Time.utc,
+ ucid: video.ucid,
+ author: video.author,
+ length_seconds: video.length_seconds,
+ live_now: video.live_now,
+ premiere_timestamp: video.premiere_timestamp,
+ views: video.views,
+ })
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
@@ -269,17 +271,10 @@ def fetch_channel(ucid, pull_all_videos : Bool)
end
break if count < 25
- page += 1
+ sleep 500.milliseconds
end
end
- channel = InvidiousChannel.new({
- id: ucid,
- author: author,
- updated: Time.utc,
- deleted: false,
- subscribed: nil,
- })
-
+ channel.updated = Time.utc
return channel
end
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index 8fdac3a7..772eecb9 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -24,7 +24,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
- return extract_items(initial_data, ucid, author)
+ return extract_items(initial_data, author, ucid)
end
# ## NOTE: DEPRECATED
diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index b495e597..23ad4e02 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
+ sort_by_numerical =
+ case sort_by
+ when "newest" then 1_i64
+ when "popular" then 2_i64
+ when "oldest" then 3_i64 # Broken as of 10/2022 :c
+ else 1_i64 # Fallback to "newest"
+ end
+
object_inner_1 = {
"110:embedded" => {
"3:embedded" => {
@@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
"1:string" => object_inner_2_encoded,
"2:string" => "00000000-0000-0000-0000-000000000000",
},
- "3:varint" => 1_i64,
+ "3:varint" => sort_by_numerical,
},
},
},
@@ -52,34 +60,66 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
return continuation
end
-def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
- continuation = produce_channel_videos_continuation(ucid, page,
- auto_generated: auto_generated, sort_by: sort_by, v2: true)
-
- return YoutubeAPI.browse(continuation)
-end
-
-def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
- videos = [] of SearchVideo
-
- # 2.times do |i|
- # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
- initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by)
- videos = extract_videos(initial_data, author, ucid)
- # end
-
- return videos.size, videos
-end
-
-def get_latest_videos(ucid)
- initial_data = get_channel_videos_response(ucid)
- author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
-
- return extract_videos(initial_data, author, ucid)
-end
-
# Used in bypass_captcha_job.cr
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
+
+module Invidious::Channel::Tabs
+ extend self
+
+ # -------------------
+ # Regular videos
+ # -------------------
+
+ def make_initial_video_ctoken(ucid, sort_by) : String
+ return produce_channel_videos_continuation(ucid, sort_by: sort_by)
+ end
+
+ # Wrapper for AboutChannel, as we still need to call get_videos with
+ # an author name and ucid directly (e.g in RSS feeds).
+ # TODO: figure out how to get rid of that
+ def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+ return get_videos(
+ channel.author, channel.ucid,
+ continuation: continuation, sort_by: sort_by
+ )
+ end
+
+ # Wrapper for InvidiousChannel, as we still need to call get_videos with
+ # an author name and ucid directly (e.g in RSS feeds).
+ # TODO: figure out how to get rid of that
+ def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest")
+ return get_videos(
+ channel.author, channel.id,
+ continuation: continuation, sort_by: sort_by
+ )
+ end
+
+ def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
+ continuation ||= make_initial_video_ctoken(ucid, sort_by)
+ initial_data = YoutubeAPI.browse(continuation: continuation)
+
+ return extract_items(initial_data, author, ucid)
+ end
+
+ def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+ if continuation.nil?
+ # Fetch the first "page" of video
+ items, next_continuation = get_videos(channel, sort_by: sort_by)
+ else
+ # Fetch a "page" of videos using the given continuation token
+ items, next_continuation = get_videos(channel, continuation: continuation)
+ end
+
+ # If there is more to load, then load a second "page"
+ # and replace the previous continuation token
+ if !next_continuation.nil?
+ items_2, next_continuation = get_videos(channel, continuation: next_continuation)
+ items.concat items_2
+ end
+
+ return items, next_continuation
+ end
+end
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 6b81c546..72d9ae5f 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -5,8 +5,6 @@ module Invidious::Routes::API::V1::Channels
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
- sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
begin
channel = get_about_info(ucid, locale)
@@ -19,16 +17,13 @@ module Invidious::Routes::API::V1::Channels
return error_json(500, ex)
end
- page = 1
- if channel.auto_generated
- videos = [] of SearchVideo
- count = 0
- else
- begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
- rescue ex
- return error_json(500, ex)
- end
+ # Retrieve "sort by" setting from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+
+ begin
+ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+ rescue ex
+ return error_json(500, ex)
end
JSON.build do |json|
@@ -134,25 +129,11 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
- locale = env.get("preferences").as(Preferences).locale
+ # Remove parameters that could affect this endpoint's behavior
+ env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
+ env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
-
- begin
- videos = get_latest_videos(ucid)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
- end
- end
- end
+ return self.videos(env)
end
def self.videos(env)
@@ -161,11 +142,6 @@ module Invidious::Routes::API::V1::Channels
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
- sort_by = env.params.query["sort"]?.try &.downcase
- sort_by ||= env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
begin
channel = get_about_info(ucid, locale)
@@ -178,17 +154,27 @@ module Invidious::Routes::API::V1::Channels
return error_json(500, ex)
end
+ # Retrieve some URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ videos, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
rescue ex
return error_json(500, ex)
end
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
end
+
+ json.field "continuation", next_continuation if next_continuation
end
end
end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index f26f29f5..2773deb7 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -32,7 +32,9 @@ module Invidious::Routes::Channels
sort_options = {"newest", "oldest", "popular"}
sort_by ||= "newest"
- count, items = get_60_videos(channel.ucid, channel.author, 1, channel.auto_generated, sort_by)
+ items, continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
end
templated "channel"
From 52ef89f02d0ab29fd0f218abc4051328b3d96809 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 12 Nov 2022 00:09:03 +0100
Subject: [PATCH 09/17] channel: Add support for shorts and livestreams
(backend only)
---
src/invidious/channels/videos.cr | 50 ++++++++++++++++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index 23ad4e02..bea406c1 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -122,4 +122,54 @@ module Invidious::Channel::Tabs
return items, next_continuation
end
+
+ # -------------------
+ # Shorts
+ # -------------------
+
+ def get_shorts(channel : AboutChannel, continuation : String? = nil)
+ if continuation.nil?
+ # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
+ # TODO: try to extract the continuation tokens that allows other sorting options
+ initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
+ else
+ initial_data = YoutubeAPI.browse(continuation: continuation)
+ end
+
+ return extract_items(initial_data, channel.author, channel.ucid)
+ end
+
+ # -------------------
+ # Livestreams
+ # -------------------
+
+ def get_livestreams(channel : AboutChannel, continuation : String? = nil)
+ if continuation.nil?
+ # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams"
+ initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")
+ else
+ initial_data = YoutubeAPI.browse(continuation: continuation)
+ end
+
+ return extract_items(initial_data, channel.author, channel.ucid)
+ end
+
+ def get_60_livestreams(channel : AboutChannel, continuation : String? = nil)
+ if continuation.nil?
+ # Fetch the first "page" of streams
+ items, next_continuation = get_livestreams(channel)
+ else
+ # Fetch a "page" of streams using the given continuation token
+ items, next_continuation = get_livestreams(channel, continuation: continuation)
+ end
+
+ # If there is more to load, then load a second "page"
+ # and replace the previous continuation token
+ if !next_continuation.nil?
+ items_2, next_continuation = get_livestreams(channel, continuation: next_continuation)
+ items.concat items_2
+ end
+
+ return items, next_continuation
+ end
end
From 5d6abd5301b14c24475bf7ad477a43c60ff78993 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 1 Dec 2022 23:01:31 +0100
Subject: [PATCH 10/17] extractors: Fix ReelItemRendererParser
---
src/invidious/yt_backend/extractors.cr | 32 ++++++++++++++++----------
1 file changed, 20 insertions(+), 12 deletions(-)
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index baf52118..bca0dcbd 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -382,7 +382,9 @@ private module Parsers
end
private def self.parse(item_contents, author_fallback)
- return VideoRendererParser.process(item_contents, author_fallback)
+ child = VideoRendererParser.process(item_contents, author_fallback)
+ child ||= ReelItemRendererParser.process(item_contents, author_fallback)
+ return child
end
def self.parser_name
@@ -406,12 +408,18 @@ private module Parsers
private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s
- video_details_container = item_contents.dig(
- "navigationEndpoint", "reelWatchEndpoint",
- "overlay", "reelPlayerOverlayRenderer",
- "reelPlayerHeaderSupportedRenderers",
- "reelPlayerHeaderRenderer"
- )
+ begin
+ video_details_container = item_contents.dig(
+ "navigationEndpoint", "reelWatchEndpoint",
+ "overlay", "reelPlayerOverlayRenderer",
+ "reelPlayerHeaderSupportedRenderers",
+ "reelPlayerHeaderRenderer"
+ )
+ rescue ex : KeyError
+ # Extract key name from original message
+ key = /"([^"]+)"/.match(ex.message || "").try &.[1]?
+ raise BrokenTubeException.new(key || "reelPlayerOverlayRenderer")
+ end
# Author infos
@@ -434,9 +442,9 @@ private module Parsers
# View count
- view_count_text = video_details_container.dig?("viewCountText", "simpleText")
- view_count_text ||= video_details_container
- .dig?("viewCountText", "accessibility", "accessibilityData", "label")
+ # View count used to be in the reelWatchEndpoint, but that changed?
+ view_count_text = item_contents.dig?("viewCountText", "simpleText")
+ view_count_text ||= video_details_container.dig?("viewCountText", "simpleText")
view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
@@ -448,8 +456,8 @@ private module Parsers
regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data)
- minutes = regex_match.try &.["min"].to_i(strict: false) || 0
- seconds = regex_match.try &.["sec"].to_i(strict: false) || 0
+ minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0
+ seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0
duration = (minutes*60 + seconds)
From 6c9754e66316d903ed4f89d2cd59cd82940509f5 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Wed, 30 Nov 2022 00:29:48 +0100
Subject: [PATCH 11/17] frontend: Add support for shorts and livestreams
---
locales/en-US.json | 9 +++--
src/invidious/channels/about.cr | 10 ++++-
src/invidious/frontend/channel_page.cr | 43 ++++++++++++++++++++
src/invidious/routes/channels.cr | 54 ++++++++++++++++++++++++--
src/invidious/routing.cr | 4 +-
src/invidious/views/channel.ecr | 30 +++++---------
src/invidious/views/community.ecr | 15 +------
src/invidious/views/playlists.ecr | 14 +------
8 files changed, 124 insertions(+), 55 deletions(-)
create mode 100644 src/invidious/frontend/channel_page.cr
diff --git a/locales/en-US.json b/locales/en-US.json
index 5554b928..44b40c24 100644
--- a/locales/en-US.json
+++ b/locales/en-US.json
@@ -404,9 +404,7 @@
"`x` marked it with a ❤": "`x` marked it with a ❤",
"Audio mode": "Audio mode",
"Video mode": "Video mode",
- "Videos": "Videos",
"Playlists": "Playlists",
- "Community": "Community",
"search_filters_title": "Filters",
"search_filters_date_label": "Upload date",
"search_filters_date_option_none": "Any date",
@@ -472,5 +470,10 @@
"crash_page_read_the_faq": "read the <a href=\"`x`\">Frequently Asked Questions (FAQ)</a>",
"crash_page_search_issue": "searched for <a href=\"`x`\">existing issues on GitHub</a>",
"crash_page_report_issue": "If none of the above helped, please <a href=\"`x`\">open a new issue on GitHub</a> (preferably in English) and include the following text in your message (do NOT translate that text):",
- "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>"
+ "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>",
+ "channel_tab_videos_label": "Videos",
+ "channel_tab_shorts_label": "Shorts",
+ "channel_tab_streams_label": "Livestreams",
+ "channel_tab_playlists_label": "Playlists",
+ "channel_tab_community_label": "Community"
}
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index bb9bd8c7..09c3427a 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -104,8 +104,14 @@ def get_about_info(ucid, locale) : AboutChannel
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
- tab_names = tabs_json.as_a
- .compact_map(&.dig?("tabRenderer", "title").try &.as_s.downcase)
+ tab_names = tabs_json.as_a.compact_map do |entry|
+ name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
+
+ # This is a small fix to not add extra code on the HTML side
+ # I.e, the URL for the "live" tab is .../streams, so use "streams"
+ # everywhere for the sake of simplicity
+ (name == "live") ? "streams" : name
+ end
# Get the currently active tab ("About")
about_tab = extract_selected_tab(tabs_json)
diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr
new file mode 100644
index 00000000..7ac0e071
--- /dev/null
+++ b/src/invidious/frontend/channel_page.cr
@@ -0,0 +1,43 @@
+module Invidious::Frontend::ChannelPage
+ extend self
+
+ enum TabsAvailable
+ Videos
+ Shorts
+ Streams
+ Playlists
+ Community
+ end
+
+ def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
+ return String.build(1500) do |str|
+ base_url = "/channel/#{channel.ucid}"
+
+ TabsAvailable.each do |tab|
+ # Ignore playlists, as it is not supported for auto-generated channels yet
+ next if (tab.playlists? && channel.auto_generated)
+
+ tab_name = tab.to_s.downcase
+
+ if channel.tabs.includes? tab_name
+ str << %(<div class="pure-u-1 pure-md-1-3">\n)
+
+ if tab == selected_tab
+ str << "\t<b>"
+ str << translate(locale, "channel_tab_#{tab_name}_label")
+ str << "</b>\n"
+ else
+ # Video tab doesn't have the last path component
+ url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
+
+ str << %(\t<a href=") << url << %(">)
+ str << translate(locale, "channel_tab_#{tab_name}_label")
+ str << "</a>\n"
+ end
+
+ str << "</div>"
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 2773deb7..78b38341 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -18,7 +18,7 @@ module Invidious::Routes::Channels
sort_options = {"last", "oldest", "newest"}
sort_by ||= "last"
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -32,11 +32,59 @@ module Invidious::Routes::Channels
sort_options = {"newest", "oldest", "popular"}
sort_by ||= "newest"
- items, continuation = Channel::Tabs.get_60_videos(
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_videos(
channel, continuation: continuation, sort_by: sort_by
)
end
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
+ templated "channel"
+ end
+
+ def self.shorts(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "shorts"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ # TODO: support sort option for shorts
+ sort_by = ""
+ sort_options = [] of String
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation
+ )
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
+ templated "channel"
+ end
+
+ def self.streams(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "streams"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ # TODO: support sort option for livestreams
+ sort_by = ""
+ sort_options = [] of String
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation
+ )
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
@@ -124,7 +172,7 @@ module Invidious::Routes::Channels
end
selected_tab = env.request.path.split("/")[-1]
- if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
+ if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index f409f13c..08739c3d 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -115,6 +115,8 @@ module Invidious::Routing
get "/channel/:ucid", Routes::Channels, :home
get "/channel/:ucid/home", Routes::Channels, :home
get "/channel/:ucid/videos", Routes::Channels, :videos
+ get "/channel/:ucid/shorts", Routes::Channels, :shorts
+ get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
get "/channel/:ucid/about", Routes::Channels, :about
@@ -122,7 +124,7 @@ module Invidious::Routing
get "/user/:user/live", Routes::Channels, :live
get "/c/:user/live", Routes::Channels, :live
- ["", "/videos", "/playlists", "/community", "/about"].each do |path|
+ {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path|
# /c/LinusTechTips
get "/c/:user#{path}", Routes::Channels, :brand_redirect
# /user/linustechtips | Not always the same as /c/
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 878587d4..f6cc3340 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -64,23 +64,8 @@
<a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
- <% if !channel.auto_generated %>
- <div class="pure-u-1 pure-md-1-3">
- <b><%= translate(locale, "Videos") %></b>
- </div>
- <% end %>
- <div class="pure-u-1 pure-md-1-3">
- <% if channel.auto_generated %>
- <b><%= translate(locale, "Playlists") %></b>
- <% else %>
- <a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
- <% end %>
- </div>
- <div class="pure-u-1 pure-md-1-3">
- <% if channel.tabs.includes? "community" %>
- <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
- <% end %>
- </div>
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
</div>
<div class="pure-u-1-3"></div>
<div class="pure-u-1-3">
@@ -111,7 +96,12 @@
</div>
<div class="pure-g h-box">
- <div class="pure-u-1 pure-u-lg-1-5"></div>
- <div class="pure-u-1 pure-u-lg-3-5"></div>
- <div class="pure-u-1 pure-u-lg-1-5"></div>
+ <div class="pure-u-1 pure-u-md-4-5"></div>
+ <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
+ <% if next_continuation %>
+ <a href="/channel/<%= ucid %>/<%= selected_tab %>?continuation=<%= next_continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Next page") %>
+ </a>
+ <% end %>
+ </div>
</div>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 3bc29e55..e467a679 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -50,19 +50,8 @@
<a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
- <% if !channel.auto_generated %>
- <div class="pure-u-1 pure-md-1-3">
- <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a>
- </div>
- <% end %>
- <div class="pure-u-1 pure-md-1-3">
- <a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
- </div>
- <div class="pure-u-1 pure-md-1-3">
- <% if channel.tabs.includes? "community" %>
- <b><%= translate(locale, "Community") %></b>
- <% end %>
- </div>
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Community) %>
</div>
<div class="pure-u-2-3"></div>
</div>
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
index c8718e7b..56d25ef5 100644
--- a/src/invidious/views/playlists.ecr
+++ b/src/invidious/views/playlists.ecr
@@ -54,19 +54,7 @@
<% end %>
</div>
- <div class="pure-u-1 pure-md-1-3">
- <a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a>
- </div>
- <div class="pure-u-1 pure-md-1-3">
- <% if !channel.auto_generated %>
- <b><%= translate(locale, "Playlists") %></b>
- <% end %>
- </div>
- <div class="pure-u-1 pure-md-1-3">
- <% if channel.tabs.includes? "community" %>
- <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
- <% end %>
- </div>
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Playlists) %>
</div>
<div class="pure-u-1-3"></div>
<div class="pure-u-1-3">
From 40c666cab22693cf9d31895978ae4b4356e6579b Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 4 Dec 2022 19:24:51 +0100
Subject: [PATCH 12/17] api: Add support for shorts and livestreams
---
src/invidious/routes/api/v1/channels.cr | 118 ++++++++++++++++++------
src/invidious/routing.cr | 3 +
2 files changed, 92 insertions(+), 29 deletions(-)
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 72d9ae5f..4e92b54e 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,11 +1,7 @@
module Invidious::Routes::API::V1::Channels
- def self.home(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
-
+ # Macro to avoid duplicating some code below
+ # This sets the `channel` variable, or handles Exceptions.
+ private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
@@ -16,6 +12,17 @@ module Invidious::Routes::API::V1::Channels
rescue ex
return error_json(500, ex)
end
+ end
+
+ def self.home(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
# Retrieve "sort by" setting from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
@@ -138,21 +145,13 @@ module Invidious::Routes::API::V1::Channels
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
- ucid = env.params.url["ucid"]
-
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
# Retrieve some URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
@@ -179,6 +178,74 @@ module Invidious::Routes::API::V1::Channels
end
end
+ def self.shorts(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ continuation = env.params.query["continuation"]?
+
+ begin
+ videos, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.streams(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ continuation = env.params.query["continuation"]?
+
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
def self.playlists(env)
locale = env.get("preferences").as(Preferences).locale
@@ -190,16 +257,9 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 08739c3d..0e6fba21 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -222,6 +222,9 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+ get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
+ get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+
{% for route in {"videos", "latest", "playlists", "community", "search"} %}
get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
From b6a4de66a5414f8ae790033fc3fc9e9fda70a860 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 4 Dec 2022 23:19:25 +0100
Subject: [PATCH 13/17] frontend: Unify the various channel pages
---
src/invidious/routes/channels.cr | 33 ++++---
src/invidious/views/channel.ecr | 95 +++++-------------
src/invidious/views/community.ecr | 65 +++----------
.../views/components/channel_info.ecr | 60 ++++++++++++
src/invidious/views/playlists.ecr | 96 -------------------
5 files changed, 116 insertions(+), 233 deletions(-)
create mode 100644 src/invidious/views/components/channel_info.ecr
delete mode 100644 src/invidious/views/playlists.ecr
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 78b38341..77d309fb 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -7,18 +7,19 @@ module Invidious::Routes::Channels
def self.videos(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
+ return data if !data.is_a?(Tuple)
+
locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_options = {"last", "oldest", "newest"}
- sort_by ||= "last"
- items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, (sort_by || "last")
+ )
+
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -30,11 +31,10 @@ module Invidious::Routes::Channels
items.each(&.author = "")
else
sort_options = {"newest", "oldest", "popular"}
- sort_by ||= "newest"
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_videos(
- channel, continuation: continuation, sort_by: sort_by
+ channel, continuation: continuation, sort_by: (sort_by || "newest")
)
end
@@ -90,24 +90,26 @@ module Invidious::Routes::Channels
def self.playlists(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
+ return data if !data.is_a?(Tuple)
+
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "last"
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, (sort_by || "last")
+ )
+
items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
items.each(&.author = "")
- templated "playlists"
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
+ templated "channel"
end
def self.community(env)
@@ -121,12 +123,15 @@ module Invidious::Routes::Channels
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
- # sort_by = env.params.query["sort_by"]?.try &.downcase
if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
+ # TODO: support sort options for community posts
+ sort_by = ""
+ sort_options = [] of String
+
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index f6cc3340..039f8752 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,8 +1,23 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url =
+ case selected_tab
+ when .shorts? then "/channel/#{ucid}/shorts"
+ when .streams? then "/channel/#{ucid}/streams"
+ when .playlists? then "/channel/#{ucid}/playlists"
+ else
+ "/channel/#{ucid}"
+ end
+
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+-%>
<% content_for "header" do %>
+<%- if selected_tab.videos? -%>
<meta name="description" content="<%= channel.description %>">
<meta property="og:site_name" content="Invidious">
<meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
@@ -14,76 +29,14 @@
<meta name="twitter:title" content="<%= author %>">
<meta name="twitter:description" content="<%= channel.description %>">
<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>">
-<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>">
-<title><%= author %> - Invidious</title>
<link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
+<%- end -%>
+
+<link rel="alternate" href="<%= youtube_url %>">
+<title><%= author %> - Invidious</title>
<% end %>
-<% if channel.banner %>
- <div class="h-box">
- <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
- </div>
-
- <div class="h-box">
- <hr>
- </div>
-<% end %>
-
-<div class="pure-g h-box">
- <div class="pure-u-2-3">
- <div class="channel-profile">
- <img src="/ggpht<%= channel_profile_pic %>">
- <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %>
- </div>
- </div>
- <div class="pure-u-1-3">
- <h3 style="text-align:right">
- <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
- </h3>
- </div>
-</div>
-
-<div class="h-box">
- <div id="descriptionWrapper">
- <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
- </div>
-</div>
-
-<div class="h-box">
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
- <div class="pure-u-1-3">
- <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a>
- <div class="pure-u-1 pure-md-1-3">
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
- <% else %>
- <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
- <% end %>
- </div>
-
- <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
- </div>
- <div class="pure-u-1-3"></div>
- <div class="pure-u-1-3">
- <div class="pure-g" style="text-align:right">
- <% sort_options.each do |sort| %>
- <div class="pure-u-1 pure-md-1-3">
- <% if sort_by == sort %>
- <b><%= translate(locale, sort) %></b>
- <% else %>
- <a href="/channel/<%= ucid %>?sort_by=<%= sort %>">
- <%= translate(locale, sort) %>
- </a>
- <% end %>
- </div>
- <% end %>
- </div>
- </div>
-</div>
+<%= rendered "components/channel_info" %>
<div class="h-box">
<hr>
@@ -99,7 +52,7 @@
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if next_continuation %>
- <a href="/channel/<%= ucid %>/<%= selected_tab %>?continuation=<%= next_continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index e467a679..9e11d562 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,60 +1,21 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url = "/channel/#{ucid}/community"
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
+-%>
<% content_for "header" do %>
+<link rel="alternate" href="<%= youtube_url %>">
<title><%= author %> - Invidious</title>
<% end %>
-<% if channel.banner %>
- <div class="h-box">
- <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
- </div>
-
- <div class="h-box">
- <hr>
- </div>
-<% end %>
-
-<div class="pure-g h-box">
- <div class="pure-u-2-3">
- <div class="channel-profile">
- <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
- <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %>
- </div>
- </div>
- <div class="pure-u-1-3" style="text-align:right">
- <h3 style="text-align:right">
- <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a>
- </h3>
- </div>
-</div>
-
-<div class="h-box">
- <div id="descriptionWrapper">
- <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
- </div>
-</div>
-
-<div class="h-box">
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
- <div class="pure-u-1-3">
- <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a>
- <div class="pure-u-1 pure-md-1-3">
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
- <% else %>
- <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
- <% end %>
- </div>
-
- <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Community) %>
- </div>
- <div class="pure-u-2-3"></div>
-</div>
+<%= rendered "components/channel_info" %>
<div class="h-box">
<hr>
diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr
new file mode 100644
index 00000000..f216359f
--- /dev/null
+++ b/src/invidious/views/components/channel_info.ecr
@@ -0,0 +1,60 @@
+<% if channel.banner %>
+ <div class="h-box">
+ <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
+ </div>
+
+ <div class="h-box">
+ <hr>
+ </div>
+<% end %>
+
+<div class="pure-g h-box">
+ <div class="pure-u-2-3">
+ <div class="channel-profile">
+ <img src="/ggpht<%= channel_profile_pic %>">
+ <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %>
+ </div>
+ </div>
+ <div class="pure-u-1-3">
+ <h3 style="text-align:right">
+ <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
+ </h3>
+ </div>
+</div>
+
+<div class="h-box">
+ <div id="descriptionWrapper">
+ <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
+ </div>
+</div>
+
+<div class="h-box">
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+</div>
+
+<div class="pure-g h-box">
+ <div class="pure-u-1-2">
+ <div class="pure-u-1 pure-md-1-3">
+ <a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a>
+ </div>
+ <div class="pure-u-1 pure-md-1-3">
+ <a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a>
+ </div>
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+ </div>
+ <div class="pure-u-1-2">
+ <div class="pure-g" style="text-align:end">
+ <% sort_options.each do |sort| %>
+ <div class="pure-u-1 pure-md-1-3">
+ <% if sort_by == sort %>
+ <b><%= translate(locale, sort) %></b>
+ <% else %>
+ <a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a>
+ <% end %>
+ </div>
+ <% end %>
+ </div>
+ </div>
+</div>
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
deleted file mode 100644
index 56d25ef5..00000000
--- a/src/invidious/views/playlists.ecr
+++ /dev/null
@@ -1,96 +0,0 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-
-<% content_for "header" do %>
-<title><%= author %> - Invidious</title>
-<% end %>
-
-<% if channel.banner %>
- <div class="h-box">
- <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
- </div>
-
- <div class="h-box">
- <hr>
- </div>
-<% end %>
-
-<div class="pure-g h-box">
- <div class="pure-u-2-3">
- <div class="channel-profile">
- <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
- <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %>
- </div>
- </div>
- <div class="pure-u-1-3" style="text-align:right">
- <h3 style="text-align:right">
- <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
- </h3>
- </div>
-</div>
-
-<div class="h-box">
- <div id="descriptionWrapper">
- <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p>
- </div>
-</div>
-
-<div class="h-box">
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
- <div class="pure-g pure-u-1-3">
- <div class="pure-u-1 pure-md-1-3">
- <a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a>
- </div>
-
- <div class="pure-u-1 pure-md-1-3">
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
- <% else %>
- <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
- <% end %>
- </div>
-
- <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Playlists) %>
- </div>
- <div class="pure-u-1-3"></div>
- <div class="pure-u-1-3">
- <div class="pure-g" style="text-align:right">
- <% {"last", "oldest", "newest"}.each do |sort| %>
- <div class="pure-u-1 pure-md-1-3">
- <% if sort_by == sort %>
- <b><%= translate(locale, sort) %></b>
- <% else %>
- <a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>">
- <%= translate(locale, sort) %>
- </a>
- <% end %>
- </div>
- <% end %>
- </div>
- </div>
-</div>
-
-<div class="h-box">
- <hr>
-</div>
-
-<div class="pure-g">
-<% items.each do |item| %>
- <%= rendered "components/item" %>
-<% end %>
-</div>
-
-<div class="pure-g h-box">
- <div class="pure-u-1 pure-u-md-4-5"></div>
- <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
- <% if continuation %>
- <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Next page") %>
- </a>
- <% end %>
- </div>
-</div>
From 4e3a9306260b737e2d13c6a763899b946a6ecfbb Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Mon, 5 Dec 2022 00:50:04 +0100
Subject: [PATCH 14/17] frontend: Add support for the "featured channels" page
---
locales/en-US.json | 3 +-
src/invidious/channels/about.cr | 50 +++++--------------------
src/invidious/frontend/channel_page.cr | 1 +
src/invidious/routes/api/v1/channels.cr | 24 ++----------
src/invidious/routes/channels.cr | 20 ++++++++++
src/invidious/routing.cr | 1 +
src/invidious/views/channel.ecr | 1 +
7 files changed, 37 insertions(+), 63 deletions(-)
diff --git a/locales/en-US.json b/locales/en-US.json
index 44b40c24..12955665 100644
--- a/locales/en-US.json
+++ b/locales/en-US.json
@@ -475,5 +475,6 @@
"channel_tab_shorts_label": "Shorts",
"channel_tab_streams_label": "Livestreams",
"channel_tab_playlists_label": "Playlists",
- "channel_tab_community_label": "Community"
+ "channel_tab_community_label": "Community",
+ "channel_tab_channels_label": "Channels"
}
diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr
index 09c3427a..0054f8f2 100644
--- a/src/invidious/channels/about.cr
+++ b/src/invidious/channels/about.cr
@@ -16,12 +16,6 @@ record AboutChannel,
tabs : Array(String),
verified : Bool
-record AboutRelatedChannel,
- ucid : String,
- author : String,
- author_url : String,
- author_thumbnail : String
-
def get_about_info(ucid, locale) : AboutChannel
begin
# "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@@ -165,41 +159,15 @@ def get_about_info(ucid, locale) : AboutChannel
)
end
-def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel)
- # params is {"2:string":"channels"} encoded
- channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
-
- tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
- tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
-
- return [] of AboutRelatedChannel if tab.nil?
-
- items = tab.dig?(
- "tabRenderer", "content",
- "sectionListRenderer", "contents", 0,
- "itemSectionRenderer", "contents", 0,
- "gridRenderer", "items"
- ).try &.as_a?
-
- related = [] of AboutRelatedChannel
- return related if (items.nil? || items.empty?)
-
- items.each do |item|
- renderer = item["gridChannelRenderer"]?
- next if !renderer
-
- related_id = renderer.dig("channelId").as_s
- related_title = renderer.dig("title", "simpleText").as_s
- related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s
- related_author_thumbnail = HelperExtractors.get_thumbnails(renderer)
-
- related << AboutRelatedChannel.new(
- ucid: related_id,
- author: related_title,
- author_url: related_author_url,
- author_thumbnail: related_author_thumbnail,
- )
+def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
+ if continuation.nil?
+ # params is {"2:string":"channels"} encoded
+ initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
+ else
+ initial_data = YoutubeAPI.browse(continuation)
end
- return related
+ items, continuation = extract_items(initial_data)
+
+ return items.select(SearchChannel), continuation
end
diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr
index 7ac0e071..53745dd5 100644
--- a/src/invidious/frontend/channel_page.cr
+++ b/src/invidious/frontend/channel_page.cr
@@ -7,6 +7,7 @@ module Invidious::Frontend::ChannelPage
Streams
Playlists
Community
+ Channels
end
def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 4e92b54e..28ccdab9 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -102,31 +102,13 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
- related_channels = fetch_related_channels(channel)
+ related_channels, _ = fetch_related_channels(channel)
rescue ex
- related_channels = [] of AboutRelatedChannel
+ related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
- json.object do
- json.field "author", related_channel.author
- json.field "authorId", related_channel.ucid
- json.field "authorUrl", related_channel.author_url
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
+ related_channel.to_json(locale, json)
end
end
end # relatedChannels
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 77d309fb..d3969d29 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -147,6 +147,26 @@ module Invidious::Routes::Channels
templated "community"
end
+ def self.channels(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if channel.auto_generated
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ items, next_continuation = fetch_related_channels(channel, continuation)
+
+ # Featured/related channels can't be sorted
+ sort_options = [] of String
+ sort_by = nil
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
+ templated "channel"
+ end
+
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 0e6fba21..84dbed5b 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -119,6 +119,7 @@ module Invidious::Routing
get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
+ get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
get "/channel/:ucid/live", Routes::Channels, :live
get "/user/:user/live", Routes::Channels, :live
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 039f8752..a29315ef 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -8,6 +8,7 @@
when .shorts? then "/channel/#{ucid}/shorts"
when .streams? then "/channel/#{ucid}/streams"
when .playlists? then "/channel/#{ucid}/playlists"
+ when .channels? then "/channel/#{ucid}/channels"
else
"/channel/#{ucid}"
end
From 69b8e0919fd0a410d35f5f5fccc4753f79faf940 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 22 Dec 2022 17:26:30 +0100
Subject: [PATCH 15/17] api: Add support for the "featured channels" endpoint
---
src/invidious/routes/api/v1/channels.cr | 31 +++++++++++++++++++++++++
src/invidious/routing.cr | 1 +
2 files changed, 32 insertions(+)
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 28ccdab9..ca2b2734 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -283,6 +283,37 @@ module Invidious::Routes::API::V1::Channels
end
end
+ def self.channels(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ continuation = env.params.query["continuation"]?
+
+ begin
+ items, next_continuation = fetch_related_channels(channel, continuation)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.object do
+ json.field "relatedChannels" do
+ json.array do
+ items.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 84dbed5b..54bd82a4 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -225,6 +225,7 @@ module Invidious::Routing
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+ get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
{% for route in {"videos", "latest", "playlists", "community", "search"} %}
get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
From f9eb839c7ae2c29e641495c4a2affd384445bf97 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 22 Dec 2022 13:05:13 +0100
Subject: [PATCH 16/17] channel: remove dead playlists code
---
spec/invidious/helpers_spec.cr | 6 ----
src/invidious/channels/playlists.cr | 55 -----------------------------
2 files changed, 61 deletions(-)
diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr
index ab361770..f81cd29a 100644
--- a/spec/invidious/helpers_spec.cr
+++ b/spec/invidious/helpers_spec.cr
@@ -23,12 +23,6 @@ Spectator.describe "Helper" do
end
end
- describe "#produce_channel_playlists_url" do
- it "correctly produces a /browse_ajax URL with the given UCID and cursor" do
- expect(produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW")).to eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en")
- end
- end
-
describe "#produce_comment_continuation" do
it "correctly produces a continuation token for comments" do
expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D")
diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr
index 772eecb9..8dc824b2 100644
--- a/src/invidious/channels/playlists.cr
+++ b/src/invidious/channels/playlists.cr
@@ -26,58 +26,3 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
return extract_items(initial_data, author, ucid)
end
-
-# ## NOTE: DEPRECATED
-# Reason -> Unstable
-# The Protobuf object must be provided with an id of the last playlist from the current "page"
-# in order to fetch the next one accurately
-# (if the id isn't included, entries shift around erratically between pages,
-# leading to repetitions and skip overs)
-#
-# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user,
-# it's better to stick to continuation tokens provided by the first request and onward
-def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false)
- object = {
- "80226972:embedded" => {
- "2:string" => ucid,
- "3:base64" => {
- "2:string" => "playlists",
- "6:varint" => 2_i64,
- "7:varint" => 1_i64,
- "12:varint" => 1_i64,
- "13:string" => "",
- "23:varint" => 0_i64,
- },
- },
- }
-
- if cursor
- cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated
- object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor
- end
-
- if auto_generated
- object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64
- else
- object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64
- case sort
- when "oldest", "oldest_created"
- object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64
- when "newest", "newest_created"
- object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
- when "last", "last_added"
- object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
- else nil # Ignore
- end
- end
-
- object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
- object["80226972:embedded"].delete("3:base64")
-
- continuation = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
-end
From a37522a03dc12f61386fc0529a9136ad296b1228 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 8 Jan 2023 13:50:52 +0100
Subject: [PATCH 17/17] Implement workaround for broken shorts objects
---
src/invidious/channels/videos.cr | 30 ++++++++++++++++++++++----
src/invidious/exceptions.cr | 5 +++++
src/invidious/yt_backend/extractors.cr | 26 ++++++++++++----------
3 files changed, 46 insertions(+), 15 deletions(-)
diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index bea406c1..befec03d 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -127,16 +127,38 @@ module Invidious::Channel::Tabs
# Shorts
# -------------------
- def get_shorts(channel : AboutChannel, continuation : String? = nil)
+ private def fetch_shorts_data(ucid : String, continuation : String? = nil)
if continuation.nil?
# EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
# TODO: try to extract the continuation tokens that allows other sorting options
- initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
+ return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
else
- initial_data = YoutubeAPI.browse(continuation: continuation)
+ return YoutubeAPI.browse(continuation: continuation)
end
+ end
- return extract_items(initial_data, channel.author, channel.ucid)
+ def get_shorts(channel : AboutChannel, continuation : String? = nil)
+ initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+
+ begin
+ # Try to parse the initial data fetched above
+ return extract_items(initial_data, channel.author, channel.ucid)
+ rescue ex : RetryOnceException
+ # Sometimes, for a completely unknown reason, the "reelItemRenderer"
+ # object is missing some critical information (it happens once in about
+ # 20 subsequent requests). Refreshing the page is required to properly
+ # show the "shorts" tab.
+ #
+ # In order to make the experience smoother for the user, we simulate
+ # said page refresh by fetching again the JSON. If that still doesn't
+ # work, we raise a BrokenTubeException, as something is really broken.
+ begin
+ initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+ return extract_items(initial_data, channel.author, channel.ucid)
+ rescue ex : RetryOnceException
+ raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers"
+ end
+ end
end
# -------------------
diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr
index 425c08da..690db907 100644
--- a/src/invidious/exceptions.cr
+++ b/src/invidious/exceptions.cr
@@ -33,3 +33,8 @@ end
class VideoNotAvailableException < Exception
end
+
+# Exception used to indicate that the JSON response from YT is missing
+# some important informations, and that the query should be sent again.
+class RetryOnceException < Exception
+end
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index bca0dcbd..65d107b2 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -408,19 +408,23 @@ private module Parsers
private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s
- begin
- video_details_container = item_contents.dig(
- "navigationEndpoint", "reelWatchEndpoint",
- "overlay", "reelPlayerOverlayRenderer",
- "reelPlayerHeaderSupportedRenderers",
- "reelPlayerHeaderRenderer"
- )
- rescue ex : KeyError
- # Extract key name from original message
- key = /"([^"]+)"/.match(ex.message || "").try &.[1]?
- raise BrokenTubeException.new(key || "reelPlayerOverlayRenderer")
+ reel_player_overlay = item_contents.dig(
+ "navigationEndpoint", "reelWatchEndpoint",
+ "overlay", "reelPlayerOverlayRenderer"
+ )
+
+ # Sometimes, the "reelPlayerOverlayRenderer" object is missing the
+ # important part of the response. We use this exception to tell
+ # the calling function to fetch the content again.
+ if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
+ raise RetryOnceException.new
end
+ video_details_container = reel_player_overlay.dig(
+ "reelPlayerHeaderSupportedRenderers",
+ "reelPlayerHeaderRenderer"
+ )
+
# Author infos
author = video_details_container