From 98143b3791f5911e2fcfd5dbd2ffea56099438b8 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 8 Aug 2021 16:34:30 -0700 Subject: [PATCH] Migrate more types to data_structs --- src/invidious.cr | 20 +- src/invidious/channels/about.cr | 37 +- src/invidious/channels/channels.cr | 117 +--- src/invidious/channels/playlists.cr | 6 +- src/invidious/channels/videos.cr | 2 +- .../data_structs/invidious/channel.cr | 5 +- .../data_structs/invidious/playlists.cr | 17 +- .../data_structs/{ => youtube}/base.cr | 0 src/invidious/data_structs/youtube/caption.cr | 10 +- src/invidious/data_structs/youtube/channel.cr | 1 - .../data_structs/youtube/playlist_videos.cr | 2 +- .../data_structs/youtube/playlists.cr | 12 +- .../youtube/renderers/category.cr | 20 +- .../youtube/renderers/playlist_renderer.cr | 8 +- .../youtube/renderers/video_renderer.cr | 2 - src/invidious/data_structs/youtube/videos.cr | 54 +- src/invidious/helpers/helpers.cr | 9 +- src/invidious/helpers/serialized_yt_data.cr | 263 -------- src/invidious/jobs/pull_popular_videos_job.cr | 4 +- src/invidious/jobs/refresh_feeds_job.cr | 2 +- src/invidious/playlists.cr | 279 +-------- src/invidious/routes/api/v1/authenticated.cr | 14 +- src/invidious/routes/api/v1/channels.cr | 4 +- src/invidious/routes/api/v1/videos.cr | 2 +- src/invidious/routes/channels.cr | 4 +- src/invidious/routes/feeds.cr | 10 +- src/invidious/routes/playlists.cr | 22 +- src/invidious/search.cr | 20 +- src/invidious/users.cr | 12 +- src/invidious/videos.cr | 585 +----------------- src/invidious/views/components/item.ecr | 8 +- src/invidious/views/edit_playlist.ecr | 2 +- src/invidious/views/playlist.ecr | 12 +- src/invidious/yt_backend/extractors.cr | 39 +- src/invidious/yt_backend/extractors_utils.cr | 10 +- 35 files changed, 178 insertions(+), 1436 deletions(-) rename src/invidious/data_structs/{ => youtube}/base.cr (100%) delete mode 100644 src/invidious/helpers/serialized_yt_data.cr diff --git a/src/invidious.cr b/src/invidious.cr index b489bc8e..d4cea85d 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -115,17 +115,17 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) if CONFIG.check_tables check_enum(PG_DB, "privacy", PlaylistPrivacy) - check_table(PG_DB, "channels", InvidiousChannel) - check_table(PG_DB, "channel_videos", ChannelVideo) - check_table(PG_DB, "playlists", InvidiousPlaylist) - check_table(PG_DB, "playlist_videos", PlaylistVideo) + check_table(PG_DB, "channels", InvidiousStructs::Channel) + check_table(PG_DB, "channel_videos", InvidiousStructs::ChannelVideo) + check_table(PG_DB, "playlists", InvidiousStructs::Playlist) + check_table(PG_DB, "playlist_videos", YouTubeStructs::PlaylistVideo) check_table(PG_DB, "nonces", Nonce) check_table(PG_DB, "session_ids", SessionId) check_table(PG_DB, "users", User) - check_table(PG_DB, "videos", Video) + check_table(PG_DB, "videos", YouTubeStructs::Video) if CONFIG.cache_annotations - check_table(PG_DB, "annotations", Annotation) + check_table(PG_DB, "annotations", YouTubeStructs::Annotation) end end @@ -655,14 +655,14 @@ get "/subscription_manager" do |env| values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" end - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) - subscriptions.sort_by!(&.author.downcase) + subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousStructs::Channel) + subscriptions.sort_by! { |channel| channel.author.downcase } if action_takeout if format == "json" env.response.content_type = "application/json" env.response.headers["content-disposition"] = "attachment" - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist) next JSON.build do |json| json.object do @@ -804,7 +804,7 @@ post "/data_control" do |env| next end - playlist_video = PlaylistVideo.new({ + playlist_video = YouTubeStructs::PlaylistVideo.new({ title: video.title, id: video.id, author: video.author, diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index c87c53e0..8111dc32 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -1,32 +1,3 @@ -# TODO: Refactor into either SearchChannel or InvidiousChannel -struct AboutChannel - include DB::Serializable - - property ucid : String - property author : String - property auto_generated : Bool - property author_url : String - property author_thumbnail : String - property banner : String? - property description_html : String - property total_views : Int64 - property sub_count : Int32 - property joined : Time - property is_family_friendly : Bool - property allowed_regions : Array(String) - property related_channels : Array(AboutRelatedChannel) - property tabs : Array(String) -end - -struct AboutRelatedChannel - include DB::Serializable - - property ucid : String - property author : String - property author_url : String - property author_thumbnail : String -end - def get_about_info(ucid, locale) begin # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} @@ -64,7 +35,7 @@ def get_about_info(ucid, locale) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) - related_channels = [] of AboutRelatedChannel + related_channels = [] of YouTubeStructs::AboutRelatedChannel else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -109,14 +80,14 @@ def get_about_info(ucid, locale) related_author_thumbnail ||= "" end - AboutRelatedChannel.new({ + YouTubeStructs::AboutRelatedChannel.new({ ucid: related_id, author: related_title, author_url: related_author_url, author_thumbnail: related_author_thumbnail, }) end - related_channels ||= [] of AboutRelatedChannel + related_channels ||= [] of YouTubeStructs::AboutRelatedChannel end total_views = 0_i64 @@ -155,7 +126,7 @@ def get_about_info(ucid, locale) sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 - AboutChannel.new({ + YouTubeStructs::AboutChannel.new({ ucid: ucid, author: author, auto_generated: auto_generated, diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 827b6534..449fc3c3 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -1,112 +1,3 @@ -struct InvidiousChannel - include DB::Serializable - - property id : String - property author : String - property updated : Time - property deleted : Bool - property subscribed : Time? -end - -struct ChannelVideo - include DB::Serializable - - property id : String - property title : String - property published : Time - property updated : Time - property ucid : String - property author : String - property length_seconds : Int32 = 0 - property live_now : Bool = false - property premiere_timestamp : Time? = nil - property views : Int64? = nil - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "shortVideo" - - json.field "title", self.title - json.field "videoId", self.id - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - - json.field "lengthSeconds", self.length_seconds - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - - json.field "viewCount", self.views - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def to_xml(locale, query_params, xml : XML::Builder) - query_params["v"] = self.id - - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") - - xml.element("author") do - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - end - end - end - - def to_xml(locale, xml : XML::Builder | Nil = nil) - if xml - to_xml(locale, xml) - else - XML.build do |xml| - to_xml(locale, xml) - end - end - end - - def to_tuple - {% begin %} - { - {{*@type.instance_vars.map(&.name)}} - } - {% end %} - end -end - class ChannelRedirect < Exception property channel_id : String @@ -152,7 +43,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma end def get_channel(id, db, refresh = true, pull_all_videos = true) - if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) + if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousStructs::Channel) if refresh && Time.utc - channel.updated > 10.minutes channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) channel_array = channel.to_a @@ -224,7 +115,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) premiere_timestamp = channel_video.try &.premiere_timestamp - video = ChannelVideo.new({ + video = InvidiousStructs::ChannelVideo.new({ id: video_id, title: title, published: published, @@ -265,7 +156,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) videos = extract_videos(initial_data, author, ucid) count = videos.size - videos = videos.map { |video| ChannelVideo.new({ + videos = videos.map { |video| InvidiousStructs::ChannelVideo.new({ id: video.id, title: video.title, published: video.published, @@ -299,7 +190,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) end end - channel = InvidiousChannel.new({ + channel = InvidiousStructs::Channel.new({ id: ucid, author: author, updated: Time.utc, diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index d5628f6a..fd525d11 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -4,9 +4,9 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) continuation_items = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - return [] of SearchItem, nil if !continuation_items + return [] of YouTubeStructs::Renderer, nil if !continuation_items - items = [] of SearchItem + items = [] of YouTubeStructs::Renderer continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| extract_item(item, author, ucid).try { |t| items << t } } @@ -28,7 +28,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) response = YT_POOL.client &.get(url) initial_data = extract_initial_data(response.body) - return [] of SearchItem, nil if !initial_data + return [] of YouTubeStructs::Renderer, nil if !initial_data items = extract_items(initial_data, author, ucid) continuation = response.body.match(/"token":"(?[^"]+)"/).try &.["continuation"]? diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 48453bb7..10483920 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -65,7 +65,7 @@ def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") - videos = [] of SearchVideo + videos = [] of YouTubeStructs::VideoRenderer 2.times do |i| initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) diff --git a/src/invidious/data_structs/invidious/channel.cr b/src/invidious/data_structs/invidious/channel.cr index 75a9e71e..bed6329d 100644 --- a/src/invidious/data_structs/invidious/channel.cr +++ b/src/invidious/data_structs/invidious/channel.cr @@ -1,11 +1,10 @@ - # Data structs used by Invidious to provide certain features. module InvidiousStructs # Struct for representing a cached YouTube channel. # # This is constructed from YouTube's RSS feeds for channels and is # currently only used for storing subscriptions in a user. - struct InvidiousChannel + struct Channel include DB::Serializable property id : String @@ -119,7 +118,7 @@ module InvidiousStructs def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| var.name }}} + {{*@type.instance_vars.map(&.name)}} } {% end %} end diff --git a/src/invidious/data_structs/invidious/playlists.cr b/src/invidious/data_structs/invidious/playlists.cr index 364cc77c..858b2438 100644 --- a/src/invidious/data_structs/invidious/playlists.cr +++ b/src/invidious/data_structs/invidious/playlists.cr @@ -1,5 +1,6 @@ module InvidiousStructs - private module PlaylistPrivacyConverter + # Converter to parse a Invidious privacy type string to enum + module PlaylistPrivacyConverter def self.from_rs(rs) return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) end @@ -16,14 +17,14 @@ module InvidiousStructs property created : Time property updated : Time - @[DB::Field(converter: PlaylistPrivacyConverter)] + @[DB::Field(converter: InvidiousStructs::PlaylistPrivacyConverter)] property privacy : PlaylistPrivacy = PlaylistPrivacy::Private property index : Array(Int64) @[DB::Field(ignore: true)] property thumbnail_id : String? - def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title @@ -45,11 +46,11 @@ module InvidiousStructs json.field "videos" do json.array do if !offset || offset == 0 - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64) + index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, video_id, as: Int64) offset = self.index.index(index) || 0 end - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| video.to_json(locale, json, offset + index) end @@ -58,12 +59,12 @@ module InvidiousStructs end end - def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) if json - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) else JSON.build do |json| - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) end end end diff --git a/src/invidious/data_structs/base.cr b/src/invidious/data_structs/youtube/base.cr similarity index 100% rename from src/invidious/data_structs/base.cr rename to src/invidious/data_structs/youtube/base.cr diff --git a/src/invidious/data_structs/youtube/caption.cr b/src/invidious/data_structs/youtube/caption.cr index c6f278d8..e0fa452b 100644 --- a/src/invidious/data_structs/youtube/caption.cr +++ b/src/invidious/data_structs/youtube/caption.cr @@ -1,16 +1,16 @@ module YouTubeStructs struct Caption property name - property languageCode - property baseUrl + property language_code + property base_url getter name : String - getter languageCode : String - getter baseUrl : String + getter language_code : String + getter base_url : String setter name - def initialize(@name, @languageCode, @baseUrl) + def initialize(@name, @language_code, @base_url) end end end diff --git a/src/invidious/data_structs/youtube/channel.cr b/src/invidious/data_structs/youtube/channel.cr index 6cf8745d..94f59a6d 100644 --- a/src/invidious/data_structs/youtube/channel.cr +++ b/src/invidious/data_structs/youtube/channel.cr @@ -14,7 +14,6 @@ module YouTubeStructs property author_thumbnail : String property banner : String? property description_html : String - property paid : Bool property total_views : Int64 property sub_count : Int32 property joined : Time diff --git a/src/invidious/data_structs/youtube/playlist_videos.cr b/src/invidious/data_structs/youtube/playlist_videos.cr index b27221bd..362be8b1 100644 --- a/src/invidious/data_structs/youtube/playlist_videos.cr +++ b/src/invidious/data_structs/youtube/playlist_videos.cr @@ -55,7 +55,7 @@ module YouTubeStructs if xml to_xml(auto_generated, xml) else - XML.build do |json| + XML.build do |xml| to_xml(auto_generated, xml) end end diff --git a/src/invidious/data_structs/youtube/playlists.cr b/src/invidious/data_structs/youtube/playlists.cr index 55749547..2df1c347 100644 --- a/src/invidious/data_structs/youtube/playlists.cr +++ b/src/invidious/data_structs/youtube/playlists.cr @@ -14,7 +14,7 @@ module YouTubeStructs property updated : Time property thumbnail : String? - def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title @@ -49,8 +49,8 @@ module YouTubeStructs json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) - videos.each_with_index do |video, index| + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) + videos.each do |video| video.to_json(locale, json) end end @@ -58,12 +58,12 @@ module YouTubeStructs end end - def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) if json - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) else JSON.build do |json| - to_json(offset, locale, json, continuation: continuation) + to_json(offset, locale, json, video_id: video_id) end end end diff --git a/src/invidious/data_structs/youtube/renderers/category.cr b/src/invidious/data_structs/youtube/renderers/category.cr index 92f512c8..5c29548b 100644 --- a/src/invidious/data_structs/youtube/renderers/category.cr +++ b/src/invidious/data_structs/youtube/renderers/category.cr @@ -18,19 +18,25 @@ module YouTubeStructs property description_html : String property badges : Array(Tuple(String, String))? - # Extracts all renderers out of the category's contents. - def extract_renderers() - target = [] of Renderer + # Extracts all renderers out of the category's contents. + def extract_renderers() + target = [] of Renderer - @contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } + @contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } - return target - end + return target + end def to_json(locale, json : JSON::Builder) json.object do json.field "title", self.title - json.field "contents", self.contents + json.field "contents" do + json.array do + self.contents.each do |item| + item.to_json(locale, json) + end + end + end end end diff --git a/src/invidious/data_structs/youtube/renderers/playlist_renderer.cr b/src/invidious/data_structs/youtube/renderers/playlist_renderer.cr index 1130085a..041a1ad9 100644 --- a/src/invidious/data_structs/youtube/renderers/playlist_renderer.cr +++ b/src/invidious/data_structs/youtube/renderers/playlist_renderer.cr @@ -37,12 +37,12 @@ module YouTubeStructs json.array do self.videos.each do |video| json.object do - json.field "title", video.title - json.field "videoId", video.id - json.field "lengthSeconds", video.length_seconds + json.field "title", video[:title] + json.field "videoId", video[:id] + json.field "lengthSeconds", video[:length_seconds] json.field "videoThumbnails" do - generate_thumbnails(json, video.id) + generate_thumbnails(json, video[:id]) end end end diff --git a/src/invidious/data_structs/youtube/renderers/video_renderer.cr b/src/invidious/data_structs/youtube/renderers/video_renderer.cr index 16c610bd..1e8bb987 100644 --- a/src/invidious/data_structs/youtube/renderers/video_renderer.cr +++ b/src/invidious/data_structs/youtube/renderers/video_renderer.cr @@ -20,7 +20,6 @@ module YouTubeStructs property description_html : String property length_seconds : Int32 property live_now : Bool - property paid : Bool property premium : Bool property premiere_timestamp : Time? @@ -101,7 +100,6 @@ module YouTubeStructs json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) json.field "lengthSeconds", self.length_seconds json.field "liveNow", self.live_now - json.field "paid", self.paid json.field "premium", self.premium json.field "isUpcoming", self.is_upcoming diff --git a/src/invidious/data_structs/youtube/videos.cr b/src/invidious/data_structs/youtube/videos.cr index 55275fdd..e26dcfea 100644 --- a/src/invidious/data_structs/youtube/videos.cr +++ b/src/invidious/data_structs/youtube/videos.cr @@ -1,10 +1,31 @@ module YouTubeStructs + # Converter to serialize first level JSON data as methods for the videos struct + module VideoJSONConverter + def self.from_rs(rs) + JSON.parse(rs.read(String)).as_h + end + end + + # Represents an watchable video in Invidious + # + # The video struct only takes three parameters: + # - ID: The video ID + # + # - Info: + # YT Video information (streams, captions, tiles, etc). This is then serialized + # into individual properties that either stores top level stuff or accesses + # further nested data. + # + # - Updated: + # A record of when the specific struct was created and inserted + # into the DB. This is then used to measure when to cache (or update) + # videos within the database. struct Video include DB::Serializable property id : String - @[DB::Field(converter: Video::JSONConverter)] + @[DB::Field(converter: YouTubeStructs::VideoJSONConverter)] property info : Hash(String, JSON::Any) property updated : Time @@ -20,12 +41,6 @@ module YouTubeStructs @[DB::Field(ignore: true)] property description : String? - module JSONConverter - def self.from_rs(rs) - JSON.parse(rs.read(String)).as_h - end - end - def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) json.object do json.field "type", "video" @@ -177,7 +192,7 @@ module YouTubeStructs self.captions.each do |caption| json.object do json.field "label", caption.name - json.field "languageCode", caption.languageCode + json.field "language_code", caption.language_code json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end @@ -277,10 +292,6 @@ module YouTubeStructs info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end - def cookie - info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - end - def allow_ratings r = info["videoDetails"]["allowRatings"]?.try &.as_bool r.nil? ? false : r @@ -458,10 +469,10 @@ module YouTubeStructs return @captions.as(Array(Caption)) if @captions captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - languageCode = caption["languageCode"].to_s - baseUrl = caption["baseUrl"].to_s + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s - caption = Caption.new(name.to_s, languageCode, baseUrl) + caption = Caption.new(name.to_s, language_code, base_url) caption.name = caption.name.split(" - ")[0] caption end @@ -516,8 +527,13 @@ module YouTubeStructs info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false end - def is_vr : Bool - info["streamingData"]?.try &.["adaptiveFormats"].as_a[0]?.try &.["projectionType"].as_s == "MESH" ? true : false || false + def is_vr : Bool? + projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s + return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type + end + + def projection_type : String? + return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s end def wilson_score : Float64 @@ -531,9 +547,5 @@ module YouTubeStructs def reason : String? info["reason"]?.try &.as_s end - - def session_token : String? - info["sessionToken"]?.try &.as_s? - end end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 9c053d74..db349cb9 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -15,13 +15,6 @@ struct SessionId property issued : String end -struct Annotation - include DB::Serializable - - property id : String - property annotations : String -end - def login_req(f_req) data = { # Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard @@ -239,7 +232,7 @@ def create_notification_stream(env, topics, connection_channel) case topic when .match(/UC[A-Za-z0-9_-]{22}/) PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", - topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| + topic, Time.unix(since.not_nil!), as: InvidiousStructs::ChannelVideo).each do |video| response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr deleted file mode 100644 index 1ba3f20e..00000000 --- a/src/invidious/helpers/serialized_yt_data.cr +++ /dev/null @@ -1,263 +0,0 @@ -struct SearchVideo - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property published : Time - property views : Int64 - property description_html : String - property length_seconds : Int32 - property live_now : Bool - property premium : Bool - property premiere_timestamp : Time? - - def to_xml(auto_generated, query_params, xml : XML::Builder) - query_params["v"] = self.id - - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") - - xml.element("author") do - if auto_generated - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } - end - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - - xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - xml.element("media:description") { xml.text html_to_content(self.description_html) } - end - - xml.element("media:community") do - xml.element("media:statistics", views: self.views) - end - end - end - - def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) - if xml - to_xml(HOST_URL, auto_generated, query_params, xml) - else - XML.build do |xml| - to_xml(HOST_URL, auto_generated, query_params, xml) - end - end - end - - def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) - json.object do - json.field "type", "video" - json.field "title", self.title - json.field "videoId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - - json.field "viewCount", self.views - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming - - if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def is_upcoming - premiere_timestamp ? true : false - end -end - -struct SearchPlaylistVideo - include DB::Serializable - - property title : String - property id : String - property length_seconds : Int32 -end - -struct SearchPlaylist - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property video_count : Int32 - property videos : Array(SearchPlaylistVideo) - property thumbnail : String? - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "playlist" - json.field "title", self.title - json.field "playlistId", self.id - json.field "playlistThumbnail", self.thumbnail - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoCount", self.video_count - json.field "videos" do - json.array do - self.videos.each do |video| - json.object do - json.field "title", video.title - json.field "videoId", video.id - json.field "lengthSeconds", video.length_seconds - - json.field "videoThumbnails" do - generate_thumbnails(json, video.id) - end - end - end - end - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end -end - -struct SearchChannel - include DB::Serializable - - property author : String - property ucid : String - property author_thumbnail : String - property subscriber_count : Int32 - property video_count : Int32 - property description_html : String - property auto_generated : Bool - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "channel" - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "autoGenerated", self.auto_generated - json.field "subCount", self.subscriber_count - json.field "videoCount", self.video_count - - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end -end - -class Category - include DB::Serializable - - property title : String - property contents : Array(SearchItem) | Array(Video) - property url : String? - property description_html : String - property badges : Array(Tuple(String, String))? - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "category" - json.field "title", self.title - json.field "contents" do - json.array do - self.contents.each do |item| - item.to_json(locale, json) - end - end - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end -end - -alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category diff --git a/src/invidious/jobs/pull_popular_videos_job.cr b/src/invidious/jobs/pull_popular_videos_job.cr index 38de816e..858a769e 100644 --- a/src/invidious/jobs/pull_popular_videos_job.cr +++ b/src/invidious/jobs/pull_popular_videos_job.cr @@ -6,7 +6,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) ORDER BY ucid, published DESC SQL - POPULAR_VIDEOS = Atomic.new([] of ChannelVideo) + POPULAR_VIDEOS = Atomic.new([] of InvidiousStructs::ChannelVideo) private getter db : DB::Database def initialize(@db) @@ -14,7 +14,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob def begin loop do - videos = db.query_all(QUERY, as: ChannelVideo) + videos = db.query_all(QUERY, as: InvidiousStructs::ChannelVideo) .sort_by!(&.published) .reverse! diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 926c27fa..a56d7d76 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -26,7 +26,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob begin # Drop outdated views column_array = get_column_array(db, view_name) - ChannelVideo.type_array.each_with_index do |name, i| + InvidiousStructs::ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}") diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 443d19d7..ead9e1c5 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -1,272 +1,13 @@ -struct PlaylistVideo - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property length_seconds : Int32 - property published : Time - property plid : String - property index : Int64 - property live_now : Bool - - def to_xml(auto_generated, xml : XML::Builder) - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}") - - xml.element("author") do - if auto_generated - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } - end - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - end - end - end - - def to_xml(auto_generated, xml : XML::Builder? = nil) - if xml - to_xml(auto_generated, xml) - else - XML.build do |xml| - to_xml(auto_generated, xml) - end - end - end - - def to_json(locale, json : JSON::Builder, index : Int32?) - json.object do - json.field "title", self.title - json.field "videoId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - - if index - json.field "index", index - json.field "indexId", self.index.to_u64.to_s(16).upcase - else - json.field "index", self.index - end - - json.field "lengthSeconds", self.length_seconds - end - end - - def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil) - if json - to_json(locale, json, index: index) - else - JSON.build do |json| - to_json(locale, json, index: index) - end - end - end -end - -struct Playlist - include DB::Serializable - - property title : String - property id : String - property author : String - property author_thumbnail : String - property ucid : String - property description : String - property description_html : String - property video_count : Int32 - property views : Int64 - property updated : Time - property thumbnail : String? - - def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) - json.object do - json.field "type", "playlist" - json.field "title", self.title - json.field "playlistId", self.id - json.field "playlistThumbnail", self.thumbnail - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "description", self.description - json.field "descriptionHtml", self.description_html - json.field "videoCount", self.video_count - - json.field "viewCount", self.views - json.field "updated", self.updated.to_unix - json.field "isListed", self.privacy.public? - - json.field "videos" do - json.array do - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) - videos.each do |video| - video.to_json(locale, json) - end - end - end - end - end - - def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) - if json - to_json(offset, locale, json, video_id: video_id) - else - JSON.build do |json| - to_json(offset, locale, json, video_id: video_id) - end - end - end - - def privacy - PlaylistPrivacy::Public - end -end - enum PlaylistPrivacy Public = 0 Unlisted = 1 Private = 2 end -struct InvidiousPlaylist - include DB::Serializable - - property title : String - property id : String - property author : String - property description : String = "" - property video_count : Int32 - property created : Time - property updated : Time - - @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)] - property privacy : PlaylistPrivacy = PlaylistPrivacy::Private - property index : Array(Int64) - - @[DB::Field(ignore: true)] - property thumbnail_id : String? - - module PlaylistPrivacyConverter - def self.from_rs(rs) - return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) - end - end - - def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) - json.object do - json.field "type", "invidiousPlaylist" - json.field "title", self.title - json.field "playlistId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", nil - json.field "authorThumbnails", [] of String - - json.field "description", html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - json.field "videoCount", self.video_count - - json.field "viewCount", self.views - json.field "updated", self.updated.to_unix - json.field "isListed", self.privacy.public? - - json.field "videos" do - json.array do - if !offset || offset == 0 - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, video_id, as: Int64) - offset = self.index.index(index) || 0 - end - - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) - videos.each_with_index do |video, index| - video.to_json(locale, json, offset + index) - end - end - end - end - end - - def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) - if json - to_json(offset, locale, json, video_id: video_id) - else - JSON.build do |json| - to_json(offset, locale, json, video_id: video_id) - end - end - end - - def thumbnail - @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" - "/vi/#{@thumbnail_id}/mqdefault.jpg" - end - - def author_thumbnail - nil - end - - def ucid - nil - end - - def views - 0_i64 - end - - def description_html - HTML.escape(self.description).gsub("\n", "
") - end -end - def create_playlist(db, title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" - playlist = InvidiousPlaylist.new({ + playlist = InvidiousStructs::Playlist.new({ title: title.byte_slice(0, 150), id: plid, author: user.email, @@ -287,7 +28,7 @@ def create_playlist(db, title, privacy, user) end def subscribe_playlist(db, user, playlist) - playlist = InvidiousPlaylist.new({ + playlist = InvidiousStructs::Playlist.new({ title: playlist.title.byte_slice(0, 150), id: playlist.id, author: user.email, @@ -346,7 +87,7 @@ end def get_playlist(db, plid, locale, refresh = true, force_refresh = false) if plid.starts_with? "IV" - if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) return playlist else raise InfoException.new("Playlist does not exist.") @@ -411,7 +152,7 @@ def fetch_playlist(plid, locale) ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || "" end - return Playlist.new({ + return YouTubeStructs::Playlist.new({ title: title, id: plid, author: author, @@ -430,12 +171,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) # Show empy playlist if requested page is out of range # (e.g, when a new playlist has been created, offset will be negative) if offset >= playlist.video_count || offset < 0 - return [] of PlaylistVideo + return [] of YouTubeStructs::PlaylistVideo end - if playlist.is_a? InvidiousPlaylist + if playlist.is_a? InvidiousStructs::Playlist db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", - playlist.id, playlist.index, offset, as: PlaylistVideo) + playlist.id, playlist.index, offset, as: YouTubeStructs::PlaylistVideo) else if video_id initial_data = YoutubeAPI.next({ @@ -445,7 +186,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset end - videos = [] of PlaylistVideo + videos = [] of YouTubeStructs::PlaylistVideo until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count # 100 videos per request @@ -461,7 +202,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo + videos = [] of YouTubeStructs::PlaylistVideo if initial_data["contents"]? tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] @@ -502,7 +243,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) length_seconds = 0 end - videos << PlaylistVideo.new({ + videos << YouTubeStructs::PlaylistVideo.new({ title: title, id: video_id, author: author, diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 7950b302..160990e4 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -78,7 +78,7 @@ module Invidious::Routes::API::V1::Authenticated values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" end - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousStructs::Channel) JSON.build do |json| json.array do @@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) + playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousStructs::Playlist) JSON.build do |json| json.array do @@ -174,7 +174,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -207,7 +207,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -230,7 +230,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -254,7 +254,7 @@ module Invidious::Routes::API::V1::Authenticated return error_json(500, ex) end - playlist_video = PlaylistVideo.new({ + playlist_video = YouTubeStructs::PlaylistVideo.new({ title: video.title, id: video.id, author: video.author, @@ -286,7 +286,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] index = env.params.url["index"].to_i64(16) - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index da39661c..9fe9601c 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -19,7 +19,7 @@ module Invidious::Routes::API::V1::Channels page = 1 if channel.auto_generated - videos = [] of SearchVideo + videos = [] of YouTubeStructs::VideoRenderer count = 0 else begin @@ -208,7 +208,7 @@ module Invidious::Routes::API::V1::Channels json.field "playlists" do json.array do items.each do |item| - item.to_json(locale, json) if item.is_a?(SearchPlaylist) + item.to_json(locale, json) if item.is_a?(YouTubeStructs::PlaylistRenderer) end end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index d483bca6..131a1ce3 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos case source when "archive" - if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) + if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: YouTubeStructs::Annotation)) annotations = cached_annotation.annotations else index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 29748cd0..b118eea5 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -29,7 +29,7 @@ module Invidious::Routes::Channels item.author end end - items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) + items = items.select(YouTubeStructs::PlaylistRenderer).map(&.as(YouTubeStructs::PlaylistRenderer)) items.each(&.author = "") else sort_options = {"newest", "oldest", "popular"} @@ -57,7 +57,7 @@ module Invidious::Routes::Channels end items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) - items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) + items = items.select(YouTubeStructs::PlaylistRenderer).map(&.as(YouTubeStructs::PlaylistRenderer)) items.each(&.author = "") templated "playlists" diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40c41dc1..adb044c7 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -15,13 +15,13 @@ module Invidious::Routes::Feeds user = user.as(User) - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist) items_created.map! do |item| item.author = "" item end - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist) items_saved.map! do |item| item.author = "" item @@ -169,7 +169,7 @@ module Invidious::Routes::Feeds description_html = entry.xpath_node("group/description").not_nil!.to_s views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 - SearchVideo.new({ + YouTubeStructs::VideoRenderer.new({ title: title, id: video_id, author: author, @@ -264,7 +264,7 @@ module Invidious::Routes::Feeds path = env.request.path if plid.starts_with? "IV" - if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) return XML.build(indent: " ", encoding: "UTF-8") do |xml| @@ -405,7 +405,7 @@ module Invidious::Routes::Feeds }.to_json PG_DB.exec("NOTIFY notifications, E'#{payload}'") - video = ChannelVideo.new({ + video = InvidiousStructs::ChannelVideo.new({ id: id, title: video.title, published: published, diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 21126d7e..59e350b4 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -85,7 +85,7 @@ module Invidious::Routes::Playlists sid = sid.as(String) plid = env.params.query["list"]? - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email return env.redirect referer end @@ -117,7 +117,7 @@ module Invidious::Routes::Playlists return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email return env.redirect referer end @@ -149,7 +149,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email return env.redirect referer end @@ -160,7 +160,7 @@ module Invidious::Routes::Playlists begin videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) rescue ex - videos = [] of PlaylistVideo + videos = [] of YouTubeStructs::PlaylistVideo end csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) @@ -190,7 +190,7 @@ module Invidious::Routes::Playlists return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email return env.redirect referer end @@ -233,7 +233,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist) if !playlist || playlist.author != user.email return env.redirect referer end @@ -245,13 +245,13 @@ module Invidious::Routes::Playlists if query begin search_query, count, items, operators = process_search_query(query, page, user, region: nil) - videos = items.select(SearchVideo).map(&.as(SearchVideo)) + videos = items.select(YouTubeStructs::VideoRenderer).map(&.as(YouTubeStructs::VideoRenderer)) rescue ex - videos = [] of SearchVideo + videos = [] of YouTubeStructs::VideoRenderer count = 0 end else - videos = [] of SearchVideo + videos = [] of YouTubeStructs::VideoRenderer count = 0 end @@ -311,7 +311,7 @@ module Invidious::Routes::Playlists begin playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) + playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousStructs::Playlist) raise "Invalid user" if playlist.author != user.email rescue ex if redirect @@ -351,7 +351,7 @@ module Invidious::Routes::Playlists end end - playlist_video = PlaylistVideo.new({ + playlist_video = YouTubeStructs::PlaylistVideo.new({ title: video.title, id: video.id, author: video.author, diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 2095721c..7aef3c05 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -17,9 +17,9 @@ def channel_search(query, page, channel) continuation_items = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - return 0, [] of SearchItem if !continuation_items + return 0, [] of YouTubeStructs::Renderer if !continuation_items - items = [] of SearchItem + items = [] of YouTubeStructs::Renderer continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| extract_item(item["itemSectionRenderer"]["contents"].as_a[0]) .try { |t| items << t } @@ -29,7 +29,7 @@ def channel_search(query, page, channel) end def search(query, search_params = produce_search_params(content_type: "all"), region = nil) - return 0, [] of SearchItem if query.empty? + return 0, [] of YouTubeStructs::Renderer if query.empty? client_config = YoutubeAPI::ClientConfig.new(region: region) initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) @@ -219,10 +219,10 @@ def process_search_query(query, page, user, region) to_tsvector(#{view_name}.author) as document FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) + ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: InvidiousStructs::ChannelVideo) count = items.size else - items = [] of ChannelVideo + items = [] of InvidiousStructs::ChannelVideo count = 0 end else @@ -234,14 +234,10 @@ def process_search_query(query, page, user, region) # Light processing to flatten search results out of Categories. # They should ideally be supported in the future. - items_without_category = [] of SearchItem | ChannelVideo + items_without_category = [] of YouTubeStructs::Renderer | InvidiousStructs::ChannelVideo items.each do |i| - if i.is_a? Category - i.contents.each do |nest_i| - if !nest_i.is_a? Video - items_without_category << nest_i - end - end + if i.is_a? YouTubeStructs::Category + items_without_category += i.extract_renderers else items_without_category << i end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 584082be..52704524 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -245,8 +245,8 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) args = arg_array(notifications) - notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo) - videos = [] of ChannelVideo + notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: InvidiousStructs::ChannelVideo) + videos = [] of InvidiousStructs::ChannelVideo notifications.sort_by!(&.published).reverse! @@ -272,11 +272,11 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) else values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" end - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo) + videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: InvidiousStructs::ChannelVideo) else # Show latest video from each channel - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) + videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: InvidiousStructs::ChannelVideo) end videos.sort_by!(&.published).reverse! @@ -289,11 +289,11 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) else values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" end - videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) + videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: InvidiousStructs::ChannelVideo) else # Sort subscriptions as normal - videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) + videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: InvidiousStructs::ChannelVideo) end end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d38a66d8..f5dcf162 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -221,583 +221,6 @@ VIDEO_FORMATS = { "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, } -struct VideoPreferences - include JSON::Serializable - - property annotations : Bool - property autoplay : Bool - property comments : Array(String) - property continue : Bool - property continue_autoplay : Bool - property controls : Bool - property listen : Bool - property local : Bool - property preferred_captions : Array(String) - property player_style : String - property quality : String - property quality_dash : String - property raw : Bool - property region : String? - property related_videos : Bool - property speed : Float32 | Float64 - property video_end : Float64 | Int32 - property video_loop : Bool - property extend_desc : Bool - property video_start : Float64 | Int32 - property volume : Int32 - property vr_mode : Bool -end - -struct Video - include DB::Serializable - - property id : String - - @[DB::Field(converter: Video::JSONConverter)] - property info : Hash(String, JSON::Any) - property updated : Time - - @[DB::Field(ignore: true)] - property captions : Array(Caption)? - - @[DB::Field(ignore: true)] - property adaptive_fmts : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] - property fmt_stream : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] - property description : String? - - module JSONConverter - def self.from_rs(rs) - JSON.parse(rs.read(String)).as_h - end - end - - def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) - json.object do - json.field "type", "video" - - json.field "title", self.title - json.field "videoId", self.id - - json.field "error", info["reason"] if info["reason"]? - - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - json.field "storyboards" do - generate_storyboards(json, self.id, self.storyboards) - end - - json.field "description", self.description - json.field "descriptionHtml", self.description_html - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - json.field "keywords", self.keywords - - json.field "viewCount", self.views - json.field "likeCount", self.likes - json.field "dislikeCount", self.dislikes - - json.field "paid", self.paid - json.field "premium", self.premium - json.field "isFamilyFriendly", self.is_family_friendly - json.field "allowedRegions", self.allowed_regions - json.field "genre", self.genre - json.field "genreUrl", self.genre_url - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "subCountText", self.sub_count_text - - json.field "lengthSeconds", self.length_seconds - json.field "allowRatings", self.allow_ratings - json.field "rating", self.average_rating - json.field "isListed", self.is_listed - json.field "liveNow", self.live_now - json.field "isUpcoming", self.is_upcoming - - if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix - end - - if hlsvp = self.hls_manifest_url - hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) - json.field "hlsUrl", hlsvp - end - - json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}" - - json.field "adaptiveFormats" do - json.array do - self.adaptive_fmts.each do |fmt| - json.object do - json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}" - json.field "bitrate", fmt["bitrate"].as_i.to_s - json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}" - json.field "url", fmt["url"] - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "clen", fmt["contentLength"] - json.field "lmt", fmt["lastModified"] - json.field "projectionType", fmt["projectionType"] - - fmt_info = itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] - - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" - - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label - - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end - end - end - end - end - end - end - - json.field "formatStreams" do - json.array do - self.fmt_stream.each do |fmt| - json.object do - json.field "url", fmt["url"] - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "quality", fmt["quality"] - - fmt_info = itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] - json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] - - if fmt_info["height"]? - json.field "resolution", "#{fmt_info["height"]}p" - - quality_label = "#{fmt_info["height"]}p" - if fps > 30 - quality_label += "60" - end - json.field "qualityLabel", quality_label - - if fmt_info["width"]? - json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" - end - end - end - end - end - end - end - - json.field "captions" do - json.array do - self.captions.each do |caption| - json.object do - json.field "label", caption.name - json.field "language_code", caption.language_code - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" - end - end - end - end - - json.field "recommendedVideos" do - json.array do - self.related_videos.each do |rv| - if rv["id"]? - json.object do - json.field "videoId", rv["id"] - json.field "title", rv["title"] - json.field "videoThumbnails" do - generate_thumbnails(json, rv["id"]) - end - - json.field "author", rv["author"] - json.field "authorUrl", rv["author_url"]? - json.field "authorId", rv["ucid"]? - if rv["author_thumbnail"]? - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality - end - end - end - end - end - - json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i - json.field "viewCountText", rv["short_view_count_text"]? - json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 - end - end - end - end - end - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def title - info["videoDetails"]["title"]?.try &.as_s || "" - end - - def ucid - info["videoDetails"]["channelId"]?.try &.as_s || "" - end - - def author - info["videoDetails"]["author"]?.try &.as_s || "" - end - - def length_seconds : Int32 - info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i || - info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 - end - - def views : Int64 - info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 - end - - def likes : Int64 - info["likes"]?.try &.as_i64 || 0_i64 - end - - def dislikes : Int64 - info["dislikes"]?.try &.as_i64 || 0_i64 - end - - def average_rating : Float64 - # (likes / (likes + dislikes) * 4 + 1) - info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0 - end - - def published : Time - info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc - end - - def published=(other : Time) - info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) - end - - def allow_ratings - r = info["videoDetails"]["allowRatings"]?.try &.as_bool - r.nil? ? false : r - end - - def live_now - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false - end - - def is_listed - info["videoDetails"]["isCrawlable"]?.try &.as_bool || false - end - - def is_upcoming - info["videoDetails"]["isUpcoming"]?.try &.as_bool || false - end - - def premiere_timestamp : Time? - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) } - end - - def keywords - info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String - end - - def related_videos - info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) - end - - def allowed_regions - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String - end - - def author_thumbnail : String - info["authorThumbnail"]?.try &.as_s || "" - end - - def sub_count_text : String - info["subCountText"]?.try &.as_s || "-" - end - - def fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream - - fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? - end - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @fmt_stream = fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) - end - - def adaptive_fmts - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts - fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? - end - # See https://github.com/TeamNewPipe/NewPipe/issues/2415 - # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out - fmt_stream.reject! { |f| !f["indexRange"]? } - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @adaptive_fmts = fmt_stream - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) - end - - def video_streams - adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video") - end - - def audio_streams - adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") - end - - def storyboards - storyboards = info["storyboards"]? - .try &.as_h - .try &.["playerStoryboardSpecRenderer"]? - .try &.["spec"]? - .try &.as_s.split("|") - - if !storyboards - if storyboard = info["storyboards"]? - .try &.as_h - .try &.["playerLiveStoryboardSpecRenderer"]? - .try &.["spec"]? - .try &.as_s - return [{ - url: storyboard.split("#")[0], - width: 106, - height: 60, - count: -1, - interval: 5000, - storyboard_width: 3, - storyboard_height: 3, - storyboard_count: -1, - }] - end - end - - items = [] of NamedTuple( - url: String, - width: Int32, - height: Int32, - count: Int32, - interval: Int32, - storyboard_width: Int32, - storyboard_height: Int32, - storyboard_count: Int32) - - return items if !storyboards - - url = URI.parse(storyboards.shift) - params = HTTP::Params.parse(url.query || "") - - storyboards.each_with_index do |storyboard, i| - width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#") - params["sigh"] = sigh - url.query = params.to_s - - width = width.to_i - height = height.to_i - count = count.to_i - interval = interval.to_i - storyboard_width = storyboard_width.to_i - storyboard_height = storyboard_height.to_i - storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i - - items << { - url: url.to_s.sub("$L", i).sub("$N", "M$M"), - width: width, - height: height, - count: count, - interval: interval, - storyboard_width: storyboard_width, - storyboard_height: storyboard_height, - storyboard_count: storyboard_count, - } - end - - items - end - - def paid - reason = info["playabilityStatus"]?.try &.["reason"]? - paid = reason == "This video requires payment to watch." ? true : false - paid - end - - def premium - keywords.includes? "YouTube Red" - end - - def captions : Array(Caption) - return @captions.as(Array(Caption)) if @captions - captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| - name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - language_code = caption["languageCode"].to_s - base_url = caption["baseUrl"].to_s - - caption = Caption.new(name.to_s, language_code, base_url) - caption.name = caption.name.split(" - ")[0] - caption - end - captions ||= [] of Caption - @captions = captions - return @captions.as(Array(Caption)) - end - - def description - description = info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["description"]?.try &.["simpleText"]?.try &.as_s || "" - end - - # TODO - def description=(value : String) - @description = value - end - - def description_html - info["descriptionHtml"]?.try &.as_s || "

" - end - - def description_html=(value : String) - info["descriptionHtml"] = JSON::Any.new(value) - end - - def short_description - info["shortDescription"]?.try &.as_s? || "" - end - - def hls_manifest_url : String? - info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s - end - - def dash_manifest_url - info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s - end - - def genre : String - info["genre"]?.try &.as_s || "" - end - - def genre_url : String? - info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil - end - - def license : String? - info["license"]?.try &.as_s - end - - def is_family_friendly : Bool - info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false - end - - def is_vr : Bool? - projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s - return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type - end - - def projection_type : String? - return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s - end - - def wilson_score : Float64 - ci_lower_bound(likes, likes + dislikes).round(4) - end - - def engagement : Float64 - (((likes + dislikes) / views) * 100).round(4) - end - - def reason : String? - info["reason"]?.try &.as_s - end -end - -struct Caption - property name - property language_code - property base_url - - getter name : String - getter language_code : String - getter base_url : String - - setter name - - def initialize(@name, @language_code, @base_url) - end -end - class VideoRedirect < Exception property video_id : String @@ -942,7 +365,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ end def get_video(id, db, refresh = true, region = nil, force_refresh = false) - if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region + if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: YouTubeStructs::Video)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -967,6 +390,8 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) return video end +# TODO make private. All instances of fetching video should be done from get_video() to +# allow for caching. def fetch_video(id, region) info = extract_video_info(video_id: id) @@ -993,7 +418,7 @@ def fetch_video(id, region) raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? - video = Video.new({ + video = YouTubeStructs::Video.new({ id: id, info: info, updated: Time.utc, @@ -1116,7 +541,7 @@ def process_video_params(query, preferences) controls ||= 1 controls = controls >= 1 - params = VideoPreferences.new({ + params = InvidiousStructs::VideoPreferences.new({ annotations: annotations, autoplay: autoplay, comments: comments, diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 5788bf51..62f29776 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -1,7 +1,7 @@
<% case item when %> - <% when SearchChannel %> + <% when YouTubeStructs::ChannelRenderer %> <% if !env.get("preferences").as(Preferences).thin_mode %>
@@ -13,7 +13,7 @@

<%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %>

<% if !item.auto_generated %>

<%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %>

<% end %>
<%= item.description_html %>
- <% when SearchPlaylist, InvidiousPlaylist %> + <% when YouTubeStructs::PlaylistRenderer, InvidiousStructs::Playlist %> <% if item.id.starts_with? "RD" %> <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %> <% else %> @@ -47,7 +47,7 @@

<%= HTML.escape(item.author) %>

 - <% when PlaylistVideo %> + <% when YouTubeStructs::PlaylistVideo %> <% if !env.get("preferences").as(Preferences).thin_mode %>
@@ -109,7 +109,7 @@
<% end %>
- <% when Category %> + <% when YouTubeStructs::Category %> <% else %> <% if !env.get("preferences").as(Preferences).thin_mode %> diff --git a/src/invidious/views/edit_playlist.ecr b/src/invidious/views/edit_playlist.ecr index 5046abc1..adf121bc 100644 --- a/src/invidious/views/edit_playlist.ecr +++ b/src/invidious/views/edit_playlist.ecr @@ -44,7 +44,7 @@ -<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> +<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>

diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 12f93a72..d68b3e4e 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -9,7 +9,7 @@

<%= title %>

- <% if playlist.is_a? InvidiousPlaylist %> + <% if playlist.is_a? InvidiousStructs::Playlist %> <% if playlist.author == user.try &.email %> <%= author %> | @@ -18,7 +18,7 @@ <% end %> <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> | - <% case playlist.as(InvidiousPlaylist).privacy when %> + <% case playlist.as(InvidiousStructs::Playlist).privacy when %> <% when PlaylistPrivacy::Public %> <%= translate(locale, "Public") %> <% when PlaylistPrivacy::Unlisted %> @@ -35,7 +35,7 @@ <% end %> - <% if !playlist.is_a? InvidiousPlaylist %> + <% if !playlist.is_a? InvidiousStructs::Playlist %>
<%= translate(locale, "View playlist on YouTube") %> @@ -50,7 +50,7 @@

- <% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> + <% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<% else %> @@ -72,7 +72,7 @@

-<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> +<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>

@@ -84,7 +84,7 @@

-<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> +<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>