Add support for premieres to search and feed

This commit is contained in:
Omar Roth 2019-03-22 12:24:47 -05:00
parent 88b70973cc
commit fd26f9f34e
10 changed files with 90 additions and 40 deletions

View File

@ -0,0 +1,3 @@
#!/bin/sh
psql invidious -c "ALTER TABLE channel_videos ADD COLUMN premiere_timestamp timestamptz;"

View File

@ -11,6 +11,8 @@ CREATE TABLE public.channel_videos
ucid text, ucid text,
author text, author text,
length_seconds integer, length_seconds integer,
live_now boolean,
premiere_timestamp timestamp with time zone,
CONSTRAINT channel_videos_id_key UNIQUE (id) CONSTRAINT channel_videos_id_key UNIQUE (id)
); );

View File

@ -2104,7 +2104,8 @@ get "/feed/channel/:ucid" do |env|
length_seconds: 0, length_seconds: 0,
live_now: false, live_now: false,
paid: false, paid: false,
premium: false premium: false,
premiere_timestamp: nil
) )
end end
@ -2372,7 +2373,7 @@ post "/feed/webhook/:token" do |env|
updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
video = get_video(id, PG_DB, proxies, region: nil) video = get_video(id, PG_DB, proxies, region: nil)
video = ChannelVideo.new(id, video.title, published, updated, video.ucid, video.author, video.length_seconds, video.live_now) video = ChannelVideo.new(id, video.title, published, updated, video.ucid, video.author, video.length_seconds, video.live_now, video.premiere_timestamp)
PG_DB.exec("UPDATE users SET notifications = notifications || $1 \ PG_DB.exec("UPDATE users SET notifications = notifications || $1 \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, video.ucid) WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, video.ucid)
@ -2382,7 +2383,8 @@ post "/feed/webhook/:token" do |env|
PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \ PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array) updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, premiere_timestamp = $9", video_array)
end end
end end
@ -2901,8 +2903,8 @@ get "/api/v1/videos/:id" do |env|
json.field "liveNow", video.live_now json.field "liveNow", video.live_now
json.field "isUpcoming", video.is_upcoming json.field "isUpcoming", video.is_upcoming
if video.is_upcoming if video.premiere_timestamp
json.field "premiereTimestamp", video.premiere_timestamp json.field "premiereTimestamp", video.premiere_timestamp.not_nil!.to_unix
end end
if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]? if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]?

View File

@ -10,14 +10,15 @@ end
class ChannelVideo class ChannelVideo
add_mapping({ add_mapping({
id: String, id: String,
title: String, title: String,
published: Time, published: Time,
updated: Time, updated: Time,
ucid: String, ucid: String,
author: String, author: String,
length_seconds: {type: Int32, default: 0}, length_seconds: {type: Int32, default: 0},
live_now: {type: Bool, default: false}, live_now: {type: Bool, default: false},
premiere_timestamp: {type: Time?, default: nil},
}) })
end end
@ -126,7 +127,19 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
live_now = channel_video.try &.live_now live_now = channel_video.try &.live_now
live_now ||= false live_now ||= false
video = ChannelVideo.new(video_id, title, published, Time.now, ucid, author, length_seconds, live_now) premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new(
video_id,
title,
published,
Time.now,
ucid,
author,
length_seconds,
live_now,
premiere_timestamp
)
db.exec("UPDATE users SET notifications = notifications || $1 \ db.exec("UPDATE users SET notifications = notifications || $1 \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid) WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
@ -134,9 +147,12 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
video_array = video.to_a video_array = video.to_a
args = arg_array(video_array) args = arg_array(video_array)
# We don't include the 'premire_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
db.exec("INSERT INTO channel_videos VALUES (#{args}) \ db.exec("INSERT INTO channel_videos VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array) updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8", video_array)
end end
else else
page = 1 page = 1
@ -163,7 +179,17 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
end end
count = nodeset.size count = nodeset.size
videos = videos.map { |video| ChannelVideo.new(video.id, video.title, video.published, Time.now, video.ucid, video.author, video.length_seconds, video.live_now) } videos = videos.map { |video| ChannelVideo.new(
video.id,
video.title,
video.published,
Time.now,
video.ucid,
video.author,
video.length_seconds,
video.live_now,
video.premiere_timestamp
) }
videos.each do |video| videos.each do |video|
ids << video.id ids << video.id
@ -176,8 +202,12 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
video_array = video.to_a video_array = video.to_a
args = arg_array(video_array) args = arg_array(video_array)
db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO UPDATE SET title = $2, \ # We don't include the 'premire_timestamp' here because channel pages don't include them,
published = $3, updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array) # meaning the above timestamp is always null
db.exec("INSERT INTO channel_videos VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8", video_array)
end end
end end

View File

@ -325,6 +325,11 @@ def extract_items(nodeset, ucid = nil, author_name = nil)
paid = true paid = true
end end
premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64
if premiere_timestamp
premiere_timestamp = Time.unix(premiere_timestamp)
end
items << SearchVideo.new( items << SearchVideo.new(
title: title, title: title,
id: id, id: id,
@ -337,7 +342,8 @@ def extract_items(nodeset, ucid = nil, author_name = nil)
length_seconds: length_seconds, length_seconds: length_seconds,
live_now: live_now, live_now: live_now,
paid: paid, paid: paid,
premium: premium premium: premium,
premiere_timestamp: premiere_timestamp
) )
end end
end end

View File

@ -1,17 +1,18 @@
class SearchVideo class SearchVideo
add_mapping({ add_mapping({
title: String, title: String,
id: String, id: String,
author: String, author: String,
ucid: String, ucid: String,
published: Time, published: Time,
views: Int64, views: Int64,
description: String, description: String,
description_html: String, description_html: String,
length_seconds: Int32, length_seconds: Int32,
live_now: Bool, live_now: Bool,
paid: Bool, paid: Bool,
premium: Bool, premium: Bool,
premiere_timestamp: Time?,
}) })
end end

View File

@ -298,11 +298,13 @@ class Video
.try &.["offlineSlate"]? .try &.["offlineSlate"]?
.try &.["liveStreamOfflineSlateRenderer"]? .try &.["liveStreamOfflineSlateRenderer"]?
.try &.["scheduledStartTime"].as_s.to_i64 .try &.["scheduledStartTime"].as_s.to_i64
return premiere_timestamp
else
return nil
end end
if premiere_timestamp
premiere_timestamp = Time.unix(premiere_timestamp)
end
return premiere_timestamp
end end
def keywords def keywords

View File

@ -68,7 +68,9 @@
<b><a style="width:100%;" href="/channel/<%= item.ucid %>"><%= item.author %></a></b> <b><a style="width:100%;" href="/channel/<%= item.ucid %>"><%= item.author %></a></b>
</p> </p>
<% if Time.now - item.published > 1.minute %> <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %>
<h5><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %></h5>
<% elsif Time.now - item.published > 1.minute %>
<h5><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></h5> <h5><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></h5>
<% end %> <% end %>
<% else %> <% else %>
@ -103,7 +105,9 @@
<b><a style="width:100%;" href="/channel/<%= item.ucid %>"><%= item.author %></a></b> <b><a style="width:100%;" href="/channel/<%= item.ucid %>"><%= item.author %></a></b>
</p> </p>
<% if Time.now - item.published > 1.minute %> <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %>
<h5><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %></h5>
<% elsif Time.now - item.published > 1.minute %>
<h5><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></h5> <h5><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></h5>
<% end %> <% end %>
<% end %> <% end %>