iv-org/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-08-11 15:50:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' into patch-1

Browse source
This commit is contained in:
Perflyst 2020-11-12 17:06:38 +01:00 • committed by GitHub
commit bb7d8735cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
123 changed files with 5663 additions and 4379 deletions
Download patch file Download diff file Expand all files Collapse all files

1264
src/invidious.cr
View file

File diff suppressed because it is too large Load diff

622
src/invidious/channels.cr
View file

@ -1,22 +1,35 @@
struct InvidiousChannel
db_mapping({
id: String,
author: String,
updated: Time,
deleted: Bool,
subscribed: Time?,
})
include DB::Serializable
property id : String
property author : String
property updated : Time
property deleted : Bool
property subscribed : Time?
end
struct ChannelVideo
def to_json(locale, config, kemal_config, json : JSON::Builder)
include DB::Serializable
property id : String
property title : String
property published : Time
property updated : Time
property ucid : String
property author : String
property length_seconds : Int32 = 0
property live_now : Bool = false
property premiere_timestamp : Time? = nil
property views : Int64? = nil
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
generate_thumbnails(json, self.id, config, Kemal.config)
generate_thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
@ -31,17 +44,17 @@ struct ChannelVideo
end
end
def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, config, kemal_config, json)
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, config, kemal_config, json)
to_json(locale, json)
end
end
end
def to_xml(locale, host_url, query_params, xml : XML::Builder)
def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
@ -49,17 +62,17 @@ struct ChannelVideo
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}")
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{host_url}/watch?#{query_params}") do
xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
@ -69,64 +82,59 @@ struct ChannelVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(locale, config, kemal_config, xml : XML::Builder | Nil = nil)
def to_xml(locale, xml : XML::Builder | Nil = nil)
if xml
to_xml(locale, config, kemal_config, xml)
to_xml(locale, xml)
else
XML.build do |xml|
to_xml(locale, config, kemal_config, xml)
to_xml(locale, xml)
end
end
end
db_mapping({
id: String,
title: String,
published: Time,
updated: Time,
ucid: String,
author: String,
length_seconds: {type: Int32, default: 0},
live_now: {type: Bool, default: false},
premiere_timestamp: {type: Time?, default: nil},
views: {type: Int64?, default: nil},
})
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| var.name }}}
}
{% end %}
end
end
struct AboutRelatedChannel
db_mapping({
ucid: String,
author: String,
author_url: String,
author_thumbnail: String,
})
include DB::Serializable
property ucid : String
property author : String
property author_url : String
property author_thumbnail : String
end
# TODO: Refactor into either SearchChannel or InvidiousChannel
struct AboutChannel
db_mapping({
ucid: String,
author: String,
auto_generated: Bool,
author_url: String,
author_thumbnail: String,
banner: String?,
description_html: String,
paid: Bool,
total_views: Int64,
sub_count: Int32,
joined: Time,
is_family_friendly: Bool,
allowed_regions: Array(String),
related_channels: Array(AboutRelatedChannel),
tabs: Array(String),
})
include DB::Serializable
property ucid : String
property author : String
property auto_generated : Bool
property author_url : String
property author_thumbnail : String
property banner : String?
property description_html : String
property paid : Bool
property total_views : Int64
property sub_count : Int32
property joined : Time
property is_family_friendly : Bool
property allowed_regions : Array(String)
property related_channels : Array(AboutRelatedChannel)
property tabs : Array(String)
end
class ChannelRedirect < Exception
@ -213,33 +221,20 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
page = 1
url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
response = YT_POOL.client &.get(url)
response = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = [] of SearchVideo
begin
json = JSON.parse(response.body)
initial_data = JSON.parse(response.body).as_a.find &.["response"]?
raise "Could not extract JSON" if !initial_data
videos = extract_videos(initial_data.as_h, author, ucid)
rescue ex
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
response.body.includes?("https://www.google.com/sorry/index")
raise "Could not extract channel info. Instance is likely blocked."
end
raise "Could not extract JSON"
end
if json["content_html"]? && !json["content_html"].as_s.empty?
document = XML.parse_html(json["content_html"].as_s)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
if auto_generated
videos = extract_videos(nodeset)
else
videos = extract_videos(nodeset, ucid, author)
end
end
videos ||= [] of ChannelVideo
rss.xpath_nodes("//feed/entry").each do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content
@ -260,41 +255,28 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new(
id: video_id,
title: title,
published: published,
updated: Time.utc,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: Time.utc,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
)
emails = db.query_all("UPDATE users SET notifications = notifications || $1 \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
video.id, video.published, ucid, as: String)
video_array = video.to_a
args = arg_array(video_array)
views: views,
})
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
db.exec("INSERT INTO channel_videos VALUES (#{args}) \
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10", args: video_array)
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
# Update all users affected by insert
if emails.empty?
values = "'{}'"
else
values = "VALUES #{emails.map { |email| %((E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}')) }.join(",")}"
end
db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})")
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
end
if pull_all_videos
@ -303,38 +285,24 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
ids = [] of String
loop do
url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
response = YT_POOL.client &.get(url)
json = JSON.parse(response.body)
response = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
initial_data = JSON.parse(response.body).as_a.find &.["response"]?
raise "Could not extract JSON" if !initial_data
videos = extract_videos(initial_data.as_h, author, ucid)
if json["content_html"]? && !json["content_html"].as_s.empty?
document = XML.parse_html(json["content_html"].as_s)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
else
break
end
nodeset = nodeset.not_nil!
if auto_generated
videos = extract_videos(nodeset)
else
videos = extract_videos(nodeset, ucid, author)
end
count = nodeset.size
videos = videos.map { |video| ChannelVideo.new(
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
count = videos.size
videos = videos.map { |video| ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views
) }
views: video.views,
}) }
videos.each do |video|
ids << video.id
@ -342,42 +310,28 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
emails = db.query_all("UPDATE users SET notifications = notifications || $1 \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email",
video.id, video.published, video.ucid, as: String)
video_array = video.to_a
args = arg_array(video_array)
# We don't update the 'premire_timestamp' here because channel pages don't include them
db.exec("INSERT INTO channel_videos VALUES (#{args}) \
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10", args: video_array)
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
# Update all users affected by insert
if emails.empty?
values = "'{}'"
else
values = "VALUES #{emails.map { |email| %((E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}')) }.join(",")}"
end
db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})")
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
end
end
if count < 25
break
end
break if count < 25
page += 1
end
# When a video is deleted from a channel, we find and remove it here
db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
end
channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil)
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
return channel
end
@ -387,23 +341,11 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated)
response = YT_POOL.client &.get(url)
json = JSON.parse(response.body)
if json["load_more_widget_html"].as_s.empty?
continuation = nil
else
continuation = XML.parse_html(json["load_more_widget_html"].as_s)
continuation = continuation.xpath_node(%q(//button[@data-uix-load-more-href]))
if continuation
continuation = extract_channel_playlists_cursor(continuation["data-uix-load-more-href"], auto_generated)
end
end
html = XML.parse_html(json["content_html"].as_s)
nodeset = html.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]?
initial_data = JSON.parse(response.body).as_a.find(&.["response"]?).try &.as_h
else
url = "/channel/#{ucid}/playlists?disable_polymer=1&flow=list&view=1"
url = "/channel/#{ucid}/playlists?flow=list&view=1"
case sort_by
when "last", "last_added"
@ -412,55 +354,58 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
url += "&sort=da"
when "newest", "newest_created"
url += "&sort=dd"
else nil # Ignore
end
response = YT_POOL.client &.get(url)
html = XML.parse_html(response.body)
continuation = html.xpath_node(%q(//button[@data-uix-load-more-href]))
if continuation
continuation = extract_channel_playlists_cursor(continuation["data-uix-load-more-href"], auto_generated)
end
nodeset = html.xpath_nodes(%q(//ul[@id="browse-items-primary"]/li[contains(@class, "feed-item-container")]))
continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]?
initial_data = extract_initial_data(response.body)
end
if auto_generated
items = extract_shelf_items(nodeset, ucid, author)
else
items = extract_items(nodeset, ucid, author)
end
return [] of SearchItem, nil if !initial_data
items = extract_items(initial_data)
continuation = extract_channel_playlists_cursor(continuation, auto_generated) if continuation
return items, continuation
end
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest")
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "videos",
"6:varint": 2_i64,
"7:varint": 1_i64,
"12:varint": 1_i64,
"13:string": "",
"23:varint": 0_i64,
"2:string" => "videos",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
if auto_generated
seed = Time.unix(1525757349)
until seed >= Time.utc
seed += 1.month
end
timestamp = seed - (page - 1).months
if !v2
if auto_generated
seed = Time.unix(1525757349)
until seed >= Time.utc
seed += 1.month
end
timestamp = seed - (page - 1).months
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}"
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}"
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}"
end
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}"
object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({
"1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({
"1:varint" => 30_i64 * (page - 1),
}))),
})))
end
case sort_by
@ -469,6 +414,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64
when "oldest"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64
else nil # Ignore
end
object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
@ -487,12 +433,12 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "playlists",
"6:varint": 2_i64,
"7:varint": 1_i64,
"12:varint": 1_i64,
"13:string": "",
"23:varint": 0_i64,
"2:string" => "playlists",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
@ -513,6 +459,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
when "last", "last_added"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
else nil # Ignore
end
end
@ -527,9 +474,8 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
def extract_channel_playlists_cursor(url, auto_generated)
cursor = URI.parse(url).query_params
.try { |i| URI.decode_www_form(i["continuation"]) }
def extract_channel_playlists_cursor(cursor, auto_generated)
cursor = URI.decode_www_form(cursor)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
@ -554,13 +500,13 @@ def extract_channel_playlists_cursor(url, auto_generated)
end
# TODO: Add "sort_by"
def fetch_channel_community(ucid, continuation, locale, config, kemal_config, format, thin_mode)
def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
if response.status_code == 404
if response.status_code != 200
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en")
end
if response.status_code == 404
if response.status_code != 200
error_message = translate(locale, "This channel does not exist.")
raise error_message
end
@ -581,16 +527,8 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
headers = HTTP::Headers.new
headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
headers["content-type"] = "application/x-www-form-urlencoded"
headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ=="
headers["x-spf-previous"] = ""
headers["x-spf-referer"] = ""
headers["x-youtube-client-name"] = "1"
headers["x-youtube-client-version"] = "2.20180719"
session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[A-Za-z0-9\_\-\=]+)"/).try &.["session_token"]? || ""
session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || ""
post_req = {
session_token: session_token,
}
@ -628,17 +566,9 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
if !post
next
end
if !post["contentText"]?
content_html = ""
else
content_html = post["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff').try { |block| HTML.escape(block) }.to_s ||
content_to_comment_html(post["contentText"]["runs"].as_a).try &.to_s || ""
end
next if !post
content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]?.try &.["simpleText"]? || ""
json.object do
@ -707,7 +637,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
json.field "title", attachment["title"]["simpleText"].as_s
json.field "videoId", video_id
json.field "videoThumbnails" do
generate_thumbnails(json, video_id, config, kemal_config)
generate_thumbnails(json, video_id)
end
json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s)
@ -845,16 +775,34 @@ def extract_channel_community_cursor(continuation)
cursor
end
INITDATA_PREQUERY = "window[\"ytInitialData\"] = {"
def get_about_info(ucid, locale)
about = YT_POOL.client &.get("/channel/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
if about.status_code == 404
about = YT_POOL.client &.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
about = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en")
if about.status_code != 200
about = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en")
end
if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/)
raise ChannelRedirect.new(channel_id: md["ucid"])
end
if about.status_code != 200
error_message = translate(locale, "This channel does not exist.")
raise error_message
end
initdata_pre = about.body.index(INITDATA_PREQUERY)
initdata_post = initdata_pre.nil? ? nil : about.body.index("};", initdata_pre)
if initdata_post.nil?
about = XML.parse_html(about.body)
error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip
error_message ||= translate(locale, "Could not get channel info.")
raise error_message
end
initdata_pre = initdata_pre.not_nil! + INITDATA_PREQUERY.size - 1
initdata = JSON.parse(about.body[initdata_pre, initdata_post - initdata_pre + 1])
about = XML.parse_html(about.body)
if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")]))
@ -862,136 +810,138 @@ def get_about_info(ucid, locale)
raise error_message
end
if about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).try &.content.empty?
error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip
error_message ||= translate(locale, "Could not get channel info.")
raise error_message
end
author = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!.content
author_url = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!["href"]
author_thumbnail = about.xpath_node(%q(//img[@class="channel-header-profile-image"])).not_nil!["src"]
author = about.xpath_node(%q(//meta[@name="title"])).not_nil!["content"]
author_url = about.xpath_node(%q(//link[@rel="canonical"])).not_nil!["href"]
author_thumbnail = about.xpath_node(%q(//link[@rel="image_src"])).not_nil!["href"]
ucid = about.xpath_node(%q(//meta[@itemprop="channelId"])).not_nil!["content"]
banner = about.xpath_node(%q(//div[@id="gh-banner"]/style)).not_nil!.content
banner = "https:" + banner.match(/background-image: url\((?<url>[^)]+)\)/).not_nil!["url"]
# Raises a KeyError on failure.
banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banner = banners.try &.[-1]?.try &.["url"].as_s?
if banner.includes? "channels/c4/default_banner"
banner = nil
end
# if banner.includes? "channels/c4/default_banner"
# banner = nil
# end
description_html = about.xpath_node(%q(//div[contains(@class,"about-description")])).try &.to_s ||
%(<div class="about-description branded-page-box-padding"><pre></pre></div>)
description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || ""
description_html = HTML.escape(description).gsub("\n", "<br>")
paid = about.xpath_node(%q(//meta[@itemprop="paid"])).not_nil!["content"] == "True"
is_family_friendly = about.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True"
allowed_regions = about.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",")
related_channels = about.xpath_nodes(%q(//div[contains(@class, "branded-page-related-channels")]/ul/li))
related_channels = related_channels.map do |node|
related_id = node["data-external-id"]?
related_id ||= ""
related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"]
.["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]?
.try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node|
renderer = node["miniChannelRenderer"]?
related_id = renderer.try &.["channelId"]?.try &.as_s?
related_id ||= ""
anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
related_title = anchor.try &.["title"]
related_title ||= ""
related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s?
related_title ||= ""
related_author_url = anchor.try &.["href"]
related_author_url ||= ""
related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]?
.try &.["url"]?.try &.as_s?
related_author_url ||= ""
related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"]
related_author_thumbnail ||= ""
related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a?
related_author_thumbnails ||= [] of JSON::Any
AboutRelatedChannel.new(
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
)
end
related_author_thumbnail = ""
if related_author_thumbnails.size > 0
related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s?
related_author_thumbnail ||= ""
end
joined = about.xpath_node(%q(//span[contains(., "Joined")]))
.try &.content.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
AboutRelatedChannel.new({
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
})
end
related_channels ||= [] of AboutRelatedChannel
total_views = about.xpath_node(%q(//span[contains(., "views")]/b))
.try &.content.try &.gsub(/\D/, "").to_i64? || 0_i64
sub_count = about.xpath_node(%q(.//span[contains(@class, "subscriber-count")]))
.try &.["title"].try { |text| short_text_to_number(text) } || 0
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
total_views = 0_i64
joined = Time.unix(0)
tabs = [] of String
auto_generated = false
if about.xpath_node(%q(//ul[@class="about-custom-links"]/li/a[@title="Auto-generated by YouTube"])) ||
about.xpath_node(%q(//span[@class="qualified-channel-title-badge"]/span[@title="Auto-generated by YouTube"]))
auto_generated = true
tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
if !tabs_json.nil?
# Retrieve information from the tabs array. The index we are looking for varies between channels.
tabs_json.each do |node|
# Try to find the about section which is located in only one of the tabs.
channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
.try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
.try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
if !channel_about_meta.nil?
total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s }
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
(channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
auto_generated = true
end
end
end
tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase }
end
tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase }
sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]) } || 0
AboutChannel.new(
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description_html: description_html,
paid: paid,
total_views: total_views,
sub_count: sub_count,
joined: joined,
AboutChannel.new({
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description_html: description_html,
paid: paid,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
related_channels: related_channels,
tabs: tabs
)
allowed_regions: allowed_regions,
related_channels: related_channels,
tabs: tabs,
})
end
def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true)
return YT_POOL.client &.get(url)
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
count = 0
videos = [] of SearchVideo
2.times do |i|
url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
response = YT_POOL.client &.get(url)
json = JSON.parse(response.body)
if json["content_html"]? && !json["content_html"].as_s.empty?
document = XML.parse_html(json["content_html"].as_s)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
if !json["load_more_widget_html"]?.try &.as_s.empty?
count += 30
end
if auto_generated
videos += extract_videos(nodeset)
else
videos += extract_videos(nodeset, ucid, author)
end
else
break
end
response = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
initial_data = JSON.parse(response.body).as_a.find &.["response"]?
break if !initial_data
videos.concat extract_videos(initial_data.as_h, author, ucid)
end
return videos, count
return videos.size, videos
end
def get_latest_videos(ucid)
videos = [] of SearchVideo
response = get_channel_videos_response(ucid, 1)
initial_data = JSON.parse(response.body).as_a.find &.["response"]?
return [] of SearchVideo if !initial_data
author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
items = extract_videos(initial_data.as_h, author, ucid)
url = produce_channel_videos_url(ucid, 0)
response = YT_POOL.client &.get(url)
json = JSON.parse(response.body)
if json["content_html"]? && !json["content_html"].as_s.empty?
document = XML.parse_html(json["content_html"].as_s)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
videos = extract_videos(nodeset, ucid)
end
return videos
return items
end

106
src/invidious/comments.cr
View file

@ -1,11 +1,23 @@
class RedditThing
JSON.mapping({
kind: String,
data: RedditComment | RedditLink | RedditMore | RedditListing,
})
include JSON::Serializable
property kind : String
property data : RedditComment | RedditLink | RedditMore | RedditListing
end
class RedditComment
include JSON::Serializable
property author : String
property body_html : String
property replies : RedditThing | String
property score : Int32
property depth : Int32
property permalink : String
@[JSON::Field(converter: RedditComment::TimeConverter)]
property created_utc : Time
module TimeConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix(value.read_float.to_i)
@ -15,51 +27,38 @@ class RedditComment
json.number(value.to_unix)
end
end
JSON.mapping({
author: String,
body_html: String,
replies: RedditThing | String,
score: Int32,
depth: Int32,
permalink: String,
created_utc: {
type: Time,
converter: RedditComment::TimeConverter,
},
})
end
struct RedditLink
JSON.mapping({
author: String,
score: Int32,
subreddit: String,
num_comments: Int32,
id: String,
permalink: String,
title: String,
})
include JSON::Serializable
property author : String
property score : Int32
property subreddit : String
property num_comments : Int32
property id : String
property permalink : String
property title : String
end
struct RedditMore
JSON.mapping({
children: Array(String),
count: Int32,
depth: Int32,
})
include JSON::Serializable
property children : Array(String)
property count : Int32
property depth : Int32
end
class RedditListing
JSON.mapping({
children: Array(RedditThing),
modhash: String,
})
include JSON::Serializable
property children : Array(RedditThing)
property modhash : String
end
def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top")
video = get_video(id, db, region: region)
session_token = video.info["session_token"]?
session_token = video.session_token
case cursor
when nil, ""
@ -85,17 +84,9 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
session_token: session_token,
}
headers = HTTP::Headers.new
headers["content-type"] = "application/x-www-form-urlencoded"
headers["cookie"] = video.info["cookie"]
headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ=="
headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"
headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"
headers["x-youtube-client-name"] = "1"
headers["x-youtube-client-version"] = "2.20180719"
headers = HTTP::Headers{
"cookie" => video.cookie,
}
response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req))
response = JSON.parse(response.body)
@ -150,8 +141,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
node_comment = node["commentRenderer"]
end
content_html = node_comment["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff').try { |block| HTML.escape(block) }.to_s ||
content_to_comment_html(node_comment["contentText"]["runs"].as_a).try &.to_s || ""
content_html = node_comment["contentText"]?.try { |t| parse_content(t) } || ""
author = node_comment["authorText"]?.try &.["simpleText"]? || ""
json.field "author", author
@ -294,7 +284,7 @@ def template_youtube_comments(comments, locale, thin_mode)
<div class="pure-u-23-24">
<p>
<a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}"
onclick="get_youtube_replies(this)">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a>
data-onclick="get_youtube_replies">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a>
</p>
</div>
</div>
@ -347,7 +337,7 @@ def template_youtube_comments(comments, locale, thin_mode)
END_HTML
else
html << <<-END_HTML
<iframe id='ivplayer' type='text/html' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' frameborder='0'></iframe>
<iframe id='ivplayer' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' style='border:none;'></iframe>
END_HTML
end
@ -356,6 +346,7 @@ def template_youtube_comments(comments, locale, thin_mode)
</div>
</div>
END_HTML
else nil # Ignore
end
end
@ -413,7 +404,7 @@ def template_youtube_comments(comments, locale, thin_mode)
<div class="pure-u-1">
<p>
<a href="javascript:void(0)" data-continuation="#{comments["continuation"]}"
onclick="get_youtube_replies(this, true)">#{translate(locale, "Load more")}</a>
data-onclick="get_youtube_replies" data-load-more>#{translate(locale, "Load more")}</a>
</p>
</div>
</div>
@ -451,7 +442,7 @@ def template_reddit_comments(root, locale)
html << <<-END_HTML
<p>
<a href="javascript:void(0)" onclick="toggle_parent(this)">[ - ]</a>
<a href="javascript:void(0)" data-onclick="toggle_parent">[ - ]</a>
<b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b>
#{translate(locale, "`x` points", number_with_separator(child.score))}
<span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span>
@ -522,6 +513,11 @@ def fill_links(html, scheme, host)
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
def parse_content(content : JSON::Any) : String
content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || ""
end
def content_to_comment_html(content)
comment_html = content.map do |run|
text = HTML.escape(run["text"].as_s)
@ -556,7 +552,7 @@ def content_to_comment_html(content)
video_id = watch_endpoint["videoId"].as_s
if length_seconds
text = %(<a href="javascript:void(0)" onclick="player.currentTime(#{length_seconds})">#{text}</a>)
text = %(<a href="javascript:void(0)" data-onclick="jump_to_time" data-jump-time="#{length_seconds}">#{text}</a>)
else
text = %(<a href="/watch?v=#{video_id}">#{text}</a>)
end
@ -609,6 +605,8 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
when "new", "newest"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
else # top
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
end
continuation = object.try { |i| Protodec::Any.cast_json(object) }

32
src/invidious/helpers/handlers.cr
View file

@ -61,7 +61,7 @@ class Kemal::ExceptionHandler
end
class FilteredCompressHandler < Kemal::Handler
exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/ggpht/*", "/api/v1/auth/notifications"]
exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/sb/*", "/ggpht/*", "/api/v1/auth/notifications"]
exclude ["/api/v1/auth/notifications", "/data_control"], "POST"
def call(env)
@ -74,10 +74,10 @@ class FilteredCompressHandler < Kemal::Handler
if request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
env.response.output = Gzip::Writer.new(env.response.output, sync_close: true)
env.response.output = Compress::Gzip::Writer.new(env.response.output, sync_close: true)
elsif request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
env.response.output = Flate::Writer.new(env.response.output, sync_close: true)
env.response.output = Compress::Deflate::Writer.new(env.response.output, sync_close: true)
end
call_next env
@ -212,29 +212,3 @@ class DenyFrame < Kemal::Handler
call_next env
end
end
# Temp fixes for https://github.com/crystal-lang/crystal/issues/7383
class HTTP::UnknownLengthContent
def read_byte
ensure_send_continue
if @io.is_a?(OpenSSL::SSL::Socket::Client)
return if @io.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
end
@io.read_byte
end
end
class HTTP::Client
private def handle_response(response)
if @socket.is_a?(OpenSSL::SSL::Socket::Client) && @host.ends_with?("googlevideo.com")
close unless response.keep_alive? || @socket.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
if @socket.as(OpenSSL::SSL::Socket::Client).@in_buffer_rem.empty?
@socket = nil
end
else
close unless response.keep_alive?
end
response
end
end

860
src/invidious/helpers/helpers.cr
View file

@ -1,217 +1,100 @@
require "./macros"
struct Nonce
db_mapping({
nonce: String,
expire: Time,
})
include DB::Serializable
property nonce : String
property expire : Time
end
struct SessionId
db_mapping({
id: String,
email: String,
issued: String,
})
include DB::Serializable
property id : String
property email : String
property issued : String
end
struct Annotation
db_mapping({
id: String,
annotations: String,
})
include DB::Serializable
property id : String
property annotations : String
end
struct ConfigPreferences
module StringToArray
def self.to_json(value : Array(String), json : JSON::Builder)
json.array do
value.each do |element|
json.string element
end
end
end
include YAML::Serializable
def self.from_json(value : JSON::PullParser) : Array(String)
begin
result = [] of String
value.read_array do
result << HTML.escape(value.read_string[0, 100])
end
rescue ex
result = [HTML.escape(value.read_string[0, 100]), ""]
end
property annotations : Bool = false
property annotations_subscribed : Bool = false
property autoplay : Bool = false
property captions : Array(String) = ["", "", ""]
property comments : Array(String) = ["youtube", ""]
property continue : Bool = false
property continue_autoplay : Bool = true
property dark_mode : String = ""
property latest_only : Bool = false
property listen : Bool = false
property local : Bool = false
property locale : String = "en-US"
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = "invidious"
property quality : String = "hd720"
property default_home : String = "Popular"
property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
property related_videos : Bool = true
property sort : String = "published"
property speed : Float32 = 1.0_f32
property thin_mode : Bool = false
property unseen_only : Bool = false
property video_loop : Bool = false
property volume : Int32 = 100
result
end
def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
yaml.sequence do
value.each do |element|
yaml.scalar element
end
end
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
begin
unless node.is_a?(YAML::Nodes::Sequence)
node.raise "Expected sequence, not #{node.class}"
end
result = [] of String
node.nodes.each do |item|
unless item.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{item.class}"
end
result << HTML.escape(item.value[0, 100])
end
rescue ex
if node.is_a?(YAML::Nodes::Scalar)
result = [HTML.escape(node.value[0, 100]), ""]
else
result = ["", ""]
end
end
result
end
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
}
{% end %}
end
module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
begin
result = value.read_string
if result.empty?
CONFIG.default_user_preferences.dark_mode
else
result
end
rescue ex
if value.read_bool
"dark"
else
"light"
end
end
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
yaml.scalar value
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
case node.value
when "true"
"dark"
when "false"
"light"
when ""
CONFIG.default_user_preferences.dark_mode
else
node.value
end
end
end
yaml_mapping({
annotations: {type: Bool, default: false},
annotations_subscribed: {type: Bool, default: false},
autoplay: {type: Bool, default: false},
captions: {type: Array(String), default: ["", "", ""], converter: StringToArray},
comments: {type: Array(String), default: ["youtube", ""], converter: StringToArray},
continue: {type: Bool, default: false},
continue_autoplay: {type: Bool, default: true},
dark_mode: {type: String, default: "", converter: BoolToString},
latest_only: {type: Bool, default: false},
listen: {type: Bool, default: false},
local: {type: Bool, default: false},
locale: {type: String, default: "en-US"},
max_results: {type: Int32, default: 40},
notifications_only: {type: Bool, default: false},
player_style: {type: String, default: "invidious"},
quality: {type: String, default: "hd720"},
default_home: {type: String, default: "Popular"},
feed_menu: {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]},
related_videos: {type: Bool, default: true},
sort: {type: String, default: "published"},
speed: {type: Float32, default: 1.0_f32},
thin_mode: {type: Bool, default: false},
unseen_only: {type: Bool, default: false},
video_loop: {type: Bool, default: false},
volume: {type: Int32, default: 100},
})
end
struct Config
module ConfigPreferencesConverter
def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
value.to_yaml(yaml)
end
include YAML::Serializable
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences
Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple)
end
end
property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
property feed_threads : Int32 # Number of threads to use for updating feeds
property db : DBConfig # Database configuration
property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required
property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
property captcha_enabled : Bool = true
property login_enabled : Bool = true
property registration_enabled : Bool = true
property statistics_enabled : Bool = false
property admins : Array(String) = [] of String
property external_port : Int32? = nil
property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("")
property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs
property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc.
property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc.
property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
module FamilyConverter
def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
case value
when Socket::Family::UNSPEC
yaml.scalar nil
when Socket::Family::INET
yaml.scalar "ipv4"
when Socket::Family::INET6
yaml.scalar "ipv6"
end
end
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument)
property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument)
property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property admin_email : String = "omarroth@protonmail.com" # Email for bug reports
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
if node.is_a?(YAML::Nodes::Scalar)
case node.value.downcase
when "ipv4"
Socket::Family::INET
when "ipv6"
Socket::Family::INET6
else
Socket::Family::UNSPEC
end
else
node.raise "Expected scalar, not #{node.class}"
end
end
end
module StringToCookies
def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
(value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
cookies = HTTP::Cookies.new
node.value.split(";").each do |cookie|
next if cookie.strip.empty?
name, value = cookie.split("=", 2)
cookies << HTTP::Cookie.new(name.strip, value.strip)
end
cookies
end
end
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format
property captcha_key : String? = nil # Key for Anti-Captcha
def disabled?(option)
case disabled = CONFIG.disable_proxy
@ -223,77 +106,20 @@ struct Config
else
return false
end
else
return false
end
end
YAML.mapping({
channel_threads: Int32, # Number of threads to use for crawling videos from channels (for updating subscriptions)
feed_threads: Int32, # Number of threads to use for updating feeds
db: DBConfig, # Database configuration
full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
top_enabled: {type: Bool, default: true},
captcha_enabled: {type: Bool, default: true},
login_enabled: {type: Bool, default: true},
registration_enabled: {type: Bool, default: true},
statistics_enabled: {type: Bool, default: false},
admins: {type: Array(String), default: [] of String},
external_port: {type: Int32?, default: nil},
default_user_preferences: {type: Preferences,
default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple),
converter: ConfigPreferencesConverter,
},
dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs
check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc.
cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc.
hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument)
host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument)
pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports
cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format
captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha
})
end
struct DBConfig
yaml_mapping({
user: String,
password: String,
host: String,
port: Int32,
dbname: String,
})
end
include YAML::Serializable
def rank_videos(db, n)
top = [] of {Float64, String}
db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs|
rs.each do
id = rs.read(String)
wilson_score = rs.read(Float64)
published = rs.read(Time)
# Exponential decay, older videos tend to rank lower
temperature = wilson_score * Math.exp(-0.000005*((Time.utc - published).total_minutes))
top << {temperature, id}
end
end
top.sort!
# Make hottest come first
top.reverse!
top = top.map { |a, b| b }
return top[0..n - 1]
property user : String
property password : String
property host : String
property port : Int32
property dbname : String
end
def login_req(f_req)
@ -334,293 +160,179 @@ def html_to_content(description_html : String)
return description
end
def extract_videos(nodeset, ucid = nil, author_name = nil)
videos = extract_items(nodeset, ucid, author_name)
videos.select { |item| item.is_a?(SearchVideo) }.map { |video| video.as(SearchVideo) }
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
end
def extract_items(nodeset, ucid = nil, author_name = nil)
# TODO: Make this a 'common', so it makes more sense to be used here
def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil)
if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
video_id = i["videoId"].as_s
title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
author_info = i["ownerText"]?.try &.["runs"].as_a[0]?
author = author_info.try &.["text"].as_s || author_fallback || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local
view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } ||
i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]?
.try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0
live_now = false
paid = false
premium = false
premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) }
i["badges"]?.try &.as_a.each do |badge|
b = badge["metadataBadgeRenderer"]
case b["label"].as_s
when "LIVE NOW"
live_now = true
when "New", "4K", "CC"
# TODO
when "Premium"
paid = true
# TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"]
premium = true
else nil # Ignore
end
end
SearchVideo.new({
title: title,
id: video_id,
author: author,
ucid: author_id,
published: published,
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
paid: paid,
premium: premium,
premiere_timestamp: premiere_timestamp,
})
elsif i = item["channelRenderer"]?
author = i["title"]["simpleText"]?.try &.as_s || author_fallback || ""
author_id = i["channelId"]?.try &.as_s || author_id_fallback || ""
author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try { |u| "https:#{u["url"]}" } || ""
subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0
auto_generated = false
auto_generated = true if !i["videoCountText"]?
video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
SearchChannel.new({
author: author,
ucid: author_id,
author_thumbnail: author_thumbnail,
subscriber_count: subscriber_count,
video_count: video_count,
description_html: description_html,
auto_generated: auto_generated,
})
elsif i = item["gridPlaylistRenderer"]?
title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
plid = i["playlistId"]?.try &.as_s || ""
video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
SearchPlaylist.new({
title: title,
id: plid,
author: author_fallback || "",
ucid: author_id_fallback || "",
video_count: video_count,
videos: [] of SearchPlaylistVideo,
thumbnail: playlist_thumbnail,
})
elsif i = item["playlistRenderer"]?
title = i["title"]["simpleText"]?.try &.as_s || ""
plid = i["playlistId"]?.try &.as_s || ""
video_count = i["videoCount"]?.try &.as_s.to_i || 0
playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || ""
author_info = i["shortBylineText"]?.try &.["runs"].as_a[0]?
author = author_info.try &.["text"].as_s || author_fallback || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
videos = i["videos"]?.try &.as_a.map do |v|
v = v["childVideoRenderer"]
v_title = v["title"]["simpleText"]?.try &.as_s || ""
v_id = v["videoId"]?.try &.as_s || ""
v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
SearchPlaylistVideo.new({
title: v_title,
id: v_id,
length_seconds: v_length_seconds,
})
end || [] of SearchPlaylistVideo
# TODO: i["publishedTimeText"]?
SearchPlaylist.new({
title: title,
id: plid,
author: author,
ucid: author_id,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail,
})
elsif i = item["radioRenderer"]? # Mix
# TODO
elsif i = item["showRenderer"]? # Show
# TODO
elsif i = item["shelfRenderer"]?
elsif i = item["horizontalCardListRenderer"]?
elsif i = item["searchPyvRenderer"]? # Ad
end
end
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
items = [] of SearchItem
nodeset.each do |node|
anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
if !anchor
next
end
title = anchor.content.strip
id = anchor["href"]
channel_v2_response = initial_data
.try &.["response"]?
.try &.["continuationContents"]?
.try &.["gridContinuation"]?
.try &.["items"]?
if anchor["href"].starts_with? "https://www.googleadservices.com"
next
end
author_id = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.["href"].split("/")[-1] || ucid || ""
author = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.content.strip || author_name || ""
description_html = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-description")])).try &.to_s || ""
tile = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-tile")]))
if !tile
next
end
case tile["class"]
when .includes? "yt-lockup-playlist"
plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"]
anchor = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-meta")]/a))
if !anchor
anchor = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/a))
end
video_count = node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) ||
node.xpath_node(%q(.//span[@class="formatted-video-count-label"]))
if video_count
video_count = video_count.content
if video_count == "50+"
author = "YouTube"
author_id = "UC-9-kyTW8ZkZNDHQJ6FgpwQ"
end
video_count = video_count.gsub(/\D/, "").to_i?
end
video_count ||= 0
videos = [] of SearchPlaylistVideo
node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video|
anchor = video.xpath_node(%q(.//a))
if anchor
video_title = anchor.content.strip
id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
end
video_title ||= ""
id ||= ""
anchor = video.xpath_node(%q(.//span/span))
if anchor
length_seconds = decode_length_seconds(anchor.content)
end
length_seconds ||= 0
videos << SearchPlaylistVideo.new(
video_title,
id,
length_seconds
)
end
playlist_thumbnail = node.xpath_node(%q(.//span/img)).try &.["data-thumb"]?
playlist_thumbnail ||= node.xpath_node(%q(.//span/img)).try &.["src"]
items << SearchPlaylist.new(
title: title,
id: plid,
author: author,
ucid: author_id,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail
)
when .includes? "yt-lockup-channel"
author = title.strip
ucid = node.xpath_node(%q(.//button[contains(@class, "yt-uix-subscription-button")])).try &.["data-channel-external-id"]?
ucid ||= id.split("/")[-1]
author_thumbnail = node.xpath_node(%q(.//div/span/img)).try &.["data-thumb"]?
author_thumbnail ||= node.xpath_node(%q(.//div/span/img)).try &.["src"]
if author_thumbnail
author_thumbnail = URI.parse(author_thumbnail)
author_thumbnail.scheme = "https"
author_thumbnail = author_thumbnail.to_s
end
author_thumbnail ||= ""
subscriber_count = node.xpath_node(%q(.//span[contains(@class, "subscriber-count")]))
.try &.["title"].try { |text| short_text_to_number(text) } || 0
video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i?
items << SearchChannel.new(
author: author,
ucid: ucid,
author_thumbnail: author_thumbnail,
subscriber_count: subscriber_count,
video_count: video_count || 0,
description_html: description_html,
auto_generated: video_count ? false : true,
)
else
id = id.lchop("/watch?v=")
metadata = node.xpath_node(%q(.//div[contains(@class,"yt-lockup-meta")]/ul))
published = metadata.try &.xpath_node(%q(.//li[contains(text(), " ago")])).try { |node| decode_date(node.content.sub(/^[a-zA-Z]+ /, "")) }
published ||= metadata.try &.xpath_node(%q(.//span[@data-timestamp])).try { |node| Time.unix(node["data-timestamp"].to_i64) }
published ||= Time.utc
view_count = metadata.try &.xpath_node(%q(.//li[contains(text(), " views")])).try &.content.gsub(/\D/, "").to_i64?
view_count ||= 0_i64
length_seconds = node.xpath_node(%q(.//span[@class="video-time"])).try { |node| decode_length_seconds(node.content) }
length_seconds ||= -1
live_now = node.xpath_node(%q(.//span[contains(@class, "yt-badge-live")])) ? true : false
premium = node.xpath_node(%q(.//span[text()="Premium"])) ? true : false
if !premium || node.xpath_node(%q(.//span[contains(text(), "Free episode")]))
paid = false
else
paid = true
end
premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64
if premiere_timestamp
premiere_timestamp = Time.unix(premiere_timestamp)
end
items << SearchVideo.new(
title: title,
id: id,
author: author,
ucid: author_id,
published: published,
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
paid: paid,
premium: premium,
premiere_timestamp: premiere_timestamp
)
end
if channel_v2_response
channel_v2_response.try &.as_a.each { |item|
extract_item(item, author_fallback, author_id_fallback)
.try { |t| items << t }
}
else
initial_data.try { |t| t["contents"]? || t["response"]? }
.try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] ||
t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] ||
t["continuationContents"]? }
.try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? }
.try &.["contents"].as_a
.each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a
.try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a ||
t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t }
.each { |item|
extract_item(item, author_fallback, author_id_fallback)
.try { |t| items << t }
} }
end
return items
end
def extract_shelf_items(nodeset, ucid = nil, author_name = nil)
items = [] of SearchPlaylist
nodeset.each do |shelf|
shelf_anchor = shelf.xpath_node(%q(.//h2[contains(@class, "branded-page-module-title")]))
next if !shelf_anchor
title = shelf_anchor.xpath_node(%q(.//span[contains(@class, "branded-page-module-title-text")])).try &.content.strip
title ||= ""
id = shelf_anchor.xpath_node(%q(.//a)).try &.["href"]
next if !id
shelf_is_playlist = false
videos = [] of SearchPlaylistVideo
shelf.xpath_nodes(%q(.//ul[contains(@class, "yt-uix-shelfslider-list") or contains(@class, "expanded-shelf-content-list")]/li)).each do |child_node|
type = child_node.xpath_node(%q(./div))
if !type
next
end
case type["class"]
when .includes? "yt-lockup-video"
shelf_is_playlist = true
anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
if anchor
video_title = anchor.content.strip
video_id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
end
video_title ||= ""
video_id ||= ""
anchor = child_node.xpath_node(%q(.//span[@class="video-time"]))
if anchor
length_seconds = decode_length_seconds(anchor.content)
end
length_seconds ||= 0
videos << SearchPlaylistVideo.new(
video_title,
video_id,
length_seconds
)
when .includes? "yt-lockup-playlist"
anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
if anchor
playlist_title = anchor.content.strip
params = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)
plid = params["list"]
end
playlist_title ||= ""
plid ||= ""
playlist_thumbnail = child_node.xpath_node(%q(.//span/img)).try &.["data-thumb"]?
playlist_thumbnail ||= child_node.xpath_node(%q(.//span/img)).try &.["src"]
video_count = child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) ||
child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]))
if video_count
video_count = video_count.content.gsub(/\D/, "").to_i?
end
video_count ||= 50
videos = [] of SearchPlaylistVideo
child_node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video|
anchor = video.xpath_node(%q(.//a))
if anchor
video_title = anchor.content.strip
id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"]
end
video_title ||= ""
id ||= ""
anchor = video.xpath_node(%q(.//span/span))
if anchor
length_seconds = decode_length_seconds(anchor.content)
end
length_seconds ||= 0
videos << SearchPlaylistVideo.new(
video_title,
id,
length_seconds
)
end
items << SearchPlaylist.new(
title: playlist_title,
id: plid,
author: author_name,
ucid: ucid,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail
)
end
end
if shelf_is_playlist
plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"]
items << SearchPlaylist.new(
title: title,
id: plid,
author: author_name,
ucid: ucid,
video_count: videos.size,
videos: videos,
thumbnail: "https://i.ytimg.com/vi/#{videos[0].id}/mqdefault.jpg"
)
end
end
return items
items
end
def check_enum(db, logger, enum_name, struct_type = nil)
return # TODO
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
logger.puts("CREATE TYPE #{enum_name}")
@ -642,18 +354,14 @@ def check_table(db, logger, table_name, struct_type = nil)
end
end
if !struct_type
return
end
return if !struct_type
struct_array = struct_type.to_type_tuple
struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map { |line| line.strip }
.try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT")
if !column_types
return
end
return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
@ -704,6 +412,15 @@ def check_table(db, logger, table_name, struct_type = nil)
end
end
end
return if column_array.size <= struct_array.size
column_array.each do |column|
if !struct_array.includes? column
logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
end
end
end
class PG::ResultSet
@ -732,9 +449,7 @@ def cache_annotation(db, id, annotations)
body = XML.parse(annotations)
nodeset = body.xpath_nodes(%q(/document/annotations/annotation))
if nodeset == 0
return
end
return if nodeset == 0
has_legacy_annotations = false
nodeset.each do |node|
@ -744,13 +459,10 @@ def cache_annotation(db, id, annotations)
end
end
if has_legacy_annotations
# TODO: Update on conflict?
db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations)
end
db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations
end
def create_notification_stream(env, config, kemal_config, decrypt_function, topics, connection_channel)
def create_notification_stream(env, topics, connection_channel)
connection = Channel(PQ::Notification).new(8)
connection_channel.send({true, connection})
@ -765,12 +477,12 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
loop do
time_span = [0, 0, 0, 0]
time_span[rand(4)] = rand(30) + 5
published = Time.utc - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3])
published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
video_id = TEST_IDS[rand(TEST_IDS.size)]
video = get_video(video_id, PG_DB)
video.published = published
response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function))
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@ -804,7 +516,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video|
response = JSON.parse(video.to_json(locale, config, Kemal.config))
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@ -846,7 +558,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
video = get_video(video_id, PG_DB)
video.published = Time.unix(published)
response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function))
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
@ -884,26 +596,46 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi
end
end
def extract_initial_data(body)
initial_data = body.match(/window\["ytInitialData"\] = (?<info>.*?);\n/).try &.["info"] || "{}"
def extract_initial_data(body) : Hash(String, JSON::Any)
initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);+\s*\n/).try &.["info"] || "{}"
if initial_data.starts_with?("JSON.parse(\"")
return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s)
return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s).as_h
else
return JSON.parse(initial_data)
return JSON.parse(initial_data).as_h
end
end
def proxy_file(response, env)
if response.headers.includes_word?("Content-Encoding", "gzip")
Gzip::Writer.open(env.response) do |deflate|
response.pipe(deflate)
Compress::Gzip::Writer.open(env.response) do |deflate|
IO.copy response.body_io, deflate
end
elsif response.headers.includes_word?("Content-Encoding", "deflate")
Flate::Writer.open(env.response) do |deflate|
response.pipe(deflate)
Compress::Deflate::Writer.open(env.response) do |deflate|
IO.copy response.body_io, deflate
end
else
response.pipe(env.response)
IO.copy response.body_io, env.response
end
end
# See https://github.com/kemalcr/kemal/pull/576
class HTTP::Server::Response::Output
def close
return if closed?
unless response.wrote_headers?
response.content_length = @out_count
end
ensure_headers_written
super
if @chunked
@io << "0\r\n\r\n"
@io.flush
end
end
end

2
src/invidious/helpers/i18n.cr
View file

@ -24,6 +24,8 @@ def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text
if !locale[translation].as_s.empty?
translation = locale[translation].as_s
end
else
raise "Invalid translation #{translation}"
end
end

370
src/invidious/helpers/jobs.cr
View file

@ -1,370 +0,0 @@
def refresh_channels(db, logger, config)
max_channel = Channel(Int32).new
spawn do
max_threads = max_channel.receive
active_threads = 0
active_channel = Channel(Bool).new
loop do
db.query("SELECT id FROM channels ORDER BY updated") do |rs|
rs.each do
id = rs.read(String)
if active_threads >= max_threads
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
channel = fetch_channel(id, db, config.full_refresh)
db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
rescue ex
if ex.message == "Deleted or invalid channel"
db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
end
logger.puts("#{id} : #{ex.message}")
end
active_channel.send(true)
end
end
end
sleep 1.minute
Fiber.yield
end
end
max_channel.send(config.channel_threads)
end
def refresh_feeds(db, logger, config)
max_channel = Channel(Int32).new
spawn do
max_threads = max_channel.receive
active_threads = 0
active_channel = Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
rs.each do
email = rs.read(String)
view_name = "subscriptions_#{sha256(email)}"
if active_threads >= max_threads
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
# Drop outdated views
column_array = get_column_array(db, view_name)
ChannelVideo.to_type_tuple.each_with_index do |name, i|
if name != column_array[i]?
logger.puts("DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
raise "view does not exist"
end
end
if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
logger.puts("Materialized view #{view_name} is out-of-date, recreating...")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
end
db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
rescue ex
# Rename old views
begin
legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}")
db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
rescue ex
begin
# While iterating through, we may have an email stored from a deleted account
if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
logger.puts("CREATE #{view_name}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
end
rescue ex
logger.puts("REFRESH #{email} : #{ex.message}")
end
end
end
active_channel.send(true)
end
end
end
sleep 5.seconds
Fiber.yield
end
end
max_channel.send(config.feed_threads)
end
def subscribe_to_feeds(db, logger, key, config)
if config.use_pubsub_feeds
case config.use_pubsub_feeds
when Bool
max_threads = config.use_pubsub_feeds.as(Bool).to_unsafe
when Int32
max_threads = config.use_pubsub_feeds.as(Int32)
end
max_channel = Channel(Int32).new
spawn do
max_threads = max_channel.receive
active_threads = 0
active_channel = Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
rs.each do
ucid = rs.read(String)
if active_threads >= max_threads.as(Int32)
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
response = subscribe_pubsub(ucid, key, config)
if response.status_code >= 400
logger.puts("#{ucid} : #{response.body}")
end
rescue ex
logger.puts("#{ucid} : #{ex.message}")
end
active_channel.send(true)
end
end
end
sleep 1.minute
Fiber.yield
end
end
max_channel.send(max_threads.as(Int32))
end
end
def pull_top_videos(config, db)
loop do
begin
top = rank_videos(db, 40)
rescue ex
sleep 1.minute
Fiber.yield
next
end
if top.size == 0
sleep 1.minute
Fiber.yield
next
end
videos = [] of Video
top.each do |id|
begin
videos << get_video(id, db)
rescue ex
next
end
end
yield videos
sleep 1.minute
Fiber.yield
end
end
def pull_popular_videos(db)
loop do
videos = db.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE ucid IN \
(SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d \
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) \
ORDER BY ucid, published DESC", as: ChannelVideo).sort_by { |video| video.published }.reverse
yield videos
sleep 1.minute
Fiber.yield
end
end
def update_decrypt_function
loop do
begin
decrypt_function = fetch_decrypt_function
yield decrypt_function
rescue ex
next
ensure
sleep 1.minute
Fiber.yield
end
end
end
def bypass_captcha(captcha_key, logger)
loop do
begin
response = YT_POOL.client &.get("/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
headers = response.cookies.add_request_headers(HTTP::Headers.new)
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
# "type" => "NoCaptchaTask",
"websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999",
"websiteKey" => site_key,
# "proxyType" => "http",
# "proxyAddress" => CONFIG.proxy_address,
# "proxyPort" => CONFIG.proxy_port,
# "proxyLogin" => CONFIG.proxy_user,
# "proxyPassword" => CONFIG.proxy_pass,
# "userAgent" => "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
},
}.to_json).body)
if response["error"]?
raise response["error"].as_s
end
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
yield response.cookies.select { |cookie| cookie.name != "PREF" }
elsif response.headers["Location"]?.try &.includes?("/sorry/index")
location = response.headers["Location"].try { |u| URI.parse(u) }
client = QUIC::Client.new(location.host.not_nil!)
response = client.get(location.full_path)
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="index"])).not_nil!
site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
},
}.to_json).body)
if response["error"]?
raise response["error"].as_s
end
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
client.close
client = QUIC::Client.new("www.google.com")
response = client.post(location.full_path, form: inputs)
headers = HTTP::Headers{
"Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
}
cookies = HTTP::Cookies.from_headers(headers)
yield cookies
end
rescue ex
logger.puts("Exception: #{ex.message}")
ensure
sleep 1.minute
Fiber.yield
end
end
end
def find_working_proxies(regions)
loop do
regions.each do |region|
proxies = get_proxies(region).first(20)
proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
# proxies = filter_proxies(proxies)
yield region, proxies
end
sleep 1.minute
Fiber.yield
end
end

76
src/invidious/helpers/macros.cr
View file

@ -1,43 +1,51 @@
macro db_mapping(mapping)
def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
end
module DB::Serializable
macro included
{% verbatim do %}
macro finished
def self.type_array
\{{ @type.instance_vars
.reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
.map { |name| name.stringify }
}}
end
def to_a
return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
end
def initialize(tuple)
\{% for var in @type.instance_vars %}
\{% ann = var.annotation(::DB::Field) %}
\{% if ann && ann[:ignore] %}
\{% else %}
@\{{var.name}} = tuple[:\{{var.name.id}}]
\{% end %}
\{% end %}
end
def self.to_type_tuple
return { {{*mapping.keys.map { |id| "#{id}" }}} }
def to_a
\{{ @type.instance_vars
.reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] }
.map { |name| name }
}}
end
end
{% end %}
end
DB.mapping( {{mapping}} )
end
macro json_mapping(mapping)
def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
module JSON::Serializable
macro included
{% verbatim do %}
macro finished
def initialize(tuple)
\{% for var in @type.instance_vars %}
\{% ann = var.annotation(::JSON::Field) %}
\{% if ann && ann[:ignore] %}
\{% else %}
@\{{var.name}} = tuple[:\{{var.name.id}}]
\{% end %}
\{% end %}
end
end
{% end %}
end
def to_a
return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
end
patched_json_mapping( {{mapping}} )
YAML.mapping( {{mapping}} )
end
macro yaml_mapping(mapping)
def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
end
def to_a
return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ]
end
def to_tuple
return { {{*mapping.keys.map { |id| "@#{id}".id }}} }
end
YAML.mapping({{mapping}})
end
macro templated(filename, template = "template")

166
src/invidious/helpers/patch_mapping.cr
View file

@ -1,166 +0,0 @@
# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24
def Object.from_json(string_or_io, default) : self
parser = JSON::PullParser.new(string_or_io)
new parser, default
end
# Adds configurable 'default'
macro patched_json_mapping(_properties_, strict = false)
{% for key, value in _properties_ %}
{% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %}
{% end %}
{% for key, value in _properties_ %}
{% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %}
{% end %}
{% for key, value in _properties_ %}
@{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
{% if value[:setter] == nil ? true : value[:setter] %}
def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }})
@{{value[:key_id]}} = _{{value[:key_id]}}
end
{% end %}
{% if value[:getter] == nil ? true : value[:getter] %}
def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}
@{{value[:key_id]}}
end
{% end %}
{% if value[:presence] %}
@{{value[:key_id]}}_present : Bool = false
def {{value[:key_id]}}_present?
@{{value[:key_id]}}_present
end
{% end %}
{% end %}
def initialize(%pull : ::JSON::PullParser, default = nil)
{% for key, value in _properties_ %}
%var{key.id} = nil
%found{key.id} = false
{% end %}
%location = %pull.location
begin
%pull.read_begin_object
rescue exc : ::JSON::ParseException
raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc)
end
until %pull.kind.end_object?
%key_location = %pull.location
key = %pull.read_object_key
case key
{% for key, value in _properties_ %}
when {{value[:key] || value[:key_id].stringify}}
%found{key.id} = true
begin
%var{key.id} =
{% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %}
{% if value[:root] %}
%pull.on_key!({{value[:root]}}) do
{% end %}
{% if value[:converter] %}
{{value[:converter]}}.from_json(%pull)
{% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
{{value[:type]}}.new(%pull)
{% else %}
::Union({{value[:type]}}).new(%pull)
{% end %}
{% if value[:root] %}
end
{% end %}
{% if value[:nilable] || value[:default] != nil %} } {% end %}
rescue exc : ::JSON::ParseException
raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc)
end
{% end %}
else
{% if strict %}
raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil)
{% else %}
%pull.skip
{% end %}
end
end
%pull.read_next
{% for key, value in _properties_ %}
{% unless value[:nilable] || value[:default] != nil %}
if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable?
raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil)
end
{% end %}
{% if value[:nilable] %}
{% if value[:default] != nil %}
@{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}})
{% else %}
@{{value[:key_id]}} = %var{key.id}
{% end %}
{% elsif value[:default] != nil %}
@{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id}
{% else %}
@{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}})
{% end %}
{% if value[:presence] %}
@{{value[:key_id]}}_present = %found{key.id}
{% end %}
{% end %}
end
def to_json(json : ::JSON::Builder)
json.object do
{% for key, value in _properties_ %}
_{{value[:key_id]}} = @{{value[:key_id]}}
{% unless value[:emit_null] %}
unless _{{value[:key_id]}}.nil?
{% end %}
json.field({{value[:key] || value[:key_id].stringify}}) do
{% if value[:root] %}
{% if value[:emit_null] %}
if _{{value[:key_id]}}.nil?
nil.to_json(json)
else
{% end %}
json.object do
json.field({{value[:root]}}) do
{% end %}
{% if value[:converter] %}
if _{{value[:key_id]}}
{{ value[:converter] }}.to_json(_{{value[:key_id]}}, json)
else
nil.to_json(json)
end
{% else %}
_{{value[:key_id]}}.to_json(json)
{% end %}
{% if value[:root] %}
{% if value[:emit_null] %}
end
{% end %}
end
end
{% end %}
end
{% unless value[:emit_null] %}
end
{% end %}
{% end %}
end
end
end

58
src/invidious/helpers/signatures.cr
View file

@ -1,69 +1,53 @@
alias SigProc = Proc(Array(String), Int32, Array(String))
def fetch_decrypt_function(id = "CvFH_6DNRCY")
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
url = document.match(/src="(?<url>\/yts\/jsbin\/player_ias-.{9}\/en_US\/base.js)"/).not_nil!["url"]
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
player = YT_POOL.client &.get(url).body
function_name = player.match(/^(?<name>[^=]+)=function\(a\){a=a\.split\(""\)/m).not_nil!["name"]
function_body = player.match(/^#{Regex.escape(function_name)}=function\(a\){(?<body>[^}]+)}/m).not_nil!["body"]
function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
function_body = function_body.split(";")[1..-2]
var_name = function_body[0][0, 2]
var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
operations = {} of String => String
operations = {} of String => SigProc
var_body.split("},").each do |operation|
op_name = operation.match(/^[^:]+/).not_nil![0]
op_body = operation.match(/\{[^}]+/).not_nil![0]
case op_body
when "{a.reverse()"
operations[op_name] = "a"
operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
when "{a.splice(0,b)"
operations[op_name] = "b"
operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
else
operations[op_name] = "c"
operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
end
end
decrypt_function = [] of {name: String, value: Int32}
decrypt_function = [] of {SigProc, Int32}
function_body.each do |function|
function = function.lchop(var_name).delete("[].")
op_name = function.match(/[^\(]+/).not_nil![0]
value = function.match(/\(a,(?<value>[\d]+)\)/).not_nil!["value"].to_i
value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
decrypt_function << {name: operations[op_name], value: value}
decrypt_function << {operations[op_name], value}
end
return decrypt_function
end
def decrypt_signature(fmt, code)
if !fmt["s"]?
return ""
def decrypt_signature(fmt : Hash(String, JSON::Any))
return "" if !fmt["s"]? || !fmt["sp"]?
sp = fmt["sp"].as_s
sig = fmt["s"].as_s.split("")
DECRYPT_FUNCTION.each do |proc, value|
sig = proc.call(sig, value)
end
a = fmt["s"]
a = a.split("")
code.each do |item|
case item[:name]
when "a"
a.reverse!
when "b"
a.delete_at(0..(item[:value] - 1))
when "c"
a = splice(a, item[:value])
end
end
signature = a.join("")
return "&#{fmt["sp"]?}=#{signature}"
end
def splice(a, b)
c = a[0]
a[0] = a[b % a.size]
a[b % a.size] = c
return a
return "&#{sp}=#{sig.join("")}"
end

4
src/invidious/helpers/static_file_handler.cr
View file

@ -81,12 +81,12 @@ def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt
condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path)
if condition && request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
Gzip::Writer.open(env.response) do |deflate|
Compress::Gzip::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
Flate::Writer.open(env.response) do |deflate|
Compress::Deflate::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
else

40
src/invidious/helpers/tokens.cr
View file

@ -1,3 +1,5 @@
require "crypto/subtle"
def generate_token(email, scopes, expire, key, db)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc)
@ -41,15 +43,10 @@ def sign_token(key, hash)
string_to_sign = [] of String
hash.each do |key, value|
if key == "signature"
next
end
next if key == "signature"
if value.is_a?(JSON::Any)
case value
when .as_a?
value = value.as_a.map { |item| item.as_s }
end
if value.is_a?(JSON::Any) && value.as_a?
value = value.as_a.map { |i| i.as_s }
end
case value
@ -76,14 +73,25 @@ def validate_request(token, session, request, key, db, locale = nil)
raise translate(locale, "Hidden field \"token\" is a required field")
end
if token["signature"] != sign_token(key, token)
raise translate(locale, "Invalid signature")
expire = token["expire"]?.try &.as_i
if expire.try &.< Time.utc.to_unix
raise translate(locale, "Token is expired, please try again")
end
if token["session"] != session
raise translate(locale, "Erroneous token")
end
scopes = token["scopes"].as_a.map { |v| v.as_s }
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
raise translate(locale, "Invalid scope")
end
if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
raise translate(locale, "Invalid signature")
end
if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
if nonce[1] > Time.utc
db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
@ -92,18 +100,6 @@ def validate_request(token, session, request, key, db, locale = nil)
end
end
scopes = token["scopes"].as_a.map { |v| v.as_s }
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
raise translate(locale, "Invalid scope")
end
expire = token["expire"]?.try &.as_i
if expire.try &.< Time.utc.to_unix
raise translate(locale, "Token is expired, please try again")
end
return {scopes, expire, token["signature"].as_s}
end

60
src/invidious/helpers/utils.cr
View file

@ -2,13 +2,16 @@ require "lsquic"
require "pool/connection"
def add_yt_headers(request)
request.headers["x-youtube-client-name"] ||= "1"
request.headers["x-youtube-client-version"] ||= "1.20180719"
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["accept-language"] ||= "en-us,en;q=0.5"
request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
return if request.resource.starts_with? "/sorry/index"
request.headers["x-youtube-client-name"] ||= "1"
request.headers["x-youtube-client-version"] ||= "2.20200609"
if !CONFIG.cookies.empty?
request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
end
struct QUICPool
@ -77,7 +80,8 @@ def elapsed_text(elapsed)
end
def make_client(url : URI, region = nil)
client = HTTPClient.new(url)
# TODO: Migrate any applicable endpoints to QUIC
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
@ -99,7 +103,7 @@ end
def decode_length_seconds(string)
length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i
length_seconds = [0] * (3 - length_seconds.size) + length_seconds
length_seconds = Time::Span.new(length_seconds[0], length_seconds[1], length_seconds[2])
length_seconds = Time::Span.new hours: length_seconds[0], minutes: length_seconds[1], seconds: length_seconds[2]
length_seconds = length_seconds.total_seconds.to_i
return length_seconds
@ -161,6 +165,7 @@ def decode_date(string : String)
return Time.utc
when "yesterday"
return Time.utc - 1.day
else nil # Continue
end
# String matches format "20 hours ago", "4 months ago"...
@ -315,7 +320,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.full_path
referer = "/" + referer.lstrip("\/\\")
referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback
@ -324,47 +329,10 @@ def get_referer(env, fallback = "/", unroll = true)
return referer
end
struct VarInt
def self.from_io(io : IO, format = IO::ByteFormat::NetworkEndian) : Int32
result = 0_u32
num_read = 0
loop do
byte = io.read_byte
raise "Invalid VarInt" if !byte
value = byte & 0x7f
result |= value.to_u32 << (7 * num_read)
num_read += 1
break if byte & 0x80 == 0
raise "Invalid VarInt" if num_read > 5
end
result.to_i32
end
def self.to_io(io : IO, value : Int32)
io.write_byte 0x00 if value == 0x00
value = value.to_u32
while value != 0
byte = (value & 0x7f).to_u8
value >>= 7
if value != 0
byte |= 0x80
end
io.write_byte byte
end
end
end
def sha256(text)
digest = OpenSSL::Digest.new("SHA256")
digest << text
return digest.hexdigest
return digest.final.hexstring
end
def subscribe_pubsub(topic, key, config)
@ -383,10 +351,8 @@ def subscribe_pubsub(topic, key, config)
nonce = Random::Secure.hex(4)
signature = "#{time}:#{nonce}"
host_url = make_host_url(config, Kemal.config)
body = {
"hub.callback" => "#{host_url}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
"hub.callback" => "#{HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
"hub.topic" => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}",
"hub.verify" => "async",
"hub.mode" => "subscribe",

13
src/invidious/jobs.cr Normal file
View file

@ -0,0 +1,13 @@
module Invidious::Jobs
JOBS = [] of BaseJob
def self.register(job : BaseJob)
JOBS << job
end
def self.start_all
JOBS.each do |job|
spawn { job.begin }
end
end
end

3
src/invidious/jobs/base_job.cr Normal file
View file

@ -0,0 +1,3 @@
abstract class Invidious::Jobs::BaseJob
abstract def begin
end

131
src/invidious/jobs/bypass_captcha_job.cr Normal file
View file

@ -0,0 +1,131 @@
class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
private getter logger : Invidious::LogHandler
private getter config : Config
def initialize(@logger, @config)
end
def begin
loop do
begin
{"/watch?v=jNQXAC9IVRw&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UC4QobU6STFB0P71PMvOGN5A")}.each do |path|
response = YT_POOL.client &.get(path)
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
headers = response.cookies.add_request_headers(HTTP::Headers.new)
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
"clientKey" => config.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => "https://www.youtube.com#{path}",
"websiteKey" => site_key,
"recaptchaDataSValue" => s_value,
},
}.to_json).body)
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
"clientKey" => config.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
response.cookies
.select { |cookie| cookie.name != "PREF" }
.each { |cookie| config.cookies << cookie }
# Persist cookies between runs
File.write("config/config.yml", config.to_yaml)
elsif response.headers["Location"]?.try &.includes?("/sorry/index")
location = response.headers["Location"].try { |u| URI.parse(u) }
headers = HTTP::Headers{":authority" => location.host.not_nil!}
response = YT_POOL.client &.get(location.full_path, headers)
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="index"])).not_nil!
site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
captcha_client.family = config.force_resolve || Socket::Family::INET
response = JSON.parse(captcha_client.post("/createTask", body: {
"clientKey" => config.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
"recaptchaDataSValue" => s_value,
},
}.to_json).body)
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(captcha_client.post("/getTaskResult", body: {
"clientKey" => config.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
headers = HTTP::Headers{
"Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
}
cookies = HTTP::Cookies.from_headers(headers)
cookies.each { |cookie| config.cookies << cookie }
# Persist cookies between runs
File.write("config/config.yml", config.to_yaml)
end
end
rescue ex
logger.puts("Exception: #{ex.message}")
ensure
sleep 1.minute
Fiber.yield
end
end
end
end

24
src/invidious/jobs/notification_job.cr Normal file
View file

@ -0,0 +1,24 @@
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
private getter pg_url : URI
def initialize(@connection_channel, @pg_url)
end
def begin
connections = [] of Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
loop do
action, connection = connection_channel.receive
case action
when true
connections << connection
when false
connections.delete(connection)
end
end
end
end

27
src/invidious/jobs/pull_popular_videos_job.cr Normal file
View file

@ -0,0 +1,27 @@
class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
QUERY = <<-SQL
SELECT DISTINCT ON (ucid) *
FROM channel_videos
WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC
SQL
POPULAR_VIDEOS = Atomic.new([] of ChannelVideo)
private getter db : DB::Database
def initialize(@db)
end
def begin
loop do
videos = db.query_all(QUERY, as: ChannelVideo)
.sort_by(&.published)
.reverse
POPULAR_VIDEOS.set(videos)
sleep 1.minute
Fiber.yield
end
end
end

59
src/invidious/jobs/refresh_channels_job.cr Normal file
View file

@ -0,0 +1,59 @@
class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
private getter db : DB::Database
private getter logger : Invidious::LogHandler
private getter config : Config
def initialize(@db, @logger, @config)
end
def begin
max_threads = config.channel_threads
lim_threads = max_threads
active_threads = 0
active_channel = Channel(Bool).new
backoff = 1.seconds
loop do
db.query("SELECT id FROM channels ORDER BY updated") do |rs|
rs.each do
id = rs.read(String)
if active_threads >= lim_threads
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
channel = fetch_channel(id, db, config.full_refresh)
lim_threads = max_threads
db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
rescue ex
logger.puts("#{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
else
lim_threads = 1
logger.puts("#{id} : backing off for #{backoff}s")
sleep backoff
if backoff < 1.days
backoff += backoff
else
backoff = 1.days
end
end
end
active_channel.send(true)
end
end
end
sleep 1.minute
Fiber.yield
end
end
end

77
src/invidious/jobs/refresh_feeds_job.cr Normal file
View file

@ -0,0 +1,77 @@
class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
private getter db : DB::Database
private getter logger : Invidious::LogHandler
private getter config : Config
def initialize(@db, @logger, @config)
end
def begin
max_threads = config.feed_threads
active_threads = 0
active_channel = Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
rs.each do
email = rs.read(String)
view_name = "subscriptions_#{sha256(email)}"
if active_threads >= max_threads
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
# Drop outdated views
column_array = get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
logger.puts("DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
raise "view does not exist"
end
end
if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
logger.puts("Materialized view #{view_name} is out-of-date, recreating...")
db.exec("DROP MATERIALIZED VIEW #{view_name}")
end
db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
rescue ex
# Rename old views
begin
legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}")
db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
rescue ex
begin
# While iterating through, we may have an email stored from a deleted account
if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
logger.puts("CREATE #{view_name}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
end
rescue ex
logger.puts("REFRESH #{email} : #{ex.message}")
end
end
end
active_channel.send(true)
end
end
end
sleep 5.seconds
Fiber.yield
end
end
end

59
src/invidious/jobs/statistics_refresh_job.cr Normal file
View file

@ -0,0 +1,59 @@
class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
STATISTICS = {
"version" => "2.0",
"software" => {
"name" => "invidious",
"version" => "",
"branch" => "",
},
"openRegistrations" => true,
"usage" => {
"users" => {
"total" => 0_i64,
"activeHalfyear" => 0_i64,
"activeMonth" => 0_i64,
},
},
"metadata" => {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
}
private getter db : DB::Database
private getter config : Config
def initialize(@db, @config, @software_config : Hash(String, String))
end
def begin
load_initial_stats
loop do
refresh_stats
sleep 1.minute
Fiber.yield
end
end
# should only be called once at the very beginning
private def load_initial_stats
STATISTICS["software"] = {
"name" => @software_config["name"],
"version" => @software_config["version"],
"branch" => @software_config["branch"],
}
STATISTICS["openRegistration"] = config.registration_enabled
end
private def refresh_stats
users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64)
users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64)
users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64)
STATISTICS["metadata"] = {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64,
}
end
end

52
src/invidious/jobs/subscribe_to_feeds_job.cr Normal file
View file

@ -0,0 +1,52 @@
class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
private getter db : DB::Database
private getter logger : Invidious::LogHandler
private getter hmac_key : String
private getter config : Config
def initialize(@db, @logger, @config, @hmac_key)
end
def begin
max_threads = 1
if config.use_pubsub_feeds.is_a?(Int32)
max_threads = config.use_pubsub_feeds.as(Int32)
end
active_threads = 0
active_channel = Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
rs.each do
ucid = rs.read(String)
if active_threads >= max_threads.as(Int32)
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
response = subscribe_pubsub(ucid, hmac_key, config)
if response.status_code >= 400
logger.puts("#{ucid} : #{response.body}")
end
rescue ex
logger.puts("#{ucid} : #{ex.message}")
end
active_channel.send(true)
end
end
end
sleep 1.minute
Fiber.yield
end
end
end

19
src/invidious/jobs/update_decrypt_function_job.cr Normal file
View file

@ -0,0 +1,19 @@
class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
DECRYPT_FUNCTION = [] of {SigProc, Int32}
def begin
loop do
begin
decrypt_function = fetch_decrypt_function
DECRYPT_FUNCTION.clear
decrypt_function.each { |df| DECRYPT_FUNCTION << df }
rescue ex
# TODO: Log error
next
ensure
sleep 1.minute
Fiber.yield
end
end
end
end

61
src/invidious/mixes.cr
View file

@ -1,32 +1,32 @@
struct MixVideo
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
length_seconds: Int32,
index: Int32,
rdid: String,
})
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property index : Int32
property rdid : String
end
struct Mix
db_mapping({
title: String,
id: String,
videos: Array(MixVideo),
})
include DB::Serializable
property title : String
property id : String
property videos : Array(MixVideo)
end
def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
headers = HTTP::Headers.new
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
if cookies
headers = cookies.add_request_headers(headers)
end
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)
video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
@ -49,23 +49,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
id = item["videoId"].as_s
title = item["title"]?.try &.["simpleText"].as_s
if !title
next
end
next if !title
author = item["longBylineText"]["runs"][0]["text"].as_s
ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s
length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s)
index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i
videos << MixVideo.new(
title,
id,
author,
ucid,
length_seconds,
index,
rdid
)
videos << MixVideo.new({
title: title,
id: id,
author: author,
ucid: ucid,
length_seconds: length_seconds,
index: index,
rdid: rdid,
})
end
if !cookies
@ -75,7 +74,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
videos.uniq! { |video| video.id }
videos = videos.first(50)
return Mix.new(mix_title, rdid, videos)
return Mix.new({
title: mix_title,
id: rdid,
videos: videos,
})
end
def template_mix(mix)

380
src/invidious/playlists.cr
View file

@ -1,26 +1,38 @@
struct PlaylistVideo
def to_xml(host_url, auto_generated, xml : XML::Builder)
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
def to_xml(auto_generated, xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}")
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do
xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
@ -29,23 +41,23 @@ struct PlaylistVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(host_url, auto_generated, xml : XML::Builder? = nil)
def to_xml(auto_generated, xml : XML::Builder? = nil)
if xml
to_xml(host_url, auto_generated, xml)
to_xml(auto_generated, xml)
else
XML.build do |json|
to_xml(host_url, auto_generated, xml)
to_xml(auto_generated, xml)
end
end
end
def to_json(locale, config, kemal_config, json : JSON::Builder, index : Int32?)
def to_json(locale, json : JSON::Builder, index : Int32?)
json.object do
json.field "title", self.title
json.field "videoId", self.id
@ -55,7 +67,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id, config, kemal_config)
generate_thumbnails(json, self.id)
end
if index
@ -69,31 +81,32 @@ struct PlaylistVideo
end
end
def to_json(locale, config, kemal_config, json : JSON::Builder? = nil, index : Int32? = nil)
def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil)
if json
to_json(locale, config, kemal_config, json, index: index)
to_json(locale, json, index: index)
else
JSON.build do |json|
to_json(locale, config, kemal_config, json, index: index)
to_json(locale, json, index: index)
end
end
end
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
length_seconds: Int32,
published: Time,
plid: String,
index: Int64,
live_now: Bool,
})
end
struct Playlist
def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil)
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@ -118,7 +131,7 @@ struct Playlist
end
end
json.field "description", html_to_content(self.description_html)
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
@ -130,39 +143,30 @@ struct Playlist
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, config, Kemal.config, json)
video.to_json(locale, json)
end
end
end
end
end
def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil)
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, config, kemal_config, json, continuation: continuation)
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, config, kemal_config, json, continuation: continuation)
to_json(offset, locale, json, continuation: continuation)
end
end
end
db_mapping({
title: String,
id: String,
author: String,
author_thumbnail: String,
ucid: String,
description_html: String,
video_count: Int32,
views: Int64,
updated: Time,
thumbnail: String?,
})
def privacy
PlaylistPrivacy::Public
end
def description_html
HTML.escape(self.description).gsub("\n", "<br>")
end
end
enum PlaylistPrivacy
@ -172,7 +176,30 @@ enum PlaylistPrivacy
end
struct InvidiousPlaylist
def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil)
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
@ -195,43 +222,23 @@ struct InvidiousPlaylist
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, config, Kemal.config, json, offset + index)
video.to_json(locale, json, offset + index)
end
end
end
end
end
def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil)
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, config, kemal_config, json, continuation: continuation)
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, config, kemal_config, json, continuation: continuation)
to_json(offset, locale, json, continuation: continuation)
end
end
end
property thumbnail_id
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
db_mapping({
title: String,
id: String,
author: String,
description: {type: String, default: ""},
video_count: Int32,
created: Time,
updated: Time,
privacy: {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter},
index: Array(Int64),
})
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
@ -257,17 +264,17 @@ end
def create_playlist(db, title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new(
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
)
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
@ -277,50 +284,25 @@ def create_playlist(db, title, privacy, user)
return playlist
end
def extract_playlist(plid, nodeset, index)
videos = [] of PlaylistVideo
def subscribe_playlist(db, user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
nodeset.each_with_index do |video, offset|
anchor = video.xpath_node(%q(.//td[@class="pl-video-title"]))
if !anchor
next
end
playlist_array = playlist.to_a
args = arg_array(playlist_array)
title = anchor.xpath_node(%q(.//a)).not_nil!.content.strip(" \n")
id = anchor.xpath_node(%q(.//a)).not_nil!["href"].lchop("/watch?v=")[0, 11]
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
anchor = anchor.xpath_node(%q(.//div[@class="pl-video-owner"]/a))
if anchor
author = anchor.content
ucid = anchor["href"].split("/")[2]
else
author = ""
ucid = ""
end
anchor = video.xpath_node(%q(.//td[@class="pl-video-time"]/div/div[1]))
if anchor && !anchor.content.empty?
length_seconds = decode_length_seconds(anchor.content)
live_now = false
else
length_seconds = 0
live_now = true
end
videos << PlaylistVideo.new(
title: title,
id: id,
author: author,
ucid: ucid,
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
index: (index + offset).to_i64,
live_now: live_now
)
end
return videos
return playlist
end
def produce_playlist_url(id, index)
@ -368,58 +350,64 @@ def fetch_playlist(plid, locale)
plid = "UU#{plid.lchop("UC")}"
end
response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en&disable_polymer=1")
response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en")
if response.status_code != 200
raise translate(locale, "Not a playlist.")
if response.headers["location"]?.try &.includes? "/sorry/index"
raise "Could not extract playlist info. Instance is likely blocked."
else
raise translate(locale, "Not a playlist.")
end
end
body = response.body.gsub(/<button[^>]+><span[^>]+>\s*less\s*<img[^>]+>\n<\/span><\/button>/, "")
document = XML.parse_html(body)
initial_data = extract_initial_data(response.body)
playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]?
title = document.xpath_node(%q(//h1[@class="pl-header-title"]))
if !title
raise translate(locale, "Playlist does not exist.")
raise "Could not extract playlist info" if !playlist_info
title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || ""
desc_item = playlist_info["description"]?
description = desc_item.try &.["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]?
.try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
else
updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
end
end
title = title.content.strip(" \n")
description_html = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1])).try &.to_s ||
document.xpath_node(%q(//span[@class="pl-header-description-text"])).try &.to_s || ""
author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]?
.try &.["videoOwner"]["videoOwnerRenderer"]?
playlist_thumbnail = document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["data-thumb"]? ||
document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["src"]
raise "Could not extract author info" if !author_info
# YouTube allows anonymous playlists, so most of this can be empty or optional
anchor = document.xpath_node(%q(//ul[@class="pl-header-details"]))
author = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.content
author ||= ""
author_thumbnail = document.xpath_node(%q(//img[@class="channel-header-profile-image"])).try &.["src"]
author_thumbnail ||= ""
ucid = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.["href"].split("/")[-1]
ucid ||= ""
author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
video_count = anchor.try &.xpath_node(%q(.//li[2])).try &.content.gsub(/\D/, "").to_i?
video_count ||= 0
views = anchor.try &.xpath_node(%q(.//li[3])).try &.content.gsub(/\D/, "").to_i64?
views ||= 0_i64
updated = anchor.try &.xpath_node(%q(.//li[4])).try &.content.lchop("Last updated on ").lchop("Updated ").try { |date| decode_date(date) }
updated ||= Time.utc
playlist = Playlist.new(
title: title,
id: plid,
author: author,
return Playlist.new({
title: title,
id: plid,
author: author,
author_thumbnail: author_thumbnail,
ucid: ucid,
description_html: description_html,
video_count: video_count,
views: views,
updated: updated,
thumbnail: playlist_thumbnail,
)
return playlist
ucid: ucid,
description: description,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
})
end
def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
@ -437,35 +425,26 @@ end
def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil)
if continuation
html = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
html = XML.parse_html(html.body)
index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1
offset = index || offset
response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en")
initial_data = extract_initial_data(response.body)
offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset
end
if video_count > 100
url = produce_playlist_url(plid, offset)
response = YT_POOL.client &.get(url)
response = JSON.parse(response.body)
if !response["content_html"]? || response["content_html"].as_s.empty?
raise translate(locale, "Empty playlist")
end
document = XML.parse_html(response["content_html"].as_s)
nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
videos = extract_playlist(plid, nodeset, offset)
initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h
elsif offset > 100
return [] of PlaylistVideo
else # Extract first page of videos
response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
document = XML.parse_html(response.body)
nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
videos = extract_playlist(plid, nodeset, 0)
response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en")
initial_data = extract_initial_data(response.body)
end
return [] of PlaylistVideo if !initial_data
videos = extract_playlist_videos(initial_data)
until videos.empty? || videos[0].index == offset
videos.shift
end
@ -473,6 +452,45 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
return videos
end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo
(initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a ||
initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item|
if i = item["playlistVideoRenderer"]?
video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s
plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s
index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64
thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s
title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || ""
author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || ""
ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || ""
length_seconds = i["lengthSeconds"]?.try &.as_s.to_i
live = false
if !length_seconds
live = true
length_seconds = 0
end
videos << PlaylistVideo.new({
title: title,
id: video_id,
author: author,
ucid: ucid,
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index,
})
end
end
return videos
end
def template_playlist(playlist)
html = <<-END_HTML
<h3>

9
src/invidious/routes/base_route.cr Normal file
View file

@ -0,0 +1,9 @@
abstract class Invidious::Routes::BaseRoute
private getter config : Config
private getter logger : Invidious::LogHandler
def initialize(@config, @logger)
end
abstract def handle(env)
end

27
src/invidious/routes/embed/index.cr Normal file
View file

@ -0,0 +1,27 @@
class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute
def handle(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(PG_DB, plid, locale: locale)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
rescue ex
error_message = ex.message
env.response.status_code = 500
return templated "error"
end
url = "/embed/#{videos[0].id}?#{env.params.query}"
if env.params.query.size > 0
url += "?#{env.params.query}"
end
else
url = "/"
end
env.redirect url
end
end

174
src/invidious/routes/embed/show.cr Normal file
View file

@ -0,0 +1,174 @@
class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute
def handle(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(PG_DB, env.params.query, plid, id)
if md = env.params.query["playlist"]?
.try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/)
video_series = md[0].split(",")
env.params.query.delete("playlist")
end
preferences = env.get("preferences").as(Preferences)
if id.includes?("%20") || id.includes?("+") || env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
id = env.params.url["id"].gsub("%20", "").delete("+")
url = "/embed/#{id}"
if env.params.query.size > 0
url += "?#{env.params.query.to_s.gsub("%20", "").delete("+")}"
end
return env.redirect url
end
# YouTube embed supports `videoseries` with either `list=PLID`
# or `playlist=VIDEO_ID,VIDEO_ID`
case id
when "videoseries"
url = ""
if plid
begin
playlist = get_playlist(PG_DB, plid, locale: locale)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
rescue ex
error_message = ex.message
env.response.status_code = 500
return templated "error"
end
url = "/embed/#{videos[0].id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
else
return env.redirect "/"
end
if env.params.query.size > 0
url += "?#{env.params.query}"
end
return env.redirect url
when "live_stream"
response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
env.params.query.delete_all("channel")
if !video_id || video_id == "live_stream"
error_message = "Video is unavailable."
return templated "error"
end
url = "/embed/#{video_id}"
if env.params.query.size > 0
url += "?#{env.params.query}"
end
return env.redirect url
when id.size > 11
url = "/embed/#{id[0, 11]}"
if env.params.query.size > 0
url += "?#{env.params.query}"
end
return env.redirect url
else nil # Continue
end
params = process_video_params(env.params.query, preferences)
user = env.get?("user").try &.as(User)
if user
subscriptions = user.subscriptions
watched = user.watched
notifications = user.notifications
end
subscriptions ||= [] of String
begin
video = get_video(id, PG_DB, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex
error_message = ex.message
env.response.status_code = 500
return templated "error"
end
if preferences.annotations_subscribed &&
subscriptions.includes?(video.ucid) &&
(env.params.query["iv_load_policy"]? || "1") == "1"
params.annotations = true
end
# if watched && !watched.includes? id
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
if notifications && notifications.includes? id
PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
end
fmt_stream = video.fmt_stream
adaptive_fmts = video.adaptive_fmts
if params.local
fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
end
video_streams = video.video_streams
audio_streams = video.audio_streams
if audio_streams.empty? && !video.live_now
if params.quality == "dash"
env.params.query.delete_all("quality")
return env.redirect "/embed/#{id}?#{env.params.query}"
elsif params.listen
env.params.query.delete_all("listen")
env.params.query["listen"] = "0"
return env.redirect "/embed/#{id}?#{env.params.query}"
end
end
captions = video.captions
preferred_captions = captions.select { |caption|
params.preferred_captions.includes?(caption.name.simpleText) ||
params.preferred_captions.includes?(caption.languageCode.split("-")[0])
}
preferred_captions.sort_by! { |caption|
(params.preferred_captions.index(caption.name.simpleText) ||
params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
}
captions = captions - preferred_captions
aspect_ratio = nil
thumbnail = "/vi/#{video.id}/maxres.jpg"
if params.raw
url = fmt_stream[0]["url"].as_s
fmt_stream.each do |fmt|
url = fmt["url"].as_s if fmt["quality"].as_s == params.quality
end
return env.redirect url
end
rendered "embed"
end
end

34
src/invidious/routes/home.cr Normal file
View file

@ -0,0 +1,34 @@
class Invidious::Routes::Home < Invidious::Routes::BaseRoute
def handle(env)
preferences = env.get("preferences").as(Preferences)
locale = LOCALES[preferences.locale]?
user = env.get? "user"
case preferences.default_home
when ""
templated "empty"
when "Popular"
templated "popular"
when "Trending"
env.redirect "/feed/trending"
when "Subscriptions"
if user
env.redirect "/feed/subscriptions"
else
templated "popular"
end
when "Playlists"
if user
env.redirect "/view_all_playlists"
else
templated "popular"
end
else
templated "empty"
end
end
private def popular_videos
Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
end
end

6
src/invidious/routes/licenses.cr Normal file
View file

@ -0,0 +1,6 @@
class Invidious::Routes::Licenses < Invidious::Routes::BaseRoute
def handle(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
rendered "licenses"
end
end

6
src/invidious/routes/privacy.cr Normal file
View file

@ -0,0 +1,6 @@
class Invidious::Routes::Privacy < Invidious::Routes::BaseRoute
def handle(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
templated "privacy"
end
end

186
src/invidious/routes/watch.cr Normal file
View file

@ -0,0 +1,186 @@
class Invidious::Routes::Watch < Invidious::Routes::BaseRoute
def handle(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
region = env.params.query["region"]?
if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
url = "/watch?" + env.params.query.to_s.gsub("%20", "").delete("+")
return env.redirect url
end
if env.params.query["v"]?
id = env.params.query["v"]
if env.params.query["v"].empty?
error_message = "Invalid parameters."
env.response.status_code = 400
return templated "error"
end
if id.size > 11
url = "/watch?v=#{id[0, 11]}"
env.params.query.delete_all("v")
if env.params.query.size > 0
url += "&#{env.params.query}"
end
return env.redirect url
end
else
return env.redirect "/"
end
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(PG_DB, env.params.query, plid, id)
nojs = env.params.query["nojs"]?
nojs ||= "0"
nojs = nojs == "1"
preferences = env.get("preferences").as(Preferences)
user = env.get?("user").try &.as(User)
if user
subscriptions = user.subscriptions
watched = user.watched
notifications = user.notifications
end
subscriptions ||= [] of String
params = process_video_params(env.params.query, preferences)
env.params.query.delete_all("listen")
begin
video = get_video(id, PG_DB, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex
error_message = ex.message
env.response.status_code = 500
logger.puts("#{id} : #{ex.message}")
return templated "error"
end
if preferences.annotations_subscribed &&
subscriptions.includes?(video.ucid) &&
(env.params.query["iv_load_policy"]? || "1") == "1"
params.annotations = true
end
env.params.query.delete_all("iv_load_policy")
if watched && !watched.includes? id
PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
end
if notifications && notifications.includes? id
PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
end
if nojs
if preferences
source = preferences.comments[0]
if source.empty?
source = preferences.comments[1]
end
if source == "youtube"
begin
comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
rescue ex
if preferences.comments[1] == "reddit"
comments, reddit_thread = fetch_reddit_comments(id)
comment_html = template_reddit_comments(comments, locale)
comment_html = fill_links(comment_html, "https", "www.reddit.com")
comment_html = replace_links(comment_html)
end
end
elsif source == "reddit"
begin
comments, reddit_thread = fetch_reddit_comments(id)
comment_html = template_reddit_comments(comments, locale)
comment_html = fill_links(comment_html, "https", "www.reddit.com")
comment_html = replace_links(comment_html)
rescue ex
if preferences.comments[1] == "youtube"
comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
end
end
else
comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
comment_html ||= ""
end
fmt_stream = video.fmt_stream
adaptive_fmts = video.adaptive_fmts
if params.local
fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) }
end
video_streams = video.video_streams
audio_streams = video.audio_streams
# Older videos may not have audio sources available.
# We redirect here so they're not unplayable
if audio_streams.empty? && !video.live_now
if params.quality == "dash"
env.params.query.delete_all("quality")
env.params.query["quality"] = "medium"
return env.redirect "/watch?#{env.params.query}"
elsif params.listen
env.params.query.delete_all("listen")
env.params.query["listen"] = "0"
return env.redirect "/watch?#{env.params.query}"
end
end
captions = video.captions
preferred_captions = captions.select { |caption|
params.preferred_captions.includes?(caption.name.simpleText) ||
params.preferred_captions.includes?(caption.languageCode.split("-")[0])
}
preferred_captions.sort_by! { |caption|
(params.preferred_captions.index(caption.name.simpleText) ||
params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
}
captions = captions - preferred_captions
aspect_ratio = "16:9"
thumbnail = "/vi/#{video.id}/maxres.jpg"
if params.raw
if params.listen
url = audio_streams[0]["url"].as_s
audio_streams.each do |fmt|
if fmt["bitrate"].as_i == params.quality.rchop("k").to_i
url = fmt["url"].as_s
end
end
else
url = fmt_stream[0]["url"].as_s
fmt_stream.each do |fmt|
if fmt["quality"].as_s == params.quality
url = fmt["url"].as_s
end
end
end
return env.redirect url
end
templated "watch"
end
end

8
src/invidious/routing.cr Normal file
View file

@ -0,0 +1,8 @@
module Invidious::Routing
macro get(path, controller)
get {{ path }} do |env|
controller_instance = {{ controller }}.new(config, logger)
controller_instance.handle(env)
end
end
end

214
src/invidious/search.cr
View file

@ -1,5 +1,20 @@
struct SearchVideo
def to_xml(host_url, auto_generated, query_params, xml : XML::Builder)
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property live_now : Bool
property paid : Bool
property premium : Bool
property premiere_timestamp : Time?
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
@ -7,22 +22,22 @@ struct SearchVideo
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}")
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{host_url}/watch?#{query_params}") do
xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg")
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
@ -33,7 +48,7 @@ struct SearchVideo
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg",
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
@ -44,17 +59,17 @@ struct SearchVideo
end
end
def to_xml(host_url, auto_generated, query_params, xml : XML::Builder | Nil = nil)
def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
to_xml(host_url, auto_generated, query_params, xml)
to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
to_xml(host_url, auto_generated, query_params, xml)
to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
def to_json(locale, config, kemal_config, json : JSON::Builder)
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
@ -65,7 +80,7 @@ struct SearchVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id, config, kemal_config)
generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
@ -78,45 +93,49 @@ struct SearchVideo
json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium
end
end
json.field "isUpcoming", self.is_upcoming
def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
if json
to_json(locale, config, kemal_config, json)
else
JSON.build do |json|
to_json(locale, config, kemal_config, json)
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
end
end
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
published: Time,
views: Int64,
description_html: String,
length_seconds: Int32,
live_now: Bool,
paid: Bool,
premium: Bool,
premiere_timestamp: Time?,
})
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def is_upcoming
premiere_timestamp ? true : false
end
end
struct SearchPlaylistVideo
db_mapping({
title: String,
id: String,
length_seconds: Int32,
})
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
def to_json(locale, config, kemal_config, json : JSON::Builder)
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@ -137,7 +156,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id, config, Kemal.config)
generate_thumbnails(json, video.id)
end
end
end
@ -146,29 +165,29 @@ struct SearchPlaylist
end
end
def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, config, kemal_config, json)
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, config, kemal_config, json)
to_json(locale, json)
end
end
end
db_mapping({
title: String,
id: String,
author: String,
ucid: String,
video_count: Int32,
videos: Array(SearchPlaylistVideo),
thumbnail: String?,
})
end
struct SearchChannel
def to_json(locale, config, kemal_config, json : JSON::Builder)
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String
property auto_generated : Bool
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
@ -198,85 +217,50 @@ struct SearchChannel
end
end
def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil)
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, config, kemal_config, json)
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, config, kemal_config, json)
to_json(locale, json)
end
end
end
db_mapping({
author: String,
ucid: String,
author_thumbnail: String,
subscriber_count: Int32,
video_count: Int32,
description_html: String,
auto_generated: Bool,
})
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
def channel_search(query, page, channel)
response = YT_POOL.client &.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
document = XML.parse_html(response.body)
canonical = document.xpath_node(%q(//link[@rel="canonical"]))
response = YT_POOL.client &.get("/channel/#{channel}?hl=en&gl=US")
response = YT_POOL.client &.get("/user/#{channel}?hl=en&gl=US") if response.headers["location"]?
response = YT_POOL.client &.get("/c/#{channel}?hl=en&gl=US") if response.headers["location"]?
if !canonical
response = YT_POOL.client &.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
document = XML.parse_html(response.body)
canonical = document.xpath_node(%q(//link[@rel="canonical"]))
end
ucid = response.body.match(/\\"channelId\\":\\"(?<ucid>[^\\]+)\\"/).try &.["ucid"]?
if !canonical
response = YT_POOL.client &.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
document = XML.parse_html(response.body)
canonical = document.xpath_node(%q(//link[@rel="canonical"]))
end
if !canonical
return 0, [] of SearchItem
end
ucid = canonical["href"].split("/")[-1]
return 0, [] of SearchItem if !ucid
url = produce_channel_search_url(ucid, query, page)
response = YT_POOL.client &.get(url)
json = JSON.parse(response.body)
initial_data = JSON.parse(response.body).as_a.find &.["response"]?
return 0, [] of SearchItem if !initial_data
author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
items = extract_items(initial_data.as_h, author, ucid)
if json["content_html"]? && !json["content_html"].as_s.empty?
document = XML.parse_html(json["content_html"].as_s)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
count = nodeset.size
items = extract_items(nodeset)
else
count = 0
items = [] of SearchItem
end
return count, items
return items.size, items
end
def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil)
if query.empty?
return {0, [] of SearchItem}
end
return 0, [] of SearchItem if query.empty?
html = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body)
if html.empty?
return {0, [] of SearchItem}
end
body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body)
return 0, [] of SearchItem if body.empty?
html = XML.parse_html(html)
nodeset = html.xpath_nodes(%q(//ol[@class="item-section"]/li))
items = extract_items(nodeset)
initial_data = extract_initial_data(body)
items = extract_items(initial_data)
return {nodeset.size, items}
# initial_data["estimatedResults"]?.try &.as_s.to_i64
return items.size, items
end
def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "",
@ -310,6 +294,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["1:varint"] = 4_i64
when "year"
object["2:embedded"].as(Hash)["1:varint"] = 5_i64
else nil # Ignore
end
case content_type
@ -334,6 +319,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["3:varint"] = 1_i64
when "long"
object["2:embedded"].as(Hash)["3:varint"] = 2_i64
else nil # Ignore
end
features.each do |feature|
@ -358,6 +344,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
object["2:embedded"].as(Hash)["23:varint"] = 1_i64
when "hdr"
object["2:embedded"].as(Hash)["25:varint"] = 1_i64
else nil # Ignore
end
end
@ -379,12 +366,9 @@ def produce_channel_search_url(ucid, query, page)
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
"15:string" => "#{page}",
"23:varint" => 0_i64,
},
"11:string" => query,
},

52
src/invidious/trending.cr
View file

@ -1,7 +1,4 @@
def fetch_trending(trending_type, region, locale)
headers = HTTP::Headers.new
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
region ||= "US"
region = region.upcase
@ -11,7 +8,7 @@ def fetch_trending(trending_type, region, locale)
if trending_type && trending_type != "Default"
trending_type = trending_type.downcase.capitalize
response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en", headers).body
response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
initial_data = extract_initial_data(response)
@ -21,51 +18,28 @@ def fetch_trending(trending_type, region, locale)
if url
url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
url += "&disable_polymer=1&gl=#{region}&hl=en"
url = "#{url}&gl=#{region}&hl=en"
trending = YT_POOL.client &.get(url).body
plid = extract_plid(url)
else
trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
end
else
trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body
end
trending = XML.parse_html(trending)
nodeset = trending.xpath_nodes(%q(//ul/li[@class="expanded-shelf-content-item-wrapper"]))
trending = extract_videos(nodeset)
initial_data = extract_initial_data(trending)
trending = extract_videos(initial_data)
return {trending, plid}
end
def extract_plid(url)
wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"]
wrapper = URI.decode_www_form(wrapper)
wrapper = Base64.decode(wrapper)
# 0xe2 0x02 0x2e
wrapper += 3
# 0x0a
wrapper += 1
# Looks like "/m/[a-z0-9]{5}", not sure what it does here
item_size = wrapper[0]
wrapper += 1
item = wrapper[0, item_size]
wrapper += item.size
# 0x12
wrapper += 1
plid_size = wrapper[0]
wrapper += 1
plid = wrapper[0, plid_size]
wrapper += plid.size
plid = String.new(plid)
return plid
return url.try { |i| URI.parse(i).query }
.try { |i| HTTP::Params.parse(i)["bp"] }
.try { |i| URI.decode_www_form(i) }
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try &.["44:0:embedded"]?.try &.["2:1:string"]?.try &.as_s
end

337
src/invidious/users.cr
View file

@ -4,6 +4,20 @@ require "crypto/bcrypt/password"
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
struct User
include DB::Serializable
property updated : Time
property notifications : Array(String)
property subscriptions : Array(String)
property email : String
@[DB::Field(converter: User::PreferencesConverter)]
property preferences : Preferences
property password : String?
property token : String
property watched : Array(String)
property feed_needs_update : Bool?
module PreferencesConverter
def self.from_rs(rs)
begin
@ -13,31 +27,78 @@ struct User
end
end
end
db_mapping({
updated: Time,
notifications: Array(String),
subscriptions: Array(String),
email: String,
preferences: {
type: Preferences,
converter: PreferencesConverter,
},
password: String?,
token: String,
watched: Array(String),
feed_needs_update: Bool?,
})
end
struct Preferences
module ProcessString
include JSON::Serializable
include YAML::Serializable
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
@[JSON::Field(converter: Preferences::StringToArray)]
@[YAML::Field(converter: Preferences::StringToArray)]
property captions : Array(String) = CONFIG.default_user_preferences.captions
@[JSON::Field(converter: Preferences::StringToArray)]
@[YAML::Field(converter: Preferences::StringToArray)]
property comments : Array(String) = CONFIG.default_user_preferences.comments
property continue : Bool = CONFIG.default_user_preferences.continue
property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
@[JSON::Field(converter: Preferences::BoolToString)]
@[YAML::Field(converter: Preferences::BoolToString)]
property dark_mode : String = CONFIG.default_user_preferences.dark_mode
property latest_only : Bool = CONFIG.default_user_preferences.latest_only
property listen : Bool = CONFIG.default_user_preferences.listen
property local : Bool = CONFIG.default_user_preferences.local
@[JSON::Field(converter: Preferences::ProcessString)]
property locale : String = CONFIG.default_user_preferences.locale
@[JSON::Field(converter: Preferences::ClampInt)]
property max_results : Int32 = CONFIG.default_user_preferences.max_results
property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only
@[JSON::Field(converter: Preferences::ProcessString)]
property player_style : String = CONFIG.default_user_preferences.player_style
@[JSON::Field(converter: Preferences::ProcessString)]
property quality : String = CONFIG.default_user_preferences.quality
property default_home : String = CONFIG.default_user_preferences.default_home
property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu
property related_videos : Bool = CONFIG.default_user_preferences.related_videos
@[JSON::Field(converter: Preferences::ProcessString)]
property sort : String = CONFIG.default_user_preferences.sort
property speed : Float32 = CONFIG.default_user_preferences.speed
property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode
property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only
property video_loop : Bool = CONFIG.default_user_preferences.video_loop
property volume : Int32 = CONFIG.default_user_preferences.volume
module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
HTML.escape(value.read_string[0, 100])
begin
result = value.read_string
if result.empty?
CONFIG.default_user_preferences.dark_mode
else
result
end
rescue ex
if value.read_bool
"dark"
else
"light"
end
end
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
@ -45,7 +106,20 @@ struct Preferences
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
HTML.escape(node.value[0, 100])
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
case node.value
when "true"
"dark"
when "false"
"light"
when ""
CONFIG.default_user_preferences.dark_mode
else
node.value
end
end
end
@ -67,33 +141,130 @@ struct Preferences
end
end
json_mapping({
annotations: {type: Bool, default: CONFIG.default_user_preferences.annotations},
annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed},
autoplay: {type: Bool, default: CONFIG.default_user_preferences.autoplay},
captions: {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray},
comments: {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray},
continue: {type: Bool, default: CONFIG.default_user_preferences.continue},
continue_autoplay: {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay},
dark_mode: {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString},
latest_only: {type: Bool, default: CONFIG.default_user_preferences.latest_only},
listen: {type: Bool, default: CONFIG.default_user_preferences.listen},
local: {type: Bool, default: CONFIG.default_user_preferences.local},
locale: {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString},
max_results: {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt},
notifications_only: {type: Bool, default: CONFIG.default_user_preferences.notifications_only},
player_style: {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString},
quality: {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString},
default_home: {type: String, default: CONFIG.default_user_preferences.default_home},
feed_menu: {type: Array(String), default: CONFIG.default_user_preferences.feed_menu},
related_videos: {type: Bool, default: CONFIG.default_user_preferences.related_videos},
sort: {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString},
speed: {type: Float32, default: CONFIG.default_user_preferences.speed},
thin_mode: {type: Bool, default: CONFIG.default_user_preferences.thin_mode},
unseen_only: {type: Bool, default: CONFIG.default_user_preferences.unseen_only},
video_loop: {type: Bool, default: CONFIG.default_user_preferences.video_loop},
volume: {type: Int32, default: CONFIG.default_user_preferences.volume},
})
module FamilyConverter
def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
case value
when Socket::Family::UNSPEC
yaml.scalar nil
when Socket::Family::INET
yaml.scalar "ipv4"
when Socket::Family::INET6
yaml.scalar "ipv6"
when Socket::Family::UNIX
raise "Invalid socket family #{value}"
end
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
if node.is_a?(YAML::Nodes::Scalar)
case node.value.downcase
when "ipv4"
Socket::Family::INET
when "ipv6"
Socket::Family::INET6
else
Socket::Family::UNSPEC
end
else
node.raise "Expected scalar, not #{node.class}"
end
end
end
module ProcessString
def self.to_json(value : String, json : JSON::Builder)
json.string value
end
def self.from_json(value : JSON::PullParser) : String
HTML.escape(value.read_string[0, 100])
end
def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
yaml.scalar value
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
HTML.escape(node.value[0, 100])
end
end
module StringToArray
def self.to_json(value : Array(String), json : JSON::Builder)
json.array do
value.each do |element|
json.string element
end
end
end
def self.from_json(value : JSON::PullParser) : Array(String)
begin
result = [] of String
value.read_array do
result << HTML.escape(value.read_string[0, 100])
end
rescue ex
result = [HTML.escape(value.read_string[0, 100]), ""]
end
result
end
def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
yaml.sequence do
value.each do |element|
yaml.scalar element
end
end
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
begin
unless node.is_a?(YAML::Nodes::Sequence)
node.raise "Expected sequence, not #{node.class}"
end
result = [] of String
node.nodes.each do |item|
unless item.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{item.class}"
end
result << HTML.escape(item.value[0, 100])
end
rescue ex
if node.is_a?(YAML::Nodes::Scalar)
result = [HTML.escape(node.value[0, 100]), ""]
else
result = ["", ""]
end
end
result
end
end
module StringToCookies
def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
(value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
end
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end
cookies = HTTP::Cookies.new
node.value.split(";").each do |cookie|
next if cookie.strip.empty?
name, value = cookie.split("=", 2)
cookies << HTTP::Cookie.new(name.strip, value.strip)
end
cookies
end
end
end
def get_user(sid, headers, db, refresh = true)
@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true)
if refresh && Time.utc - user.updated > 1.minute
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
user_array[4] = user_array[4].to_json
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
db.exec("INSERT INTO users VALUES (#{args}) \
@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true)
else
user, sid = fetch_user(sid, headers, db)
user_array = user.to_a
user_array[4] = user_array[4].to_json
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user.to_a)
db.exec("INSERT INTO users VALUES (#{args}) \
@ -166,7 +335,17 @@ def fetch_user(sid, headers, db)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true)
user = User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: channels,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: nil,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
@ -174,7 +353,17 @@ def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true)
user = User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: [] of String,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: password.to_s,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
@ -267,7 +456,7 @@ def subscribe_ajax(channel_id, action, env_headers)
end
headers = cookies.add_request_headers(headers)
if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
session_token = match["session_token"]
headers["content-type"] = "application/x-www-form-urlencoded"
@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers)
end
end
# TODO: Playlist stub, sync with YouTube for Google accounts
# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers)
# headers = HTTP::Headers.new
# headers["Cookie"] = env_headers["Cookie"]
#
# html = YT_POOL.client &.get("/view_all_playlists?disable_polymer=1", headers)
#
# cookies = HTTP::Cookies.from_headers(headers)
# html.cookies.each do |cookie|
# if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
# if cookies[cookie.name]?
# cookies[cookie.name] = cookie
# else
# cookies << cookie
# end
# end
# end
# headers = cookies.add_request_headers(headers)
#
# if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
# session_token = match["session_token"]
#
# headers["content-type"] = "application/x-www-form-urlencoded"
#
# post_req = {
# video_ids: [] of String,
# source_playlist_id: "",
# n: name,
# p: privacy,
# session_token: session_token,
# }
# post_url = "/playlist_ajax?#{action}=1"
#
# response = client.post(post_url, headers, form: post_req)
# if response.status_code == 200
# return JSON.parse(response.body)["result"]["playlistId"].as_s
# else
# return nil
# end
# end
# end
def get_subscription_feed(db, user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
@ -350,6 +497,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
notifications.sort_by! { |video| video.author }
when "channel name - reverse"
notifications.sort_by! { |video| video.author }.reverse!
else nil # Ignore
end
else
if user.preferences.latest_only
@ -398,6 +546,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
videos.sort_by! { |video| video.author }
when "channel name - reverse"
videos.sort_by! { |video| video.author }.reverse!
else nil # Ignore
end
notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String))

1084
src/invidious/videos.cr
View file

File diff suppressed because it is too large Load diff

12
src/invidious/views/add_playlist_items.ecr
View file

@ -20,12 +20,14 @@
<div class="pure-u-1 pure-u-lg-1-5"></div>
</div>
<script>
var playlist_data = {
csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
}
<script id="playlist_data" type="application/json">
<%=
{
"csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
}.to_pretty_json
%>
</script>
<script src="/js/playlist_widget.js"></script>
<script src="/js/playlist_widget.js?v=<%= ASSET_COMMIT %>"></script>
<div class="pure-g">
<% videos.each_slice(4) do |slice| %>

6
src/invidious/views/channel.ecr
View file

@ -28,7 +28,7 @@
</div>
<div class="h-box">
<p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
<p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
</div>
<div class="h-box">
@ -92,7 +92,7 @@
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= sort_by %><% end %>">
<a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
@ -100,7 +100,7 @@
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count == 60 %>
<a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= sort_by %><% end %>">
<a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>

20
src/invidious/views/community.ecr
View file

@ -71,14 +71,16 @@
</div>
<% end %>
<script>
var community_data = {
ucid: '<%= channel.ucid %>',
youtube_comments_text: '<%= HTML.escape(translate(locale, "View YouTube comments")) %>',
comments_text: '<%= HTML.escape(translate(locale, "View `x` comments", "{commentCount}")) %>',
hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>',
show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>',
preferences: <%= env.get("preferences").as(Preferences).to_json %>,
}
<script id="community_data" type="application/json">
<%=
{
"ucid" => channel.ucid,
"youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")),
"comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")),
"hide_replies_text" => HTML.escape(translate(locale, "Hide replies")),
"show_replies_text" => HTML.escape(translate(locale, "Show replies")),
"preferences" => env.get("preferences").as(Preferences)
}.to_pretty_json
%>
</script>
<script src="/js/community.js?v=<%= ASSET_COMMIT %>"></script>

28
src/invidious/views/components/feed_menu.ecr
View file

@ -1,19 +1,11 @@
<div class="h-box pure-g">
<div class="pure-u-1 pure-u-md-1-4"></div>
<div class="pure-u-1 pure-u-md-1-2">
<div class="pure-g">
<% feed_menu = env.get("preferences").as(Preferences).feed_menu.dup %>
<% if !env.get?("user") %>
<% feed_menu.reject! {|item| {"Subscriptions", "Playlists"}.includes? item} %>
<% end %>
<% feed_menu.each do |feed| %>
<div class="pure-u-1-2 pure-u-md-1-<%= feed_menu.size %>">
<a href="/feed/<%= feed.downcase %>" class="pure-menu-heading" style="text-align:center">
<%= translate(locale, feed) %>
</a>
</div>
<% end %>
</div>
</div>
<div class="pure-u-1 pure-u-md-1-4"></div>
<div class="feed-menu">
<% feed_menu = env.get("preferences").as(Preferences).feed_menu.dup %>
<% if !env.get?("user") %>
<% feed_menu.reject! {|item| {"Subscriptions", "Playlists"}.includes? item} %>
<% end %>
<% feed_menu.each do |feed| %>
<a href="/feed/<%= feed.downcase %>" class="feed-menu-item pure-menu-heading">
<%= translate(locale, feed) %>
</a>
<% end %>
</div>

25
src/invidious/views/components/item.ecr
View file

@ -44,7 +44,7 @@
<% end %>
</div>
<% end %>
<p><%= item.title %></p>
<p><%= HTML.escape(item.title) %></p>
</a>
<p>
<b>
@ -57,10 +57,10 @@
<div class="thumbnail">
<img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/>
<% if plid = env.get?("remove_playlist_items") %>
<form onsubmit="return false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
<form data-onsubmit="return_false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
<a onclick="remove_playlist_item(this)" data-index="<%= item.index %>" data-plid="<%= plid %>" href="javascript:void(0)">
<a data-onclick="remove_playlist_item" data-index="<%= item.index %>" data-plid="<%= plid %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i class="icon ion-md-trash"></i>
</button>
@ -76,7 +76,7 @@
<% end %>
</div>
<% end %>
<p><a href="/watch?v=<%= item.id %>"><%= item.title %></a></p>
<p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p>
</a>
<p>
<b>
@ -85,7 +85,7 @@
</p>
<h5 class="pure-g">
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %>
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div>
<% elsif Time.utc - item.published > 1.minute %>
<div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div>
@ -103,13 +103,12 @@
<div class="thumbnail">
<img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/>
<% if env.get? "show_watched" %>
<form onsubmit="return false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post">
<form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
<a onclick="mark_watched(this)" data-id="<%= item.id %>" href="javascript:void(0)">
<a data-onclick="mark_watched" data-id="<%= item.id %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i onmouseenter='this.setAttribute("class", "icon ion-ios-eye-off")'
onmouseleave='this.setAttribute("class", "icon ion-ios-eye")'
<i data-mouse="switch_classes" data-switch-classes="ion-ios-eye-off,ion-ios-eye"
class="icon ion-ios-eye">
</i>
</button>
@ -117,10 +116,10 @@
</p>
</form>
<% elsif plid = env.get? "add_playlist_items" %>
<form onsubmit="return false" action="/playlist_ajax?action_add_video=1&video_id=<%= item.id %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
<form data-onsubmit="return_false" action="/playlist_ajax?action_add_video=1&video_id=<%= item.id %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
<a onclick="add_playlist_item(this)" data-id="<%= item.id %>" data-plid="<%= plid %>" href="javascript:void(0)">
<a data-onclick="add_playlist_item" data-id="<%= item.id %>" data-plid="<%= plid %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i class="icon ion-md-add"></i>
</button>
@ -137,7 +136,7 @@
</div>
</a>
<% end %>
<p><a href="/watch?v=<%= item.id %>"><%= item.title %></a></p>
<p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p>
<p style="display: flex;">
<b style="flex: 1;">
<a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a>
@ -148,7 +147,7 @@
</p>
<h5 class="pure-g">
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %>
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div>
<% elsif Time.utc - item.published > 1.minute %>
<div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div>

35
src/invidious/views/components/player.ecr
View file

@ -1,28 +1,25 @@
<video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>" title="<%= HTML.escape(video.title) %>"
id="player" class="video-js player-style-<%= params.player_style %>"
onmouseenter='this["data-title"]=this["title"];this["title"]=""'
onmouseleave='this["title"]=this["data-title"];this["data-title"]=""'
oncontextmenu='this["title"]=this["data-title"]'
<video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>"
id="player" class="on-video_player video-js player-style-<%= params.player_style %>"
<% if params.autoplay %>autoplay<% end %>
<% if params.video_loop %>loop<% end %>
<% if params.controls %>controls<% end %>>
<% if hlsvp && !CONFIG.disabled?("livestreams") %>
<source src="<%= hlsvp %>?local=true" type="application/x-mpegURL" label="livestream">
<% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
<source src="<%= URI.parse(hlsvp).full_path %>?local=true" type="application/x-mpegURL" label="livestream">
<% else %>
<% if params.listen %>
<% audio_streams.each_with_index do |fmt, i| %>
<source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>">
<source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>">
<% end %>
<% else %>
<% if params.quality == "dash" %>
<source src="/api/manifest/dash/id/<%= video.id %>?local=true" type='application/dash+xml' label="dash">
<source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
<% end %>
<% fmt_stream.each_with_index do |fmt, i| %>
<% if params.quality %>
<source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= params.quality == fmt["label"].split(" - ")[0] %>">
<source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= params.quality == fmt["quality"] %>">
<% else %>
<source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= i == 0 ? true : false %>">
<source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= i == 0 ? true : false %>">
<% end %>
<% end %>
<% end %>
@ -39,12 +36,14 @@
<% end %>
</video>
<script>
var player_data = {
aspect_ratio: '<%= aspect_ratio %>',
title: "<%= video.title.dump_unquoted %>",
description: "<%= HTML.escape(video.short_description) %>",
thumbnail: "<%= thumbnail %>"
}
<script id="player_data" type="application/json">
<%=
{
"aspect_ratio" => aspect_ratio,
"title" => video.title,
"description" => HTML.escape(video.short_description),
"thumbnail" => thumbnail
}.to_pretty_json
%>
</script>
<script src="/js/player.js?v=<%= ASSET_COMMIT %>"></script>

2
src/invidious/views/components/player_sources.ecr
View file

@ -3,6 +3,8 @@
<link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/videojs-vtt-thumbnails-fix.css?v=<%= ASSET_COMMIT %>">
<script src="/js/global.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/videojs-http-source-selector.min.js?v=<%= ASSET_COMMIT %>"></script>

20
src/invidious/views/components/subscribe_widget.ecr
View file

@ -19,15 +19,17 @@
</p>
<% end %>
<script>
var subscribe_data = {
ucid: '<%= ucid %>',
author: '<%= HTML.escape(author) %>',
sub_count_text: '<%= HTML.escape(sub_count_text) %>',
csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
subscribe_text: '<%= HTML.escape(translate(locale, "Subscribe")) %>',
unsubscribe_text: '<%= HTML.escape(translate(locale, "Unsubscribe")) %>'
}
<script id="subscribe_data" type="application/json">
<%=
{
"ucid" => ucid,
"author" => HTML.escape(author),
"sub_count_text" => HTML.escape(sub_count_text),
"csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || ""),
"subscribe_text" => HTML.escape(translate(locale, "Subscribe")),
"unsubscribe_text" => HTML.escape(translate(locale, "Unsubscribe"))
}.to_pretty_json
%>
</script>
<script src="/js/subscribe_widget.js?v=<%= ASSET_COMMIT %>"></script>
<% else %>

37
src/invidious/views/embed.ecr
View file

@ -10,33 +10,24 @@
<script src="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>"></script>
<link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/embed.css?v=<%= ASSET_COMMIT %>">
<title><%= HTML.escape(video.title) %> - Invidious</title>
<style>
#player {
position: fixed;
right: 0;
bottom: 0;
min-width: 100%;
min-height: 100%;
width: auto;
height: auto;
z-index: -100;
}
</style>
</head>
<body>
<script>
var video_data = {
id: '<%= video.id %>',
index: '<%= continuation %>',
plid: '<%= plid %>',
length_seconds: '<%= video.length_seconds.to_f %>',
video_series: <%= video_series.to_json %>,
params: <%= params.to_json %>,
preferences: <%= preferences.to_json %>,
premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %>
}
<script id="video_data" type="application/json">
<%=
{
"id" => video.id,
"index" => continuation,
"plid" => plid,
"length_seconds" => video.length_seconds.to_f,
"video_series" => video_series,
"params" => params,
"preferences" => preferences,
"premiere_timestamp" => video.premiere_timestamp.try &.to_unix
}.to_pretty_json
%>
</script>
<%= rendered "components/player" %>

14
src/invidious/views/history.ecr
View file

@ -18,10 +18,12 @@
</div>
</div>
<script>
var watched_data = {
csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
}
<script id="watched_data" type="application/json">
<%=
{
"csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
}.to_pretty_json
%>
</script>
<script src="/js/watched_widget.js"></script>
@ -34,10 +36,10 @@ var watched_data = {
<% if !env.get("preferences").as(Preferences).thin_mode %>
<div class="thumbnail">
<img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/>
<form onsubmit="return false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post">
<form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<p class="watched">
<a onclick="mark_unwatched(this)" data-id="<%= item %>" href="javascript:void(0)">
<a data-onclick="mark_unwatched" data-id="<%= item %>" href="javascript:void(0)">
<button type="submit" style="all:unset">
<i class="icon ion-md-trash"></i>
</button>

126
src/invidious/views/login.ecr
View file

@ -22,69 +22,6 @@
<hr>
<% case account_type when %>
<% when "invidious" %>
<form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post">
<fieldset>
<% if email %>
<input name="email" type="hidden" value="<%= email %>">
<% else %>
<label for="email"><%= translate(locale, "User ID") %> :</label>
<input required class="pure-input-1" name="email" type="text" placeholder="<%= translate(locale, "User ID") %>">
<% end %>
<% if password %>
<input name="password" type="hidden" value="<%= HTML.escape(password) %>">
<% else %>
<label for="password"><%= translate(locale, "Password") %> :</label>
<input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>">
<% end %>
<% if captcha %>
<% case captcha_type when %>
<% when "image" %>
<% captcha = captcha.not_nil! %>
<img style="width:50%" src='<%= captcha[:question] %>'/>
<% captcha[:tokens].each_with_index do |token, i| %>
<input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>">
<% end %>
<input type="hidden" name="captcha_type" value="image">
<label for="answer"><%= translate(locale, "Time (h:mm:ss):") %></label>
<input type="text" name="answer" type="text" placeholder="h:mm:ss">
<% when "text" %>
<% captcha = captcha.not_nil! %>
<% captcha[:tokens].each_with_index do |token, i| %>
<input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>">
<% end %>
<input type="hidden" name="captcha_type" value="text">
<label for="answer"><%= captcha[:question] %></label>
<input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>">
<% end %>
<button type="submit" name="action" value="signin" class="pure-button pure-button-primary">
<%= translate(locale, "Register") %>
</button>
<% case captcha_type when %>
<% when "image" %>
<label>
<button type="submit" name="change_type" class="pure-button pure-button-primary" value="text">
<%= translate(locale, "Text CAPTCHA") %>
</button>
</label>
<% when "text" %>
<label>
<button type="submit" name="change_type" class="pure-button pure-button-primary" value="image">
<%= translate(locale, "Image CAPTCHA") %>
</button>
</label>
<% end %>
<% else %>
<button type="submit" name="action" value="signin" class="pure-button pure-button-primary">
<%= translate(locale, "Sign In") %>/<%= translate(locale, "Register") %>
</button>
<% end %>
</fieldset>
</form>
<% when "google" %>
<form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post">
<fieldset>
@ -121,6 +58,69 @@
<button type="submit" class="pure-button pure-button-primary"><%= translate(locale, "Sign In") %></button>
</fieldset>
</form>
<% else # "invidious" %>
<form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post">
<fieldset>
<% if email %>
<input name="email" type="hidden" value="<%= email %>">
<% else %>
<label for="email"><%= translate(locale, "User ID") %> :</label>
<input required class="pure-input-1" name="email" type="text" placeholder="<%= translate(locale, "User ID") %>">
<% end %>
<% if password %>
<input name="password" type="hidden" value="<%= HTML.escape(password) %>">
<% else %>
<label for="password"><%= translate(locale, "Password") %> :</label>
<input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>">
<% end %>
<% if captcha %>
<% case captcha_type when %>
<% when "image" %>
<% captcha = captcha.not_nil! %>
<img style="width:50%" src='<%= captcha[:question] %>'/>
<% captcha[:tokens].each_with_index do |token, i| %>
<input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>">
<% end %>
<input type="hidden" name="captcha_type" value="image">
<label for="answer"><%= translate(locale, "Time (h:mm:ss):") %></label>
<input type="text" name="answer" type="text" placeholder="h:mm:ss">
<% else # "text" %>
<% captcha = captcha.not_nil! %>
<% captcha[:tokens].each_with_index do |token, i| %>
<input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>">
<% end %>
<input type="hidden" name="captcha_type" value="text">
<label for="answer"><%= captcha[:question] %></label>
<input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>">
<% end %>
<button type="submit" name="action" value="signin" class="pure-button pure-button-primary">
<%= translate(locale, "Register") %>
</button>
<% case captcha_type when %>
<% when "image" %>
<label>
<button type="submit" name="change_type" class="pure-button pure-button-primary" value="text">
<%= translate(locale, "Text CAPTCHA") %>
</button>
</label>
<% else # "text" %>
<label>
<button type="submit" name="change_type" class="pure-button pure-button-primary" value="image">
<%= translate(locale, "Image CAPTCHA") %>
</button>
</label>
<% end %>
<% else %>
<button type="submit" name="action" value="signin" class="pure-button pure-button-primary">
<%= translate(locale, "Sign In") %>/<%= translate(locale, "Register") %>
</button>
<% end %>
</fieldset>
</form>
<% end %>
</div>
</div>

18
src/invidious/views/playlist.ecr
View file

@ -45,6 +45,12 @@
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
<% else %>
<% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %>
<div class="pure-u-1-3"><a href="/subscribe_playlist?list=<%= plid %>"><i class="icon ion-md-add"></i></a></div>
<% else %>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
<% end %>
<% end %>
<div class="pure-u-1-3"><a href="/feed/playlist/<%= plid %>"><i class="icon ion-logo-rss"></i></a></div>
</div>
@ -69,12 +75,14 @@
</div>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<script>
var playlist_data = {
csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
}
<script id="playlist_data" type="application/json">
<%=
{
"csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
}.to_pretty_json
%>
</script>
<script src="/js/playlist_widget.js"></script>
<script src="/js/playlist_widget.js?v=<%= ASSET_COMMIT %>"></script>
<% end %>
<div class="pure-g">

2
src/invidious/views/playlists.ecr
View file

@ -90,7 +90,7 @@
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if continuation %>
<a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= sort_by %><% end %>">
<a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>

15
src/invidious/views/preferences.ecr
View file

@ -2,12 +2,6 @@
<title><%= translate(locale, "Preferences") %> - Invidious</title>
<% end %>
<script>
function update_value(element) {
document.getElementById('volume-value').innerText = element.value;
}
</script>
<div class="h-box">
<form class="pure-form pure-form-aligned" action="/preferences?referer=<%= URI.encode_www_form(referer) %>" method="post">
<fieldset>
@ -65,7 +59,7 @@ function update_value(element) {
<div class="pure-control-group">
<label for="volume"><%= translate(locale, "Player volume: ") %></label>
<input name="volume" id="volume" oninput="update_value(this);" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>">
<input name="volume" id="volume" data-onrange="update_volume_value" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>">
<span class="pure-form-message-inline" id="volume-value"><%= preferences.volume %></span>
</div>
@ -205,7 +199,7 @@ function update_value(element) {
<% # Web notifications are only supported over HTTPS %>
<% if Kemal.config.ssl || config.https_only %>
<div class="pure-control-group">
<a href="#" onclick="Notification.requestPermission()"><%= translate(locale, "Enable web notifications") %></a>
<a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a>
</div>
<% end %>
<% end %>
@ -233,11 +227,6 @@ function update_value(element) {
<% end %>
</div>
<div class="pure-control-group">
<label for="top_enabled"><%= translate(locale, "Top enabled: ") %></label>
<input name="top_enabled" id="top_enabled" type="checkbox" <% if config.top_enabled %>checked<% end %>>
</div>
<div class="pure-control-group">
<label for="captcha_enabled"><%= translate(locale, "CAPTCHA enabled: ") %></label>
<input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if config.captcha_enabled %>checked<% end %>>

18
src/invidious/views/search.ecr
View file

@ -2,6 +2,24 @@
<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title>
<% end %>
<div class="pure-g h-box v-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page - 1 %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count >= 20 %>
<a href="/search?q=<%= HTML.escape(query.not_nil!) %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
</div>
</div>
<div class="pure-g">
<% videos.each_slice(4) do |slice| %>
<% slice.each do |item| %>

33
src/invidious/views/subscription_manager.ecr
View file

@ -37,9 +37,9 @@
<div class="pure-u-2-5"></div>
<div class="pure-u-1-5" style="text-align:right">
<h3 style="padding-right:0.5em">
<form onsubmit="return false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post">
<form data-onsubmit="return_false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<a onclick="remove_subscription(this)" data-ucid="<%= channel.id %>" href="#">
<a data-onclick="remove_subscription" data-ucid="<%= channel.id %>" href="#">
<input style="all:unset" type="submit" value="<%= translate(locale, "unsubscribe") %>">
</a>
</form>
@ -52,32 +52,3 @@
<% end %>
</div>
<% end %>
<script>
function remove_subscription(target) {
var row = target.parentNode.parentNode.parentNode.parentNode.parentNode;
row.style.display = 'none';
var count = document.getElementById('count');
count.innerText = count.innerText - 1;
var url = '/subscription_ajax?action_remove_subscriptions=1&redirect=false' +
'&referer=<%= env.get("current_page") %>' +
'&c=' + target.getAttribute('data-ucid');
var xhr = new XMLHttpRequest();
xhr.responseType = 'json';
xhr.timeout = 10000;
xhr.open('POST', url, true);
xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
xhr.onreadystatechange = function() {
if (xhr.readyState == 4) {
if (xhr.status != 200) {
count.innerText = parseInt(count.innerText) + 1;
row.style.display = '';
}
}
}
xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>');
}
</script>

10
src/invidious/views/subscriptions.ecr
View file

@ -45,10 +45,12 @@
<hr>
</div>
<script>
var watched_data = {
csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>',
}
<script id="watched_data" type="application/json">
<%=
{
"csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
}.to_pretty_json
%>
</script>
<script src="/js/watched_widget.js"></script>

19
src/invidious/views/template.ecr
View file

@ -111,7 +111,7 @@
<div class="footer">
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-3">
<a href="https://github.com/omarroth/invidious">
<a href="https://github.com/iv-org/invidious">
<%= translate(locale, "Released under the AGPLv3 by Omar Roth.") %>
</a>
</div>
@ -140,23 +140,24 @@
</div>
<div class="pure-u-1 pure-u-md-1-3">
<i class="icon ion-logo-github"></i>
<%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %>
<i class="icon ion-logo-github"></i>
<%= CURRENT_BRANCH %>
<%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>
</div>
</div>
</div>
</div>
<div class="pure-u-1 pure-u-md-2-24"></div>
</div>
<script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script>
<% if env.get? "user" %>
<script src="/js/sse.js?v=<%= ASSET_COMMIT %>"></script>
<script>
var notification_data = {
upload_text: '<%= HTML.escape(translate(locale, "`x` uploaded a video")) %>',
live_upload_text: '<%= HTML.escape(translate(locale, "`x` is live")) %>',
}
<script id="notification_data" type="application/json">
<%=
{
"upload_text" => HTML.escape(translate(locale, "`x` uploaded a video")),
"live_upload_text" => HTML.escape(translate(locale, "`x` is live"))
}.to_pretty_json
%>
</script>
<script src="/js/notifications.js?v=<%= ASSET_COMMIT %>"></script>
<% end %>

33
src/invidious/views/token_manager.ecr
View file

@ -29,9 +29,9 @@
</div>
<div class="pure-u-1-5" style="text-align:right">
<h3 style="padding-right:0.5em">
<form onsubmit="return false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post">
<form data-onsubmit="return_false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post">
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
<a onclick="revoke_token(this)" data-session="<%= token[:session] %>" href="#">
<a data-onclick="revoke_token" data-session="<%= token[:session] %>" href="#">
<input style="all:unset" type="submit" value="<%= translate(locale, "revoke") %>">
</a>
</form>
@ -44,32 +44,3 @@
<% end %>
</div>
<% end %>
<script>
function revoke_token(target) {
var row = target.parentNode.parentNode.parentNode.parentNode.parentNode;
row.style.display = 'none';
var count = document.getElementById('count');
count.innerText = count.innerText - 1;
var url = '/token_ajax?action_revoke_token=1&redirect=false' +
'&referer=<%= env.get("current_page") %>' +
'&session=' + target.getAttribute('data-session');
var xhr = new XMLHttpRequest();
xhr.responseType = 'json';
xhr.timeout = 10000;
xhr.open('POST', url, true);
xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
xhr.onreadystatechange = function() {
if (xhr.readyState == 4) {
if (xhr.status != 200) {
count.innerText = parseInt(count.innerText) + 1;
row.style.display = '';
}
}
}
xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>');
}
</script>

20
src/invidious/views/top.ecr
View file

@ -1,20 +0,0 @@
<% content_for "header" do %>
<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>">
<title>
<% if env.get("preferences").as(Preferences).default_home != "Top" %>
<%= translate(locale, "Top") %> - Invidious
<% else %>
Invidious
<% end %>
</title>
<% end %>
<%= rendered "components/feed_menu" %>
<div class="pure-g">
<% top_videos.each_slice(4) do |slice| %>
<% slice.each do |item| %>
<%= rendered "components/item" %>
<% end %>
<% end %>
</div>

18
src/invidious/views/view_all_playlists.ecr
View file

@ -6,7 +6,7 @@
<div class="pure-g h-box">
<div class="pure-u-2-3">
<h3><%= translate(locale, "`x` playlists", %(<span id="count">#{items.size}</span>)) %></h3>
<h3><%= translate(locale, "`x` created playlists", %(<span id="count">#{items_created.size}</span>)) %></h3>
</div>
<div class="pure-u-1-3" style="text-align:right">
<h3>
@ -16,7 +16,21 @@
</div>
<div class="pure-g">
<% items.each_slice(4) do |slice| %>
<% items_created.each_slice(4) do |slice| %>
<% slice.each do |item| %>
<%= rendered "components/item" %>
<% end %>
<% end %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1">
<h3><%= translate(locale, "`x` saved playlists", %(<span id="count">#{items_saved.size}</span>)) %></h3>
</div>
</div>
<div class="pure-g">
<% items_saved.each_slice(4) do |slice| %>
<% slice.each do |item| %>
<%= rendered "components/item" %>
<% end %>

148
src/invidious/views/watch.ecr
View file

@ -3,47 +3,49 @@
<meta name="description" content="<%= HTML.escape(video.short_description) %>">
<meta name="keywords" content="<%= video.keywords.join(",") %>">
<meta property="og:site_name" content="Invidious">
<meta property="og:url" content="<%= host_url %>/watch?v=<%= video.id %>">
<meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
<meta property="og:title" content="<%= HTML.escape(video.title) %>">
<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg">
<meta property="og:description" content="<%= HTML.escape(video.short_description) %>">
<meta property="og:description" content="<%= video.short_description %>">
<meta property="og:type" content="video.other">
<meta property="og:video:url" content="<%= host_url %>/embed/<%= video.id %>">
<meta property="og:video:secure_url" content="<%= host_url %>/embed/<%= video.id %>">
<meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta property="og:video:secure_url" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta property="og:video:type" content="text/html">
<meta property="og:video:width" content="1280">
<meta property="og:video:height" content="720">
<meta name="twitter:card" content="player">
<meta name="twitter:site" content="@omarroth1">
<meta name="twitter:url" content="<%= host_url %>/watch?v=<%= video.id %>">
<meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
<meta name="twitter:title" content="<%= HTML.escape(video.title) %>">
<meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>">
<meta name="twitter:image" content="<%= host_url %>/vi/<%= video.id %>/maxres.jpg">
<meta name="twitter:player" content="<%= host_url %>/embed/<%= video.id %>">
<meta name="twitter:description" content="<%= video.short_description %>">
<meta name="twitter:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg">
<meta name="twitter:player" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta name="twitter:player:width" content="1280">
<meta name="twitter:player:height" content="720">
<%= rendered "components/player_sources" %>
<title><%= HTML.escape(video.title) %> - Invidious</title>
<% end %>
<script>
var video_data = {
id: '<%= video.id %>',
index: '<%= continuation %>',
plid: '<%= plid %>',
length_seconds: <%= video.length_seconds.to_f %>,
play_next: <%= !rvs.empty? && !plid && params.continue %>,
next_video: '<%= rvs.select { |rv| rv["id"]? }[0]?.try &.["id"] %>',
youtube_comments_text: '<%= HTML.escape(translate(locale, "View YouTube comments")) %>',
reddit_comments_text: '<%= HTML.escape(translate(locale, "View Reddit comments")) %>',
reddit_permalink_text: '<%= HTML.escape(translate(locale, "View more comments on Reddit")) %>',
comments_text: '<%= HTML.escape(translate(locale, "View `x` comments", "{commentCount}")) %>',
hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>',
show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>',
params: <%= params.to_json %>,
preferences: <%= preferences.to_json %>,
premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %>
}
<script id="video_data" type="application/json">
<%=
{
"id" => video.id,
"index" => continuation,
"plid" => plid,
"length_seconds" => video.length_seconds.to_f,
"play_next" => !video.related_videos.empty? && !plid && params.continue,
"next_video" => video.related_videos.select { |rv| rv["id"]? }[0]?.try &.["id"],
"youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")),
"reddit_comments_text" => HTML.escape(translate(locale, "View Reddit comments")),
"reddit_permalink_text" => HTML.escape(translate(locale, "View more comments on Reddit")),
"comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")),
"hide_replies_text" => HTML.escape(translate(locale, "Hide replies")),
"show_replies_text" => HTML.escape(translate(locale, "Show replies")),
"params" => params,
"preferences" => preferences,
"premiere_timestamp" => video.premiere_timestamp.try &.to_unix
}.to_pretty_json
%>
</script>
<div id="player-container" class="h-box">
@ -70,13 +72,13 @@ var video_data = {
</h3>
<% end %>
<% if !reason.empty? %>
<% if video.reason %>
<h3>
<%= reason %>
<%= video.reason %>
</h3>
<% elsif video.premiere_timestamp %>
<% elsif video.premiere_timestamp.try &.> Time.utc %>
<h3>
<%= translate(locale, "Premieres in `x`", recode_date((video.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
<%= video.premiere_timestamp.try { |t| translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %>
</h3>
<% end %>
</div>
@ -84,10 +86,10 @@ var video_data = {
<div class="pure-g">
<div class="pure-u-1 pure-u-lg-1-5">
<div class="h-box">
<span>
<span id="watch-on-youtube">
<a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a>
</span>
<p>
<p id="annotations">
<% if params.annotations %>
<a href="/watch?<%= env.params.query %>&iv_load_policy=3">
<%= translate(locale, "Hide annotations") %>
@ -99,26 +101,54 @@ var video_data = {
<% end %>
</p>
<% if user %>
<% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %>
<% if !playlists.empty? %>
<form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post">
<div class="pure-control-group">
<label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label>
<select style="width:100%" name="playlist_id" id="playlist_id">
<% playlists.each do |plid, title| %>
<option data-plid="<%= plid %>" value="<%= plid %>"><%= title %></option>
<% end %>
</select>
</div>
<button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary">
<b><%= translate(locale, "Add to playlist") %></b>
</button>
</form>
<script id="playlist_data" type="application/json">
<%=
{
"csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "")
}.to_pretty_json
%>
</script>
<script src="/js/playlist_widget.js?v=<%= Time.utc.to_unix_ms %>"></script>
<% end %>
<% end %>
<% if CONFIG.dmca_content.includes?(video.id) || CONFIG.disabled?("downloads") %>
<p><%= translate(locale, "Download is disabled.") %></p>
<p id="download"><%= translate(locale, "Download is disabled.") %></p>
<% else %>
<form class="pure-form pure-form-stacked" action="/latest_version" method="get" rel="noopener" target="_blank">
<div class="pure-control-group">
<label for="download_widget"><%= translate(locale, "Download as: ") %></label>
<select style="width:100%" name="download_widget" id="download_widget">
<% fmt_stream.each do |option| %>
<option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'>
<%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["type"].split(";")[0] %>
<option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'>
<%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["mimeType"].as_s.split(";")[0] %>
</option>
<% end %>
<% video_streams.each do |option| %>
<option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'>
<%= option["quality_label"] %> - <%= option["type"].split(";")[0] %> @ <%= option["fps"] %>fps - video only
<option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'>
<%= option["qualityLabel"] %> - <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["fps"] %>fps - video only
</option>
<% end %>
<% audio_streams.each do |option| %>
<option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'>
<%= option["type"].split(";")[0] %> @ <%= option["bitrate"] %>k - audio only
<option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'>
<%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["bitrate"]?.try &.as_i./ 1000 %>k - audio only
</option>
<% end %>
<% captions.each do |caption| %>
@ -135,23 +165,23 @@ var video_data = {
</form>
<% end %>
<p><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
<p><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
<p><i class="icon ion-ios-thumbs-down"></i> <%= number_with_separator(video.dislikes) %></p>
<p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
<p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
<p id="dislikes"><i class="icon ion-ios-thumbs-down"></i> <%= number_with_separator(video.dislikes) %></p>
<p id="genre"><%= translate(locale, "Genre: ") %>
<% if video.genre_url.empty? %>
<% if !video.genre_url %>
<%= video.genre %>
<% else %>
<a href="<%= video.genre_url %>"><%= video.genre %></a>
<% end %>
</p>
<% if !video.license.empty? %>
<% if video.license %>
<p id="license"><%= translate(locale, "License: ") %><%= video.license %></p>
<% end %>
<p id="family_friendly"><%= translate(locale, "Family friendly? ") %><%= translate_bool(locale, video.is_family_friendly) %></p>
<p id="wilson"><%= translate(locale, "Wilson score: ") %><%= video.wilson_score.round(4) %></p>
<p id="rating"><%= translate(locale, "Rating: ") %><%= rating.round(4) %> / 5</p>
<p id="engagement"><%= translate(locale, "Engagement: ") %><%= engagement.round(2) %>%</p>
<p id="wilson"><%= translate(locale, "Wilson score: ") %><%= video.wilson_score %></p>
<p id="rating"><%= translate(locale, "Rating: ") %><%= video.average_rating %> / 5</p>
<p id="engagement"><%= translate(locale, "Engagement: ") %><%= video.engagement %>%</p>
<% if video.allowed_regions.size != REGIONS.size %>
<p id="allowed_regions">
<% if video.allowed_regions.size < REGIONS.size // 2 %>
@ -168,8 +198,10 @@ var video_data = {
<div class="h-box">
<a href="/channel/<%= video.ucid %>" style="display:block;width:fit-content;width:-moz-fit-content">
<div class="channel-profile">
<img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>">
<span><%= video.author %></span>
<% if !video.author_thumbnail.empty? %>
<img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>">
<% end %>
<span id="channel-name"><%= video.author %></span>
</div>
</a>
@ -178,9 +210,9 @@ var video_data = {
<% sub_count_text = video.sub_count_text %>
<%= rendered "components/subscribe_widget" %>
<p>
<% if video.premiere_timestamp %>
<b><%= translate(locale, "Premieres `x`", video.premiere_timestamp.not_nil!.to_s("%B %-d, %R UTC")) %></b>
<p id="published-date">
<% if video.premiere_timestamp.try &.> Time.utc %>
<b><%= video.premiere_timestamp.try { |t| translate(locale, "Premieres `x`", t.to_s("%B %-d, %R UTC")) } %></b>
<% else %>
<b><%= translate(locale, "Shared `x`", video.published.to_s("%B %-d, %Y")) %></b>
<% end %>
@ -214,7 +246,7 @@ var video_data = {
<% if params.related_videos %>
<div class="h-box">
<% if !rvs.empty? %>
<% if !video.related_videos.empty? %>
<div <% if plid %>style="display:none"<% end %>>
<div class="pure-control-group">
<label for="continue"><%= translate(locale, "Play next by default: ") %></label>
@ -224,7 +256,7 @@ var video_data = {
</div>
<% end %>
<% rvs.each do |rv| %>
<% video.related_videos.each do |rv| %>
<% if rv["id"]? %>
<a href="/watch?v=<%= rv["id"] %>">
<% if !env.get("preferences").as(Preferences).thin_mode %>
@ -237,15 +269,17 @@ var video_data = {
<h5 class="pure-g">
<div class="pure-u-14-24">
<% if rv["ucid"]? %>
<b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"] %></a></b>
<b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %></a></b>
<% else %>
<b style="width:100%"><%= rv["author"] %></b>
<b style="width:100%"><%= rv["author"]? %></b>
<% end %>
</div>
<div class="pure-u-10-24" style="text-align:right">
<% if views = rv["short_view_count_text"]?.try &.delete(", views watching") %>
<b class="width:100%"><%= translate(locale, "`x` views", views) %></b>
<% if !views.empty? %>
<b class="width:100%"><%= translate(locale, "`x` views", views) %></b>
<% end %>
<% end %>
</div>
</h5>