Git-Mirrors/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-08-02 03:26:17 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' into stream-sort

Browse source
This commit is contained in:
src-tinkerer 2024-03-26 12:19:45 +00:00 • committed by GitHub
commit 72fe8af850
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
84 changed files with 1343 additions and 1015 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/invidious/channels/about.cr
View file

@ -18,8 +18,8 @@ record AboutChannel,
def get_about_info(ucid, locale) : AboutChannel
begin
# "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==")
# Fetch channel information from channel home page
initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end

2
src/invidious/channels/channels.cr
View file

@ -93,7 +93,7 @@ struct ChannelVideo
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map(&.name)}}
{{@type.instance_vars.map(&.name).splat}}
}
{% end %}
end

6
src/invidious/channels/videos.cr
View file

@ -73,12 +73,6 @@ def make_initial_content_ctoken(ucid, content_type, sort_by) : String
return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
end
# Used in bypass_captcha_job.cr
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
continuation = produce_channel_content_continuation(ucid, "videos", page, auto_generated, sort_by, v2)
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
module Invidious::Channel::Tabs
extend self

6
src/invidious/config.cr
View file

@ -48,7 +48,7 @@ struct ConfigPreferences
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
{{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}}
}
{% end %}
end
@ -133,10 +133,6 @@ class Config
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new
# Key for Anti-Captcha
property captcha_key : String? = nil
# API URL for Anti-Captcha
property captcha_api_url : String = "https://api.anti-captcha.com"
# Playlist length limit
property playlist_length_limit : Int32 = 500

4
src/invidious/database/statistics.cr
View file

@ -15,7 +15,7 @@ module Invidious::Database::Statistics
PG_DB.query_one(request, as: Int64)
end
def count_users_active_1m : Int64
def count_users_active_6m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '6 months'
@ -24,7 +24,7 @@ module Invidious::Database::Statistics
PG_DB.query_one(request, as: Int64)
end
def count_users_active_6m : Int64
def count_users_active_1m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '1 month'

2
src/invidious/frontend/comments_reddit.cr
View file

@ -33,7 +33,7 @@ module Invidious::Frontend::Comments
<a href="javascript:void(0)" data-onclick="toggle_parent">[ − ]</a>
<b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b>
#{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)}
<span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span>
<span title="#{child.created_utc.to_s("%a %B %-d %T %Y UTC")}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span>
<a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a>
</p>
<div>

30
src/invidious/frontend/comments_youtube.cr
View file

@ -107,6 +107,36 @@ module Invidious::Frontend::Comments
</div>
END_HTML
end
when "multiImage"
html << <<-END_HTML
<section class="carousel">
<a class="skip-link" href="#skip-#{child["commentId"]}">#{translate(locale, "carousel_skip")}</a>
<div class="slides">
END_HTML
image_array = attachment["images"].as_a
image_array.each_index do |i|
html << <<-END_HTML
<div class="slides-item slide-#{i + 1}" id="#{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_slide", {"current" => (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0">
<img loading="lazy" src="/ggpht#{URI.parse(image_array[i][1]["url"].as_s).request_target}" alt="" />
</div>
END_HTML
end
html << <<-END_HTML
</div>
<div class="carousel__nav">
END_HTML
attachment["images"].as_a.each_index do |i|
html << <<-END_HTML
<a class="slider-nav" href="##{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_go_to", (i + 1).to_s)}" tabindex="-1" aria-hidden="true">#{i + 1}</a>
END_HTML
end
html << <<-END_HTML
</div>
<div id="skip-#{child["commentId"]}"></div>
</section>
END_HTML
else nil # Ignore
end
end

6
src/invidious/helpers/errors.cr
View file

@ -3,7 +3,7 @@
# -------------------
macro error_template(*args)
error_template_helper(env, {{*args}})
error_template_helper(env, {{args.splat}})
end
def github_details(summary : String, content : String)
@ -95,7 +95,7 @@ end
# -------------------
macro error_atom(*args)
error_atom_helper(env, {{*args}})
error_atom_helper(env, {{args.splat}})
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
@ -121,7 +121,7 @@ end
# -------------------
macro error_json(*args)
error_json_helper(env, {{*args}})
error_json_helper(env, {{args.splat}})
end
def error_json_helper(

59
src/invidious/helpers/handlers.cr
View file

@ -142,63 +142,8 @@ class APIHandler < Kemal::Handler
exclude ["/api/v1/auth/notifications"], "POST"
def call(env)
return call_next env unless only_match? env
env.response.headers["Access-Control-Allow-Origin"] = "*"
# Since /api/v1/notifications is an event-stream, we don't want
# to wrap the response
return call_next env if exclude_match? env
# Here we swap out the socket IO so we can modify the response as needed
output = env.response.output
env.response.output = IO::Memory.new
begin
call_next env
env.response.output.rewind
if env.response.output.as(IO::Memory).size != 0 &&
env.response.headers.includes_word?("Content-Type", "application/json")
response = JSON.parse(env.response.output)
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
if env.params.query["pretty"]?.try &.== "1"
response = response.to_pretty_json
else
response = response.to_json
end
else
response = env.response.output.gets_to_end
end
rescue ex
env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html")
env.response.status_code = 500
if env.response.headers.includes_word?("Content-Type", "application/json")
response = {"error" => ex.message || "Unspecified error"}
if env.params.query["pretty"]?.try &.== "1"
response = response.to_pretty_json
else
response = response.to_json
end
end
ensure
env.response.output = output
env.response.print response
env.response.flush
end
env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env)
call_next env
end
end

44
src/invidious/helpers/helpers.cr
View file

@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel)
video.published = published
response = JSON.parse(video.to_json(locale, nil))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}"
env.response.puts
@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel)
Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}"
env.response.puts
@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel)
video.published = Time.unix(published)
response = JSON.parse(video.to_json(locale, nil))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}"
env.response.puts
@ -208,3 +181,20 @@ def proxy_file(response, env)
IO.copy response.body_io, env.response
end
end
# Fetch the playback requests tracker from the statistics endpoint.
#
# Creates a new tracker when unavailable.
def get_playback_statistic
if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
tracker = {
"totalRequests" => 0_i64,
"successfulRequests" => 0_i64,
"ratio" => 0_f64,
}
Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
end
return tracker.as(Hash(String, Int64 | Float64))
end

19
src/invidious/helpers/i18n.cr
View file

@ -78,7 +78,7 @@ def load_all_locales
return locales
end
def translate(locale : String?, key : String, text : String | Nil = nil) : String
def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String
# Log a warning if "key" doesn't exist in en-US locale and return
# that key as the text, so this is more or less transparent to the user.
if !LOCALES["en-US"].has_key?(key)
@ -101,10 +101,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin
match_length = 0
raw_data.as_h.each do |hash_key, value|
if md = text.try &.match(/#{hash_key}/)
if md[0].size >= match_length
translation = value.as_s
match_length = md[0].size
if text.is_a?(String)
if md = text.try &.match(/#{hash_key}/)
if md[0].size >= match_length
translation = value.as_s
match_length = md[0].size
end
end
end
end
@ -114,8 +116,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin
raise "Invalid translation \"#{raw_data}\""
end
if text
if text.is_a?(String)
translation = translation.gsub("`x`", text)
elsif text.is_a?(Hash(String, String))
# adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic
text.each_key do |hash_key|
translation = translation.gsub("{{#{hash_key}}}", text[hash_key])
end
end
return translation

22
src/invidious/helpers/i18next.cr
View file

@ -47,19 +47,19 @@ module I18next::Plurals
private PLURAL_SETS = {
PluralForms::Single_gt_one => [
"ach", "ak", "am", "arn", "br", "fil", "gun", "ln", "mfe", "mg",
"mi", "oc", "pt", "tg", "tl", "ti", "tr", "uz", "wa",
"ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg",
"mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa",
],
PluralForms::Single_not_one => [
"af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en",
"eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi",
"eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi",
"hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr",
"nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms",
"ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw",
"ta", "te", "tk", "ur", "yo",
],
PluralForms::None => [
"ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky",
"ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky",
"lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh",
],
PluralForms::Dual_Slavic => [
@ -90,11 +90,13 @@ module I18next::Plurals
"sk" => PluralForms::Special_Czech_Slovak,
"sl" => PluralForms::Special_Slovenian,
# Mixed v3/v4 rules
"fr" => PluralForms::Special_French_Portuguese,
"hr" => PluralForms::Special_Hungarian_Serbian,
"it" => PluralForms::Special_Spanish_Italian,
"pt-BR" => PluralForms::Special_French_Portuguese,
"sr" => PluralForms::Special_Hungarian_Serbian,
"es" => PluralForms::Special_Spanish_Italian,
"fr" => PluralForms::Special_French_Portuguese,
"hr" => PluralForms::Special_Hungarian_Serbian,
"it" => PluralForms::Special_Spanish_Italian,
"pt" => PluralForms::Special_French_Portuguese,
"pt" => PluralForms::Special_French_Portuguese,
"sr" => PluralForms::Special_Hungarian_Serbian,
}
# These are the v1 and v2 compatible suffixes.
@ -165,7 +167,7 @@ module I18next::Plurals
def get_plural_form(locale : String) : PluralForms
# Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code
if !locale.matches?(/^pt-BR$/)
if !locale.matches?(/^pt-PT$/)
locale = locale.split('-')[0]
end

248
src/invidious/helpers/json_filter.cr
View file

@ -1,248 +0,0 @@
module JSONFilter
alias BracketIndex = Hash(Int64, Int64)
alias GroupedFieldsValue = String | Array(GroupedFieldsValue)
alias GroupedFieldsList = Array(GroupedFieldsValue)
class FieldsParser
class ParseError < Exception
end
# Returns the `Regex` pattern used to match nest groups
def self.nest_group_pattern : Regex
# uses a '.' character to match json keys as they are allowed
# to contain any unicode codepoint
/(?:|,)(?<groupname>[^,\n]*?)\(/
end
# Returns the `Regex` pattern used to check if there are any empty nest groups
def self.unnamed_nest_group_pattern : Regex
/^\(|\(\(|\/\(/
end
def self.parse_fields(fields_text : String, &) : Nil
if fields_text.empty?
raise FieldsParser::ParseError.new "Fields is empty"
end
opening_bracket_count = fields_text.count('(')
closing_bracket_count = fields_text.count(')')
if opening_bracket_count != closing_bracket_count
bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing"
raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})"
elsif match_result = unnamed_nest_group_pattern.match(fields_text)
raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}"
end
# first, handle top-level single nested properties: items/id, playlistItems/snippet, etc
parse_single_nests(fields_text) { |nest_list| yield nest_list }
# next, handle nest groups: items(id, etag, etc)
parse_nest_groups(fields_text) { |nest_list| yield nest_list }
end
def self.parse_single_nests(fields_text : String, &) : Nil
single_nests = remove_nest_groups(fields_text)
if !single_nests.empty?
property_nests = single_nests.split(',')
property_nests.each do |nest|
nest_list = nest.split('/')
if nest_list.includes? ""
raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}"
end
yield nest_list
end
# else
# raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}"
end
end
def self.parse_nest_groups(fields_text : String, &) : Nil
nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
bracket_pairs = get_bracket_pairs(fields_text, true)
text_index = 0
regex_index = 0
while regex_result = self.nest_group_pattern.match(fields_text, regex_index)
raw_match = regex_result[0]
group_name = regex_result["groupname"]
text_index = regex_result.begin
regex_index = regex_result.end
if text_index.nil? || regex_index.nil?
raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}"
end
offset = raw_match.starts_with?(',') ? 1 : 0
opening_bracket_index = (text_index + group_name.size) + offset
closing_bracket_index = bracket_pairs[opening_bracket_index]
content_start = opening_bracket_index + 1
content = fields_text[content_start...closing_bracket_index]
if content.empty?
raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}"
else
content = remove_nest_groups(content)
end
while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index]
if nest_stack.size
nest_stack.pop
end
end
group_name.split('/').each do |name|
nest_stack.push({
group_name: name,
closing_bracket_index: closing_bracket_index,
})
end
if !content.empty?
properties = content.split(',')
properties.each do |prop|
nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] }
if !prop.empty?
if prop.includes?('/')
parse_single_nests(prop) { |list| nest_list += list }
else
nest_list.push prop
end
else
raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}"
end
yield nest_list
end
end
end
end
def self.remove_nest_groups(text : String) : String
content_bracket_pairs = get_bracket_pairs(text, false)
content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket|
closing_bracket = content_bracket_pairs[opening_bracket]
last_comma = text.rindex(',', opening_bracket) || 0
text = text[0...last_comma] + text[closing_bracket + 1...text.size]
end
return text.starts_with?(',') ? text[1...text.size] : text
end
def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex
istart = [] of Int64
bracket_index = BracketIndex.new
text.each_char_with_index do |char, index|
if char == '('
istart.push(index.to_i64)
end
if char == ')'
begin
opening = istart.pop
if recursive || (!recursive && istart.size == 0)
bracket_index[opening] = index.to_i64
end
rescue
raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}"
end
end
end
if istart.size != 0
idx = istart.pop
raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}"
end
return bracket_index
end
end
class FieldsGrouper
alias SkeletonValue = Hash(String, SkeletonValue)
def self.create_json_skeleton(fields_text : String) : SkeletonValue
root_hash = {} of String => SkeletonValue
FieldsParser.parse_fields(fields_text) do |nest_list|
current_item = root_hash
nest_list.each do |key|
if current_item[key]?
current_item = current_item[key]
else
current_item[key] = {} of String => SkeletonValue
current_item = current_item[key]
end
end
end
root_hash
end
def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList
grouped_fields_list = GroupedFieldsList.new
json_skeleton.each do |key, value|
grouped_fields_list.push key
nested_keys = create_grouped_fields_list(value)
grouped_fields_list.push nested_keys unless nested_keys.empty?
end
return grouped_fields_list
end
end
class FilterError < Exception
end
def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true)
skeleton = FieldsGrouper.create_json_skeleton(fields_text)
grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton)
filter(item, grouped_fields_list, in_place)
end
def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any
item = item.clone unless in_place
if !item.as_h? && !item.as_a?
raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}"
end
top_level_keys = Array(String).new
grouped_fields_list.each do |value|
if value.is_a? String
top_level_keys.push value
elsif value.is_a? Array
if !top_level_keys.empty?
key_to_filter = top_level_keys.last
if item.as_h?
filter(item[key_to_filter], value, in_place: true)
elsif item.as_a?
item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) }
end
else
raise FilterError.new "Tried to filter while top level keys list is empty"
end
end
end
if item.as_h?
item.as_h.select! top_level_keys
elsif item.as_a?
item.as_a.map { |value| filter(value, top_level_keys, in_place: true) }
end
item
end
end

2
src/invidious/helpers/utils.cr
View file

@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.request_target
referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,0-9a-zA-Z]/, "").lstrip("/\\")
referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback

16
src/invidious/helpers/webvtt.cr
View file

@ -4,13 +4,23 @@
module WebVTT
# A WebVTT builder generates WebVTT files
private class Builder
# See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload
private ESCAPE_SUBSTITUTIONS = {
'&' => "&amp;",
'<' => "&lt;",
'>' => "&gt;",
'\u200E' => "&lrm;",
'\u200F' => "&rlm;",
'\u00A0' => "&nbsp;",
}
def initialize(@io : IO)
end
# Writes an vtt cue with the specified time stamp and contents
def cue(start_time : Time::Span, end_time : Time::Span, text : String)
timestamp(start_time, end_time)
@io << text
@io << self.escape(text)
@io << "\n\n"
end
@ -29,6 +39,10 @@ module WebVTT
@io << '.' << timestamp.milliseconds.to_s.rjust(3, '0')
end
private def escape(text : String) : String
return text.gsub(ESCAPE_SUBSTITUTIONS)
end
def document(setting_fields : Hash(String, String)? = nil, &)
@io << "WEBVTT\n"

135
src/invidious/jobs/bypass_captcha_job.cr
View file

@ -1,135 +0,0 @@
class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
def begin
loop do
begin
random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String})
if !random_video
random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"}
end
{"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
response = YT_POOL.client &.get(path)
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
headers = response.cookies.add_request_headers(HTTP::Headers.new)
response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => "https://www.youtube.com#{path}",
"websiteKey" => site_key,
"recaptchaDataSValue" => s_value,
},
}.to_json).body)
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
response.cookies
.select { |cookie| cookie.name != "PREF" }
.each { |cookie| CONFIG.cookies << cookie }
# Persist cookies between runs
File.write("config/config.yml", CONFIG.to_yaml)
elsif response.headers["Location"]?.try &.includes?("/sorry/index")
location = response.headers["Location"].try { |u| URI.parse(u) }
headers = HTTP::Headers{":authority" => location.host.not_nil!}
response = YT_POOL.client &.get(location.request_target, headers)
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="index"])).not_nil!
site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url))
captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
response = JSON.parse(captcha_client.post("/createTask",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
"recaptchaDataSValue" => s_value,
},
}.to_json).body)
captcha_client.close
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(captcha_client.post("/getTaskResult",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
headers = HTTP::Headers{
"Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
}
cookies = HTTP::Cookies.from_client_headers(headers)
cookies.each { |cookie| CONFIG.cookies << cookie }
# Persist cookies between runs
File.write("config/config.yml", CONFIG.to_yaml)
end
end
rescue ex
LOGGER.error("BypassCaptchaJob: #{ex.message}")
ensure
sleep 1.minute
Fiber.yield
end
end
end
end

16
src/invidious/jobs/statistics_refresh_job.cr
View file

@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
#
# "totalRequests" => 0_i64,
# "successfulRequests" => 0_i64
# "ratio" => 0_i64
#
"playback" => {} of String => Int64 | Float64,
}
private getter db : DB::Database
@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
loop do
refresh_stats
sleep 1.minute
sleep 10.minute
Fiber.yield
end
end
@ -49,12 +56,15 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
users["total"] = Invidious::Database::Statistics.count_users_total
users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m
users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m
users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m
users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m
STATISTICS["metadata"] = {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
# Reset playback requests tracker
STATISTICS["playback"] = {} of String => Int64 | Float64
end
end

18
src/invidious/routes/api/v1/misc.cr
View file

@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
# Calculate playback success rate
if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
tracker = tracker.as(Hash(String, Int64 | Float64))
if !tracker.empty?
total_requests = tracker["totalRequests"]
success_count = tracker["successfulRequests"]
if total_requests.zero?
tracker["ratio"] = 1_i64
else
tracker["ratio"] = (success_count / (total_requests)).round(2)
end
end
end
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
@ -175,6 +191,8 @@ module Invidious::Routes::API::V1::Misc
json.object do
json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
json.field "params", params.try &.as_s
json.field "pageType", pageType
end

7
src/invidious/routes/api/v1/search.cr
View file

@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search
begin
client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt"
client.before_request { |r| add_yt_headers(r) }
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
response = client.get(url).body
client.close
body = JSON.parse(response[5..-1]).as_a
body = JSON.parse(response[19..-2]).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|

43
src/invidious/routes/api/v1/videos.cr
View file

@ -363,4 +363,47 @@ module Invidious::Routes::API::V1::Videos
end
end
end
def self.clips(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
clip_id = env.params.url["id"]
region = env.params.query["region"]?
proxy = {"1", "true"}.any? &.== env.params.query["local"]?
response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
return error_json(400, "Invalid clip ID") if response["error"]?
video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
return error_json(400, "Invalid clip ID") if video_id.nil?
start_time = nil
end_time = nil
clip_title = nil
if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
start_time, end_time, clip_title = parse_clip_parameters(params)
end
begin
video = get_video(video_id, region: region)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
json.object do
json.field "startTime", start_time
json.field "endTime", end_time
json.field "clipTitle", clip_title
json.field "video" do
Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
end
end
end
end
end

23
src/invidious/routes/feeds.cr
View file

@ -407,14 +407,23 @@ module Invidious::Routes::Feeds
end
spawn do
rss = XML.parse_html(body)
rss.xpath_nodes("//feed/entry").each do |entry|
id = entry.xpath_node("videoid").not_nil!.content
author = entry.xpath_node("author/name").not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
# TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"default" => "http://www.w3.org/2005/Atom",
}
rss = XML.parse(body)
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
video = get_video(id, force_refresh: true)
begin
video = get_video(id, force_refresh: true)
rescue
next # skip this video since it raised an exception (e.g. it is a scheduled live event)
end
if CONFIG.enable_user_notifications
# Deliver notifications to `/api/v1/auth/notifications`

13
src/invidious/routes/video_playback.cr
View file

@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback
headers["Range"] = "bytes=#{range_for_head}"
end
client = make_client(URI.parse(host), region)
client = make_client(URI.parse(host), region, force_resolve = true)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
client = make_client(URI.parse(new_host), region)
client = make_client(URI.parse(new_host), region, force_resolve = true)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
client = make_client(URI.parse(host), region)
client = make_client(URI.parse(host), region, force_resolve = true)
rescue ex
error = ex.message
end
@ -80,9 +80,14 @@ module Invidious::Routes::VideoPlayback
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
playback_statistics = get_playback_statistic()
playback_statistics["totalRequests"] += 1
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
else
playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
@ -191,7 +196,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
client = make_client(URI.parse(host), region)
client = make_client(URI.parse(host), region, force_resolve = true)
end
end

6
src/invidious/routes/watch.cr
View file

@ -275,6 +275,12 @@ module Invidious::Routes::Watch
return error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
start_time, end_time, _ = parse_clip_parameters(params)
env.params.query["start"] = start_time.to_s if start_time != nil
env.params.query["end"] = end_time.to_s if end_time != nil
end
return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else
return error_template(404, "The requested clip doesn't exist")

1
src/invidious/routing.cr
View file

@ -235,6 +235,7 @@ module Invidious::Routing
get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
# Feeds
get "/api/v1/trending", {{namespace}}::Feeds, :trending

6
src/invidious/search/filters.cr
View file

@ -300,9 +300,9 @@ module Invidious::Search
object["9:varint"] = ((page - 1) * 20).to_i64
end
# If the object is empty, return an empty string,
# otherwise encode to protobuf then to base64
return "" if object.empty?
# Prevent censoring of self harm topics
# See https://github.com/iv-org/invidious/issues/4398
object["30:varint"] = 1.to_i64
return object
.try { |i| Protodec::Any.cast_json(i) }

18
src/invidious/videos.cr
View file

@ -227,8 +227,22 @@ struct Video
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
def dash_manifest_url
info.dig?("streamingData", "dashManifestUrl").try &.as_s
def dash_manifest_url : String?
raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
return nil if raw_dash_url.nil?
# Use manifest v5 parameter to reduce file size
# See https://github.com/iv-org/invidious/issues/4186
dash_url = URI.parse(raw_dash_url)
dash_query = dash_url.query || ""
if dash_query.empty?
dash_url.path = "#{dash_url.path}/mpd_version/5"
else
dash_url.query = "#{dash_query}&mpd_version=5"
end
return dash_url.to_s
end
def genre_url : String?

22
src/invidious/videos/clip.cr Normal file
View file

@ -0,0 +1,22 @@
require "json"
# returns start_time, end_time and clip_title
def parse_clip_parameters(params) : {Float64?, Float64?, String?}
decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
start_time = decoded_protobuf
.try(&.["50:0:embedded"]["2:1:varint"].as_i64)
.try { |i| i/1000 }
end_time = decoded_protobuf
.try(&.["50:0:embedded"]["3:2:varint"].as_i64)
.try { |i| i/1000 }
clip_title = decoded_protobuf
.try(&.["50:0:embedded"]["4:3:string"].as_s)
return start_time, end_time, clip_title
end

5
src/invidious/videos/parser.cr
View file

@ -78,6 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
# Line to be reverted if one day we solve the video not available issue.
# Although technically not a call to /videoplayback the fact that YouTube is returning the
# wrong video means that we should count it as a failure.
get_playback_statistic()["totalRequests"] += 1
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"),

36
src/invidious/views/components/item.ecr
View file

@ -82,11 +82,19 @@
</div>
<div class="video-card-row flexible">
<div class="flex-left"><a href="/channel/<%= item.ucid %>">
<p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
<%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
</p>
</a></div>
<div class="flex-left">
<% if !item.ucid.to_s.empty? %>
<a href="/channel/<%= item.ucid %>">
<p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
<%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
</p>
</a>
<% else %>
<p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
<%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
</p>
<% end %>
</div>
</div>
<% when Category %>
<% else %>
@ -160,11 +168,19 @@
</div>
<div class="video-card-row flexible">
<div class="flex-left"><a href="/channel/<%= item.ucid %>">
<p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
<%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
</p>
</a></div>
<div class="flex-left">
<% if !item.ucid.to_s.empty? %>
<a href="/channel/<%= item.ucid %>">
<p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
<%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
</p>
</a>
<% else %>
<p class="channel-name" dir="auto"><%= HTML.escape(item.author) %>
<%- if author_verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end -%>
</p>
<% end %>
</div>
<%= rendered "components/video-context-buttons" %>
</div>

26
src/invidious/views/template.ecr
View file

@ -1,5 +1,9 @@
<%
locale = env.get("preferences").as(Preferences).locale
dark_mode = env.get("preferences").as(Preferences).dark_mode
%>
<!DOCTYPE html>
<html lang="<%= env.get("preferences").as(Preferences).locale %>">
<html lang="<%= locale %>">
<head>
<meta charset="utf-8">
@ -17,19 +21,14 @@
<link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/carousel.css?v=<%= ASSET_COMMIT %>">
<script src="/js/_helpers.js?v=<%= ASSET_COMMIT %>"></script>
</head>
<%
locale = env.get("preferences").as(Preferences).locale
dark_mode = env.get("preferences").as(Preferences).dark_mode
%>
<body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme">
<span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span>
<span style="display:none" id="dark_mode_pref"><%= dark_mode %></span>
<div class="pure-g">
<div class="pure-u-1 pure-u-md-2-24"></div>
<div class="pure-u-1 pure-u-md-20-24" id="contents">
<div class="pure-u-1 pure-u-xl-20-24" id="contents">
<div class="pure-g navbar h-box">
<% if navbar_search %>
<div class="pure-u-1 pure-u-md-4-24">
@ -43,8 +42,8 @@
<div class="pure-u-1 pure-u-md-8-24 user-field">
<% if env.get? "user" %>
<div class="pure-u-1-4">
<a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading">
<% if env.get("preferences").as(Preferences).dark_mode == "dark" %>
<a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>">
<% if dark_mode == "dark" %>
<i class="icon ion-ios-sunny"></i>
<% else %>
<i class="icon ion-ios-moon"></i>
@ -81,8 +80,8 @@
</div>
<% else %>
<div class="pure-u-1-3">
<a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading">
<% if env.get("preferences").as(Preferences).dark_mode == "dark" %>
<a id="toggle_theme" href="/toggle_theme?referer=<%= env.get?("current_page") %>" class="pure-menu-heading" title="<%= translate(locale, "toggle_theme") %>">
<% if dark_mode == "dark" %>
<i class="icon ion-ios-sunny"></i>
<% else %>
<i class="icon ion-ios-moon"></i>
@ -156,7 +155,6 @@
</footer>
</div>
<div class="pure-u-1 pure-u-md-2-24"></div>
</div>
<script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script>

4
src/invidious/views/watch.ecr
View file

@ -118,7 +118,7 @@ we're going to need to do it here in order to allow for translations.
link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}")
if !plid.nil? && !continuation.nil?
link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]}
link_yt_param = URI::Params{"list" => [plid], "index" => [continuation.to_s]}
link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param)
link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param)
end
@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations.
<h5 class="pure-g">
<div class="pure-u-14-24">
<% if rv["ucid"]? %>
<% if !rv["ucid"].empty? %>
<b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b>
<% else %>
<b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %></b>

24
src/invidious/yt_backend/connection_pool.cr
View file

@ -1,7 +1,6 @@
def add_yt_headers(request)
if request.headers["User-Agent"] == "Crystal"
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
end
request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
@ -27,7 +26,7 @@ struct YoutubeConnectionPool
def client(region = nil, &block)
if region
conn = make_client(url, region)
conn = make_client(url, region, force_resolve = true)
response = yield conn
else
conn = pool.checkout
@ -37,7 +36,7 @@ struct YoutubeConnectionPool
conn.close
conn = HTTP::Client.new(url)
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
response = yield conn
@ -52,7 +51,7 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
conn = HTTP::Client.new(url)
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
conn
@ -60,9 +59,14 @@ struct YoutubeConnectionPool
end
end
def make_client(url : URI, region = nil)
def make_client(url : URI, region = nil, force_resolve : Bool = false)
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
# Some services do not support IPv6.
if force_resolve
client.family = CONFIG.force_resolve
end
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
@ -81,8 +85,8 @@ def make_client(url : URI, region = nil)
return client
end
def make_client(url : URI, region = nil, &block)
client = make_client(url, region)
def make_client(url : URI, region = nil, force_resolve : Bool = false, &block)
client = make_client(url, region, force_resolve)
begin
yield client
ensure

4
src/invidious/yt_backend/extractors.cr
View file

@ -822,9 +822,9 @@ module HelperExtractors
end
# Retrieves the ID required for querying the InnerTube browse endpoint.
# Raises when it's unable to do so
# Returns an empty string when it's unable to do so
def self.get_browse_id(container)
return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s
return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
end

33
src/invidious/yt_backend/youtube_api.cr
View file

@ -7,17 +7,18 @@ module YoutubeAPI
private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
private ANDROID_APP_VERSION = "18.20.38"
# github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308
private ANDROID_USER_AGENT = "com.google.android.youtube/18.20.38 (Linux; U; Android 12; US) gzip"
# For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
private ANDROID_APP_VERSION = "19.09.36"
private ANDROID_USER_AGENT = "com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip"
private ANDROID_SDK_VERSION = 31_i64
private ANDROID_VERSION = "12"
private IOS_APP_VERSION = "18.21.3"
# github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330
private IOS_USER_AGENT = "com.google.ios.youtube/18.21.3 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)"
# github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224
private IOS_VERSION = "15.6.0.19G71"
# For Apple device names, see https://gist.github.com/adamawolf/3048717
# For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
# then go to the dedicated article of the major version you want.
private IOS_APP_VERSION = "19.09.3"
private IOS_USER_AGENT = "com.google.ios.youtube/19.09.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)"
private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build
private WINDOWS_VERSION = "10.0"
@ -45,7 +46,7 @@ module YoutubeAPI
ClientType::Web => {
name: "WEB",
name_proto: "1",
version: "2.20230602.01.00",
version: "2.20240304.00.00",
api_key: DEFAULT_API_KEY,
screen: "WATCH_FULL_SCREEN",
os_name: "Windows",
@ -55,7 +56,7 @@ module YoutubeAPI
ClientType::WebEmbeddedPlayer => {
name: "WEB_EMBEDDED_PLAYER",
name_proto: "56",
version: "1.20220803.01.00",
version: "1.20240303.00.00",
api_key: DEFAULT_API_KEY,
screen: "EMBED",
os_name: "Windows",
@ -65,7 +66,7 @@ module YoutubeAPI
ClientType::WebMobile => {
name: "MWEB",
name_proto: "2",
version: "2.20230531.05.00",
version: "2.20240304.08.00",
api_key: DEFAULT_API_KEY,
os_name: "Android",
os_version: ANDROID_VERSION,
@ -74,7 +75,7 @@ module YoutubeAPI
ClientType::WebScreenEmbed => {
name: "WEB",
name_proto: "1",
version: "2.20220804.00.00",
version: "2.20240304.00.00",
api_key: DEFAULT_API_KEY,
screen: "EMBED",
os_name: "Windows",
@ -99,7 +100,7 @@ module YoutubeAPI
name: "ANDROID_EMBEDDED_PLAYER",
name_proto: "55",
version: ANDROID_APP_VERSION,
api_key: DEFAULT_API_KEY,
api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw",
},
ClientType::AndroidScreenEmbed => {
name: "ANDROID",
@ -143,9 +144,9 @@ module YoutubeAPI
ClientType::IOSMusic => {
name: "IOS_MUSIC",
name_proto: "26",
version: "5.21",
version: "6.42",
api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s",
user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)",
user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)",
device_make: "Apple",
device_model: "iPhone14,5",
os_name: "iPhone",
@ -158,7 +159,7 @@ module YoutubeAPI
ClientType::TvHtml5 => {
name: "TVHTML5",
name_proto: "7",
version: "7.20220325",
version: "7.20240304.10.00",
api_key: DEFAULT_API_KEY,
},
ClientType::TvHtml5ScreenEmbed => {