mirror of
https://github.com/iv-org/invidious.git
synced 2025-08-12 16:15:47 -04:00
Merge branch 'iv-org:master' into master
This commit is contained in:
Aural Glow
GitHub
commit
77d4fd390a
79 changed files with 339 additions and 1026 deletions
|
|
@ -58,11 +58,10 @@ end
|
|||
alias IV = Invidious
|
||||
|
||||
CONFIG = Config.load
|
||||
HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
|
||||
HMAC_KEY = CONFIG.hmac_key
|
||||
|
||||
PG_DB = DB.open CONFIG.database_url
|
||||
ARCHIVE_URL = URI.parse("https://archive.org")
|
||||
LOGIN_URL = URI.parse("https://accounts.google.com")
|
||||
PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
|
||||
REDDIT_URL = URI.parse("https://www.reddit.com")
|
||||
YT_URL = URI.parse("https://www.youtube.com")
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
|
|||
case sort_by
|
||||
when "newest" then 1_i64
|
||||
when "popular" then 2_i64
|
||||
when "oldest" then 3_i64 # Broken as of 10/2022 :c
|
||||
when "oldest" then 4_i64
|
||||
else 1_i64 # Fallback to "newest"
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class Config
|
|||
# Used to tell Invidious it is behind a proxy, so links to resources should be https://
|
||||
property https_only : Bool?
|
||||
# HMAC signing key for CSRF tokens and verifying pubsub subscriptions
|
||||
property hmac_key : String?
|
||||
property hmac_key : String = ""
|
||||
# Domain to be used for links to resources on the site where an absolute URL is required
|
||||
property domain : String?
|
||||
# Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
|
||||
|
|
@ -206,6 +206,16 @@ class Config
|
|||
end
|
||||
{% end %}
|
||||
|
||||
# HMAC_key is mandatory
|
||||
# See: https://github.com/iv-org/invidious/issues/3854
|
||||
if config.hmac_key.empty?
|
||||
puts "Config: 'hmac_key' is required/can't be empty"
|
||||
exit(1)
|
||||
elsif config.hmac_key == "CHANGE_ME!!"
|
||||
puts "Config: The value of 'hmac_key' needs to be changed!!"
|
||||
exit(1)
|
||||
end
|
||||
|
||||
# Build database_url from db.* if it's not set directly
|
||||
if config.database_url.to_s.empty?
|
||||
if db = config.db
|
||||
|
|
@ -218,7 +228,7 @@ class Config
|
|||
path: db.dbname,
|
||||
)
|
||||
else
|
||||
puts "Config : Either database_url or db.* is required"
|
||||
puts "Config: Either database_url or db.* is required"
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -22,31 +22,6 @@ struct Annotation
|
|||
property annotations : String
|
||||
end
|
||||
|
||||
def login_req(f_req)
|
||||
data = {
|
||||
# Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard
|
||||
# Generally this is much longer (>1250 characters), see also
|
||||
# https://github.com/ytdl-org/youtube-dl/commit/baf67a604d912722b0fe03a40e9dc5349a2208cb .
|
||||
# For now this can be empty.
|
||||
"bgRequest" => %|["identifier",""]|,
|
||||
"pstMsg" => "1",
|
||||
"checkConnection" => "youtube",
|
||||
"checkedDomains" => "youtube",
|
||||
"hl" => "en",
|
||||
"deviceinfo" => %|[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]|,
|
||||
"f.req" => f_req,
|
||||
"flowName" => "GlifWebSignIn",
|
||||
"flowEntry" => "ServiceLogin",
|
||||
# "cookiesDisabled" => "false",
|
||||
# "gmscoreversion" => "undefined",
|
||||
# "continue" => "https://accounts.google.com/ManageAccount",
|
||||
# "azt" => "",
|
||||
# "bgHash" => "",
|
||||
}
|
||||
|
||||
return HTTP::Params.encode(data)
|
||||
end
|
||||
|
||||
def html_to_content(description_html : String)
|
||||
description = description_html.gsub(/(<br>)|(<br\/>)/, {
|
||||
"<br>": "\n",
|
||||
|
|
|
|||
|
|
@ -111,24 +111,27 @@ def decode_date(string : String)
|
|||
else nil # Continue
|
||||
end
|
||||
|
||||
# String matches format "20 hours ago", "4 months ago"...
|
||||
date = string.split(" ")[-3, 3]
|
||||
delta = date[0].to_i
|
||||
# String matches format "20 hours ago", "4 months ago", "20s ago", "15min ago"...
|
||||
match = string.match(/(?<count>\d+) ?(?<span>[smhdwy]\w*) ago/)
|
||||
|
||||
case date[1]
|
||||
when .includes? "second"
|
||||
raise "Could not parse #{string}" if match.nil?
|
||||
|
||||
delta = match["count"].to_i
|
||||
|
||||
case match["span"]
|
||||
when .starts_with? "s" # second(s)
|
||||
delta = delta.seconds
|
||||
when .includes? "minute"
|
||||
when .starts_with? "mi" # minute(s)
|
||||
delta = delta.minutes
|
||||
when .includes? "hour"
|
||||
when .starts_with? "h" # hour(s)
|
||||
delta = delta.hours
|
||||
when .includes? "day"
|
||||
when .starts_with? "d" # day(s)
|
||||
delta = delta.days
|
||||
when .includes? "week"
|
||||
when .starts_with? "w" # week(s)
|
||||
delta = delta.weeks
|
||||
when .includes? "month"
|
||||
when .starts_with? "mo" # month(s)
|
||||
delta = delta.months
|
||||
when .includes? "year"
|
||||
when .starts_with? "y" # year(s)
|
||||
delta = delta.years
|
||||
else
|
||||
raise "Could not parse #{string}"
|
||||
|
|
@ -437,7 +440,7 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
|
|||
# - https://github.com/iv-org/invidious/issues/3062
|
||||
text = %(<a href="#{url}">#{text}</a>)
|
||||
else
|
||||
text = %(<a href="#{url}">#{reduce_uri(url)}</a>)
|
||||
text = %(<a href="#{url}">#{reduce_uri(text)}</a>)
|
||||
end
|
||||
end
|
||||
return text
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ module Invidious::Jobs
|
|||
JOBS = [] of BaseJob
|
||||
|
||||
# Automatically generate a structure that wraps the various
|
||||
# jobs' configs, so that the follwing YAML config can be used:
|
||||
# jobs' configs, so that the following YAML config can be used:
|
||||
#
|
||||
# jobs:
|
||||
# job_name:
|
||||
|
|
|
|||
|
|
@ -42,11 +42,6 @@ module Invidious::Routes::Account
|
|||
sid = sid.as(String)
|
||||
token = env.params.body["csrf_token"]?
|
||||
|
||||
# We don't store passwords for Google accounts
|
||||
if !user.password
|
||||
return error_template(400, "Cannot change password for Google accounts")
|
||||
end
|
||||
|
||||
begin
|
||||
validate_request(token, sid, env.request, HMAC_KEY, locale)
|
||||
rescue ex
|
||||
|
|
@ -54,7 +49,7 @@ module Invidious::Routes::Account
|
|||
end
|
||||
|
||||
password = env.params.body["password"]?
|
||||
if !password
|
||||
if password.nil? || password.empty?
|
||||
return error_template(401, "Password is a required field")
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -178,10 +178,6 @@ module Invidious::Routes::API::V1::Authenticated
|
|||
Invidious::Database::Users.subscribe_channel(user, ucid)
|
||||
end
|
||||
|
||||
# For Google accounts, access tokens don't have enough information to
|
||||
# make a request on the user's behalf, which is why we don't sync with
|
||||
# YouTube.
|
||||
|
||||
env.response.status_code = 204
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -80,49 +80,23 @@ module Invidious::Routes::BeforeAll
|
|||
raise "Cannot use token as SID"
|
||||
end
|
||||
|
||||
# Invidious users only have SID
|
||||
if !env.request.cookies.has_key? "SSID"
|
||||
if email = Invidious::Database::SessionIDs.select_email(sid)
|
||||
user = Invidious::Database::Users.select!(email: email)
|
||||
csrf_token = generate_response(sid, {
|
||||
":authorize_token",
|
||||
":playlist_ajax",
|
||||
":signout",
|
||||
":subscription_ajax",
|
||||
":token_ajax",
|
||||
":watch_ajax",
|
||||
}, HMAC_KEY, 1.week)
|
||||
if email = Database::SessionIDs.select_email(sid)
|
||||
user = Database::Users.select!(email: email)
|
||||
csrf_token = generate_response(sid, {
|
||||
":authorize_token",
|
||||
":playlist_ajax",
|
||||
":signout",
|
||||
":subscription_ajax",
|
||||
":token_ajax",
|
||||
":watch_ajax",
|
||||
}, HMAC_KEY, 1.week)
|
||||
|
||||
preferences = user.preferences
|
||||
env.set "preferences", preferences
|
||||
preferences = user.preferences
|
||||
env.set "preferences", preferences
|
||||
|
||||
env.set "sid", sid
|
||||
env.set "csrf_token", csrf_token
|
||||
env.set "user", user
|
||||
end
|
||||
else
|
||||
headers = HTTP::Headers.new
|
||||
headers["Cookie"] = env.request.headers["Cookie"]
|
||||
|
||||
begin
|
||||
user, sid = get_user(sid, headers, false)
|
||||
csrf_token = generate_response(sid, {
|
||||
":authorize_token",
|
||||
":playlist_ajax",
|
||||
":signout",
|
||||
":subscription_ajax",
|
||||
":token_ajax",
|
||||
":watch_ajax",
|
||||
}, HMAC_KEY, 1.week)
|
||||
|
||||
preferences = user.preferences
|
||||
env.set "preferences", preferences
|
||||
|
||||
env.set "sid", sid
|
||||
env.set "csrf_token", csrf_token
|
||||
env.set "user", user
|
||||
rescue ex
|
||||
end
|
||||
env.set "sid", sid
|
||||
env.set "csrf_token", csrf_token
|
||||
env.set "user", user
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -83,10 +83,6 @@ module Invidious::Routes::Feeds
|
|||
headers = HTTP::Headers.new
|
||||
headers["Cookie"] = env.request.headers["Cookie"]
|
||||
|
||||
if !user.password
|
||||
user, sid = get_user(sid, headers)
|
||||
end
|
||||
|
||||
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
|
||||
max_results ||= user.preferences.max_results
|
||||
max_results ||= CONFIG.default_user_preferences.max_results
|
||||
|
|
|
|||
|
|
@ -24,9 +24,6 @@ module Invidious::Routes::Login
|
|||
captcha_type = env.params.query["captcha"]?
|
||||
captcha_type ||= "image"
|
||||
|
||||
tfa = env.params.query["tfa"]?
|
||||
prompt = nil
|
||||
|
||||
templated "user/login"
|
||||
end
|
||||
|
||||
|
|
@ -47,283 +44,18 @@ module Invidious::Routes::Login
|
|||
account_type ||= "invidious"
|
||||
|
||||
case account_type
|
||||
when "google"
|
||||
tfa_code = env.params.body["tfa"]?.try &.lchop("G-")
|
||||
traceback = IO::Memory.new
|
||||
|
||||
# See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
|
||||
begin
|
||||
client = nil # Declare variable
|
||||
{% unless flag?(:disable_quic) %}
|
||||
client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL)
|
||||
{% else %}
|
||||
client = HTTP::Client.new(LOGIN_URL)
|
||||
{% end %}
|
||||
|
||||
headers = HTTP::Headers.new
|
||||
|
||||
login_page = client.get("/ServiceLogin")
|
||||
headers = login_page.cookies.add_request_headers(headers)
|
||||
|
||||
lookup_req = {
|
||||
email, nil, [] of String, nil, "US", nil, nil, 2, false, true,
|
||||
{nil, nil,
|
||||
{2, 1, nil, 1,
|
||||
"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
|
||||
nil, [] of String, 4},
|
||||
1,
|
||||
{nil, nil, [] of String},
|
||||
nil, nil, nil, true,
|
||||
},
|
||||
email,
|
||||
}.to_json
|
||||
|
||||
traceback << "Getting lookup..."
|
||||
|
||||
headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
|
||||
headers["Google-Accounts-XSRF"] = "1"
|
||||
|
||||
response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
|
||||
lookup_results = JSON.parse(response.body[5..-1])
|
||||
|
||||
traceback << "done, returned #{response.status_code}.<br/>"
|
||||
|
||||
user_hash = lookup_results[0][2]
|
||||
|
||||
if token = env.params.body["token"]?
|
||||
answer = env.params.body["answer"]?
|
||||
captcha = {token, answer}
|
||||
else
|
||||
captcha = nil
|
||||
end
|
||||
|
||||
challenge_req = {
|
||||
user_hash, nil, 1, nil,
|
||||
{1, nil, nil, nil,
|
||||
{password, captcha, true},
|
||||
},
|
||||
{nil, nil,
|
||||
{2, 1, nil, 1,
|
||||
"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
|
||||
nil, [] of String, 4},
|
||||
1,
|
||||
{nil, nil, [] of String},
|
||||
nil, nil, nil, true,
|
||||
},
|
||||
}.to_json
|
||||
|
||||
traceback << "Getting challenge..."
|
||||
|
||||
response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req))
|
||||
headers = response.cookies.add_request_headers(headers)
|
||||
challenge_results = JSON.parse(response.body[5..-1])
|
||||
|
||||
traceback << "done, returned #{response.status_code}.<br/>"
|
||||
|
||||
headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
|
||||
|
||||
if challenge_results[0][3]?.try &.== 7
|
||||
return error_template(423, "Account has temporarily been disabled")
|
||||
end
|
||||
|
||||
if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s
|
||||
account_type = "google"
|
||||
captcha_type = "image"
|
||||
prompt = nil
|
||||
tfa = tfa_code
|
||||
captcha = {tokens: [token], question: ""}
|
||||
|
||||
return templated "user/login"
|
||||
end
|
||||
|
||||
if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
|
||||
return error_template(401, "Incorrect password")
|
||||
end
|
||||
|
||||
prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]?
|
||||
if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type
|
||||
traceback << "Handling prompt #{prompt_type}.<br/>"
|
||||
case prompt_type
|
||||
when "TWO_STEP_VERIFICATION"
|
||||
prompt_type = 2
|
||||
else # "LOGIN_CHALLENGE"
|
||||
prompt_type = 4
|
||||
end
|
||||
|
||||
# Prefer Authenticator app and SMS over unsupported protocols
|
||||
if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2
|
||||
tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0]
|
||||
|
||||
traceback << "Selecting challenge #{tfa[8]}..."
|
||||
select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json
|
||||
|
||||
tl = challenge_results[1][2]
|
||||
|
||||
tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body
|
||||
tfa = tfa[5..-1]
|
||||
tfa = JSON.parse(tfa)[0][-1]
|
||||
|
||||
traceback << "done.<br/>"
|
||||
else
|
||||
traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.<br/>"
|
||||
tfa = challenge_results[0][-1][0][0]
|
||||
end
|
||||
|
||||
if tfa[5] == "QUOTA_EXCEEDED"
|
||||
return error_template(423, "Quota exceeded, try again in a few hours")
|
||||
end
|
||||
|
||||
if !tfa_code
|
||||
account_type = "google"
|
||||
captcha_type = "image"
|
||||
|
||||
case tfa[8]
|
||||
when 6, 9
|
||||
prompt = "Google verification code"
|
||||
when 12
|
||||
prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
|
||||
when 15
|
||||
prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
|
||||
else
|
||||
prompt = "Google verification code"
|
||||
end
|
||||
|
||||
tfa = nil
|
||||
captcha = nil
|
||||
return templated "user/login"
|
||||
end
|
||||
|
||||
tl = challenge_results[1][2]
|
||||
|
||||
request_type = tfa[8]
|
||||
case request_type
|
||||
when 6 # Authenticator app
|
||||
tfa_req = {
|
||||
user_hash, nil, 2, nil,
|
||||
{6, nil, nil, nil, nil,
|
||||
{tfa_code, false},
|
||||
},
|
||||
}.to_json
|
||||
when 9 # Voice or text message
|
||||
tfa_req = {
|
||||
user_hash, nil, 2, nil,
|
||||
{9, nil, nil, nil, nil, nil, nil, nil,
|
||||
{nil, tfa_code, false, 2},
|
||||
},
|
||||
}.to_json
|
||||
when 12 # Recovery email
|
||||
tfa_req = {
|
||||
user_hash, nil, 4, nil,
|
||||
{12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
|
||||
{tfa_code},
|
||||
},
|
||||
}.to_json
|
||||
when 15 # Security question
|
||||
tfa_req = {
|
||||
user_hash, nil, 5, nil,
|
||||
{15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
|
||||
{tfa_code},
|
||||
},
|
||||
}.to_json
|
||||
else
|
||||
return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.")
|
||||
end
|
||||
|
||||
traceback << "Submitting challenge..."
|
||||
|
||||
response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req))
|
||||
headers = response.cookies.add_request_headers(headers)
|
||||
challenge_results = JSON.parse(response.body[5..-1])
|
||||
|
||||
if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") ||
|
||||
(challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT")
|
||||
return error_template(401, "Invalid TFA code")
|
||||
end
|
||||
|
||||
traceback << "done.<br/>"
|
||||
end
|
||||
|
||||
traceback << "Logging in..."
|
||||
|
||||
location = URI.parse(challenge_results[0][-1][2].to_s)
|
||||
cookies = HTTP::Cookies.from_client_headers(headers)
|
||||
|
||||
headers.delete("Content-Type")
|
||||
headers.delete("Google-Accounts-XSRF")
|
||||
|
||||
loop do
|
||||
if !location || location.path == "/ManageAccount"
|
||||
break
|
||||
end
|
||||
|
||||
# Occasionally there will be a second page after login confirming
|
||||
# the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle.
|
||||
|
||||
if location.path.starts_with? "/b/0/SmsAuthInterstitial"
|
||||
traceback << "Unhandled dialog /b/0/SmsAuthInterstitial."
|
||||
end
|
||||
|
||||
login = client.get(location.request_target, headers)
|
||||
|
||||
headers = login.cookies.add_request_headers(headers)
|
||||
location = login.headers["Location"]?.try { |u| URI.parse(u) }
|
||||
end
|
||||
|
||||
cookies = HTTP::Cookies.from_client_headers(headers)
|
||||
sid = cookies["SID"]?.try &.value
|
||||
if !sid
|
||||
raise "Couldn't get SID."
|
||||
end
|
||||
|
||||
user, sid = get_user(sid, headers)
|
||||
|
||||
# We are now logged in
|
||||
traceback << "done.<br/>"
|
||||
|
||||
host = URI.parse(env.request.headers["Host"]).host
|
||||
|
||||
cookies.each do |cookie|
|
||||
cookie.secure = Invidious::User::Cookies::SECURE
|
||||
|
||||
if cookie.extension
|
||||
cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
|
||||
cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "")
|
||||
end
|
||||
env.response.cookies << cookie
|
||||
end
|
||||
|
||||
if env.request.cookies["PREFS"]?
|
||||
user.preferences = env.get("preferences").as(Preferences)
|
||||
Invidious::Database::Users.update_preferences(user)
|
||||
|
||||
cookie = env.request.cookies["PREFS"]
|
||||
cookie.expires = Time.utc(1990, 1, 1)
|
||||
env.response.cookies << cookie
|
||||
end
|
||||
|
||||
env.redirect referer
|
||||
rescue ex
|
||||
traceback.rewind
|
||||
# error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.")
|
||||
error_message = %(#{ex.message}<br/>Traceback:<br/><div style="padding-left:2em" id="traceback">#{traceback.gets_to_end}</div>)
|
||||
return error_template(500, error_message)
|
||||
end
|
||||
when "invidious"
|
||||
if !email
|
||||
if email.nil? || email.empty?
|
||||
return error_template(401, "User ID is a required field")
|
||||
end
|
||||
|
||||
if !password
|
||||
if password.nil? || password.empty?
|
||||
return error_template(401, "Password is a required field")
|
||||
end
|
||||
|
||||
user = Invidious::Database::Users.select(email: email)
|
||||
|
||||
if user
|
||||
if !user.password
|
||||
return error_template(400, "Please sign in using 'Log in with Google'")
|
||||
end
|
||||
|
||||
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
|
||||
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
|
||||
Invidious::Database::SessionIDs.insert(sid, email)
|
||||
|
|
@ -367,8 +99,6 @@ module Invidious::Routes::Login
|
|||
captcha_type ||= "image"
|
||||
|
||||
account_type = "invidious"
|
||||
tfa = false
|
||||
prompt = ""
|
||||
|
||||
if captcha_type == "image"
|
||||
captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
|
||||
|
|
@ -481,11 +211,4 @@ module Invidious::Routes::Login
|
|||
|
||||
env.redirect referer
|
||||
end
|
||||
|
||||
def self.captcha(env)
|
||||
headers = HTTP::Headers{":authority" => "accounts.google.com"}
|
||||
response = YT_POOL.client &.get(env.request.resource, headers)
|
||||
env.response.headers["Content-Type"] = response.headers["Content-Type"]
|
||||
response.body
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -24,50 +24,6 @@ module Invidious::Routes::Notifications
|
|||
|
||||
user = user.as(User)
|
||||
|
||||
if !user.password
|
||||
channel_req = {} of String => String
|
||||
|
||||
channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
|
||||
channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
|
||||
channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
|
||||
|
||||
channel_req.reject! { |k, v| v != "true" && v != "false" }
|
||||
|
||||
headers = HTTP::Headers.new
|
||||
headers["Cookie"] = env.request.headers["Cookie"]
|
||||
|
||||
html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
|
||||
|
||||
cookies = HTTP::Cookies.from_client_headers(headers)
|
||||
html.cookies.each do |cookie|
|
||||
if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
|
||||
if cookies[cookie.name]?
|
||||
cookies[cookie.name] = cookie
|
||||
else
|
||||
cookies << cookie
|
||||
end
|
||||
end
|
||||
end
|
||||
headers = cookies.add_request_headers(headers)
|
||||
|
||||
if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
|
||||
session_token = match["session_token"]
|
||||
else
|
||||
return env.redirect referer
|
||||
end
|
||||
|
||||
headers["content-type"] = "application/x-www-form-urlencoded"
|
||||
channel_req["session_token"] = session_token
|
||||
|
||||
subs = XML.parse_html(html.body)
|
||||
subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel|
|
||||
channel_id = channel.content.lstrip("/channel/").not_nil!
|
||||
channel_req["channel_id"] = channel_id
|
||||
|
||||
YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
|
||||
end
|
||||
end
|
||||
|
||||
if redirect
|
||||
env.redirect referer
|
||||
else
|
||||
|
|
|
|||
|
|
@ -320,10 +320,6 @@ module Invidious::Routes::Playlists
|
|||
end
|
||||
end
|
||||
|
||||
if !user.password
|
||||
# TODO: Playlist stub, sync with YouTube for Google accounts
|
||||
# playlist_ajax(playlist_id, action, env.request.headers)
|
||||
end
|
||||
email = user.email
|
||||
|
||||
case action
|
||||
|
|
@ -410,8 +406,8 @@ module Invidious::Routes::Playlists
|
|||
return error_template(500, ex)
|
||||
end
|
||||
|
||||
page_count = (playlist.video_count / 100).to_i
|
||||
page_count += 1 if (playlist.video_count % 100) > 0
|
||||
page_count = (playlist.video_count / 200).to_i
|
||||
page_count += 1 if (playlist.video_count % 200) > 0
|
||||
|
||||
if page > page_count
|
||||
return env.redirect "/playlist?list=#{plid}&page=#{page_count}"
|
||||
|
|
@ -422,7 +418,7 @@ module Invidious::Routes::Playlists
|
|||
end
|
||||
|
||||
begin
|
||||
videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
|
||||
videos = get_playlist_videos(playlist, offset: (page - 1) * 200)
|
||||
rescue ex
|
||||
return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
|
||||
end
|
||||
|
|
|
|||
|
|
@ -43,11 +43,6 @@ module Invidious::Routes::Subscriptions
|
|||
channel_id = env.params.query["c"]?
|
||||
channel_id ||= ""
|
||||
|
||||
if !user.password
|
||||
# Sync subscriptions with YouTube
|
||||
subscribe_ajax(channel_id, action, env.request.headers)
|
||||
end
|
||||
|
||||
case action
|
||||
when "action_create_subscription_to_channel"
|
||||
if !user.subscriptions.includes? channel_id
|
||||
|
|
@ -82,14 +77,6 @@ module Invidious::Routes::Subscriptions
|
|||
user = user.as(User)
|
||||
sid = sid.as(String)
|
||||
|
||||
if !user.password
|
||||
# Refresh account
|
||||
headers = HTTP::Headers.new
|
||||
headers["Cookie"] = env.request.headers["Cookie"]
|
||||
|
||||
user, sid = get_user(sid, headers)
|
||||
end
|
||||
|
||||
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
|
||||
action_takeout ||= 0
|
||||
action_takeout = action_takeout == 1
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@ module Invidious::Routing
|
|||
get "/login", Routes::Login, :login_page
|
||||
post "/login", Routes::Login, :login
|
||||
post "/signout", Routes::Login, :signout
|
||||
get "/Captcha", Routes::Login, :captcha
|
||||
|
||||
# User preferences
|
||||
get "/preferences", Routes::PreferencesRoute, :show
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ struct Invidious::User
|
|||
|
||||
# Parse a youtube CSV subscription file
|
||||
def parse_subscription_export_csv(csv_content : String)
|
||||
rows = CSV.new(csv_content, headers: true)
|
||||
rows = CSV.new(csv_content.strip('\n'), headers: true)
|
||||
subscriptions = Array(String).new
|
||||
|
||||
# Counter to limit the amount of imports.
|
||||
|
|
@ -32,10 +32,10 @@ struct Invidious::User
|
|||
|
||||
def parse_playlist_export_csv(user : User, raw_input : String)
|
||||
# Split the input into head and body content
|
||||
raw_head, raw_body = raw_input.split("\n\n", limit: 2, remove_empty: true)
|
||||
raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
|
||||
|
||||
# Create the playlist from the head content
|
||||
csv_head = CSV.new(raw_head, headers: true)
|
||||
csv_head = CSV.new(raw_head.strip('\n'), headers: true)
|
||||
csv_head.next
|
||||
title = csv_head[4]
|
||||
description = csv_head[5]
|
||||
|
|
@ -51,7 +51,7 @@ struct Invidious::User
|
|||
Invidious::Database::Playlists.update_description(playlist.id, description)
|
||||
|
||||
# Add each video to the playlist from the body content
|
||||
csv_body = CSV.new(raw_body, headers: true)
|
||||
csv_body = CSV.new(raw_body.strip('\n'), headers: true)
|
||||
csv_body.each do |row|
|
||||
video_id = row[0]
|
||||
if playlist
|
||||
|
|
|
|||
|
|
@ -3,75 +3,6 @@ require "crypto/bcrypt/password"
|
|||
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
|
||||
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
|
||||
|
||||
def get_user(sid, headers, refresh = true)
|
||||
if email = Invidious::Database::SessionIDs.select_email(sid)
|
||||
user = Invidious::Database::Users.select!(email: email)
|
||||
|
||||
if refresh && Time.utc - user.updated > 1.minute
|
||||
user, sid = fetch_user(sid, headers)
|
||||
|
||||
Invidious::Database::Users.insert(user, update_on_conflict: true)
|
||||
Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
|
||||
|
||||
begin
|
||||
view_name = "subscriptions_#{sha256(user.email)}"
|
||||
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
|
||||
rescue ex
|
||||
end
|
||||
end
|
||||
else
|
||||
user, sid = fetch_user(sid, headers)
|
||||
|
||||
Invidious::Database::Users.insert(user, update_on_conflict: true)
|
||||
Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
|
||||
|
||||
begin
|
||||
view_name = "subscriptions_#{sha256(user.email)}"
|
||||
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
|
||||
rescue ex
|
||||
end
|
||||
end
|
||||
|
||||
return user, sid
|
||||
end
|
||||
|
||||
def fetch_user(sid, headers)
|
||||
feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
|
||||
feed = XML.parse_html(feed.body)
|
||||
|
||||
channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
|
||||
if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
|
||||
nil
|
||||
else
|
||||
channel["href"].lstrip("/channel/")
|
||||
end
|
||||
end
|
||||
|
||||
channels = get_batch_channels(channels)
|
||||
|
||||
email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
|
||||
if email
|
||||
email = email.content.strip
|
||||
else
|
||||
email = ""
|
||||
end
|
||||
|
||||
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
|
||||
|
||||
user = Invidious::User.new({
|
||||
updated: Time.utc,
|
||||
notifications: [] of String,
|
||||
subscriptions: channels,
|
||||
email: email,
|
||||
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
|
||||
password: nil,
|
||||
token: token,
|
||||
watched: [] of String,
|
||||
feed_needs_update: true,
|
||||
})
|
||||
return user, sid
|
||||
end
|
||||
|
||||
def create_user(sid, email, password)
|
||||
password = Crypto::Bcrypt::Password.create(password, cost: 10)
|
||||
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
|
||||
|
|
@ -91,38 +22,6 @@ def create_user(sid, email, password)
|
|||
return user, sid
|
||||
end
|
||||
|
||||
def subscribe_ajax(channel_id, action, env_headers)
|
||||
headers = HTTP::Headers.new
|
||||
headers["Cookie"] = env_headers["Cookie"]
|
||||
|
||||
html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
|
||||
|
||||
cookies = HTTP::Cookies.from_client_headers(headers)
|
||||
html.cookies.each do |cookie|
|
||||
if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
|
||||
if cookies[cookie.name]?
|
||||
cookies[cookie.name] = cookie
|
||||
else
|
||||
cookies << cookie
|
||||
end
|
||||
end
|
||||
end
|
||||
headers = cookies.add_request_headers(headers)
|
||||
|
||||
if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/)
|
||||
session_token = match["session_token"]
|
||||
|
||||
headers["content-type"] = "application/x-www-form-urlencoded"
|
||||
|
||||
post_req = {
|
||||
session_token: session_token,
|
||||
}
|
||||
post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"
|
||||
|
||||
YT_POOL.client &.post(post_url, headers, form: post_req)
|
||||
end
|
||||
end
|
||||
|
||||
def get_subscription_feed(user, max_results = 40, page = 1)
|
||||
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
|
||||
offset = (page - 1) * limit
|
||||
|
|
|
|||
|
|
@ -394,7 +394,9 @@ def fetch_video(id, region)
|
|||
if reason = info["reason"]?
|
||||
if reason == "Video unavailable"
|
||||
raise NotFoundException.new(reason.as_s || "")
|
||||
else
|
||||
elsif !reason.as_s.starts_with? "Premieres"
|
||||
# dont error when it's a premiere.
|
||||
# we already parsed most of the data and display the premiere date
|
||||
raise InfoException.new(reason.as_s || "")
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -78,7 +78,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
|
|||
elsif video_id != player_response.dig("videoDetails", "videoId")
|
||||
# YouTube may return a different video player response than expected.
|
||||
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
|
||||
raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)")
|
||||
# Line to be reverted if one day we solve the video not available issue.
|
||||
return {
|
||||
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
|
||||
"reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"),
|
||||
}
|
||||
else
|
||||
reason = nil
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<form class="pure-form" action="/search" method="get">
|
||||
<fieldset>
|
||||
<input type="search" id="searchbox" autocomplete="off" autocorrect="off"
|
||||
<input type="search" id="searchbox" autocorrect="off"
|
||||
autocapitalize="none" spellcheck="false" <% if autofocus %>autofocus<% end %>
|
||||
name="q" placeholder="<%= translate(locale, "search") %>"
|
||||
title="<%= translate(locale, "search") %>"
|
||||
|
|
|
|||
|
|
@ -16,12 +16,11 @@
|
|||
<li>a list of channel UCIDs the user is subscribed to</li>
|
||||
<li>a user ID (for persistent storage of subscriptions and preferences)</li>
|
||||
<li>a json object containing user preferences</li>
|
||||
<li>a hashed password if applicable (not present on google accounts)</li>
|
||||
<li>a hashed password</li>
|
||||
<li>a randomly generated token for providing an RSS feed of a user's subscriptions</li>
|
||||
<li>a list of video IDs identifying watched videos</li>
|
||||
</ul>
|
||||
<p>Users can clear their watch history using the <a href="/clear_watch_history">clear watch history</a> page.</p>
|
||||
<p>If a user is logged in with a Google account, no password will ever be stored. This website uses the session token provided by Google to identify a user, but does not store the information required to make requests on a user's behalf without their knowledge or consent.</p>
|
||||
|
||||
<h3>Data you passively provide</h3>
|
||||
<p>When you request any resource from this website (for example: a page, a font, an image, or an API endpoint) information about the request may be logged.</p>
|
||||
|
|
|
|||
|
|
@ -7,42 +7,6 @@
|
|||
<div class="pure-u-1 pure-u-lg-3-5">
|
||||
<div class="h-box">
|
||||
<% case account_type when %>
|
||||
<% when "google" %>
|
||||
<form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post">
|
||||
<fieldset>
|
||||
<% if email %>
|
||||
<input name="email" type="hidden" value="<%= HTML.escape(email) %>">
|
||||
<% else %>
|
||||
<label for="email"><%= translate(locale, "E-mail") %> :</label>
|
||||
<input required class="pure-input-1" name="email" type="email" placeholder="<%= translate(locale, "E-mail") %>">
|
||||
<% end %>
|
||||
|
||||
<% if password %>
|
||||
<input name="password" type="hidden" value="<%= HTML.escape(password) %>">
|
||||
<% else %>
|
||||
<label for="password"><%= translate(locale, "Password") %> :</label>
|
||||
<input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>">
|
||||
<% end %>
|
||||
|
||||
<% if prompt %>
|
||||
<label for="tfa"><%= translate(locale, prompt) %> :</label>
|
||||
<input required class="pure-input-1" name="tfa" type="text" placeholder="<%= translate(locale, prompt) %>">
|
||||
<% end %>
|
||||
|
||||
<% if tfa %>
|
||||
<input type="hidden" name="tfa" value="<%= tfa %>">
|
||||
<% end %>
|
||||
|
||||
<% if captcha %>
|
||||
<img style="width:50%" src="/Captcha?v=2&ctoken=<%= captcha[:tokens][0] %>"/>
|
||||
<input type="hidden" name="token" value="<%= captcha[:tokens][0] %>">
|
||||
<label for="answer"><%= translate(locale, "Answer") %> :</label>
|
||||
<input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>">
|
||||
<% end %>
|
||||
|
||||
<button type="submit" class="pure-button pure-button-primary"><%= translate(locale, "Sign In") %></button>
|
||||
</fieldset>
|
||||
</form>
|
||||
<% else # "invidious" %>
|
||||
<form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post">
|
||||
<fieldset>
|
||||
|
|
|
|||
|
|
@ -8,13 +8,15 @@
|
|||
|
||||
def add_yt_headers(request)
|
||||
if request.headers["User-Agent"] == "Crystal"
|
||||
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
|
||||
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
|
||||
end
|
||||
|
||||
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
|
||||
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
|
||||
|
||||
# Preserve original cookies and add new YT consent cookie for EU servers
|
||||
request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+"
|
||||
request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
|
||||
if !CONFIG.cookies.empty?
|
||||
request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -381,7 +381,7 @@ private module Parsers
|
|||
# Parses an InnerTube itemSectionRenderer into a SearchVideo.
|
||||
# Returns nil when the given object isn't a ItemSectionRenderer
|
||||
#
|
||||
# A itemSectionRenderer seems to be a simple wrapper for a videoRenderer, used
|
||||
# A itemSectionRenderer seems to be a simple wrapper for a videoRenderer or a playlistRenderer, used
|
||||
# by the result page for channel searches. It is located inside a continuationItems
|
||||
# container.It is very similar to RichItemRendererParser
|
||||
#
|
||||
|
|
@ -394,6 +394,8 @@ private module Parsers
|
|||
|
||||
private def self.parse(item_contents, author_fallback)
|
||||
child = VideoRendererParser.process(item_contents, author_fallback)
|
||||
child ||= PlaylistRendererParser.process(item_contents, author_fallback)
|
||||
|
||||
return child
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -7,16 +7,18 @@ module YoutubeAPI
|
|||
|
||||
private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
||||
|
||||
private ANDROID_APP_VERSION = "17.33.42"
|
||||
private ANDROID_APP_VERSION = "18.20.38"
|
||||
# github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308
|
||||
private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US) gzip"
|
||||
private ANDROID_USER_AGENT = "com.google.android.youtube/18.20.38 (Linux; U; Android 12; US) gzip"
|
||||
private ANDROID_SDK_VERSION = 31_i64
|
||||
private ANDROID_VERSION = "12"
|
||||
private IOS_APP_VERSION = "17.33.2"
|
||||
|
||||
private IOS_APP_VERSION = "18.21.3"
|
||||
# github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330
|
||||
private IOS_USER_AGENT = "com.google.ios.youtube/17.33.2 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)"
|
||||
private IOS_USER_AGENT = "com.google.ios.youtube/18.21.3 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)"
|
||||
# github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224
|
||||
private IOS_VERSION = "15.6.0.19G71"
|
||||
private IOS_VERSION = "15.6.0.19G71"
|
||||
|
||||
private WINDOWS_VERSION = "10.0"
|
||||
|
||||
# Enumerate used to select one of the clients supported by the API
|
||||
|
|
@ -43,7 +45,7 @@ module YoutubeAPI
|
|||
ClientType::Web => {
|
||||
name: "WEB",
|
||||
name_proto: "1",
|
||||
version: "2.20221118.01.00",
|
||||
version: "2.20230602.01.00",
|
||||
api_key: DEFAULT_API_KEY,
|
||||
screen: "WATCH_FULL_SCREEN",
|
||||
os_name: "Windows",
|
||||
|
|
@ -63,7 +65,7 @@ module YoutubeAPI
|
|||
ClientType::WebMobile => {
|
||||
name: "MWEB",
|
||||
name_proto: "2",
|
||||
version: "2.20220805.01.00",
|
||||
version: "2.20230531.05.00",
|
||||
api_key: DEFAULT_API_KEY,
|
||||
os_name: "Android",
|
||||
os_version: ANDROID_VERSION,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue