Merge branch 'master' into api-only

This commit is contained in:
Omar Roth 2018-11-17 17:42:19 -06:00
commit bc711de22c
7 changed files with 178 additions and 125 deletions

View File

@ -20,6 +20,7 @@
- Support for Reddit comments in place of YT comments
- Import/Export subscriptions, watch history, preferences
- Does not use any of the official YouTube APIs
- Developer [API](https://github.com/omarroth/invidious/wiki/API)
Liberapay: https://liberapay.com/omarroth
Patreon: https://patreon.com/omarroth

View File

@ -100,10 +100,11 @@ get "/api/v1/captions/:id" do |env|
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]?
client = make_client(YT_URL)
begin
video = fetch_video(id, proxies)
video = fetch_video(id, proxies, region)
rescue ex : VideoRedirect
next env.redirect "/api/v1/captions/#{ex.message}"
rescue ex
@ -332,9 +333,10 @@ get "/api/v1/videos/:id" do |env|
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]?
begin
video = fetch_video(id, proxies)
video = fetch_video(id, proxies, region)
rescue ex : VideoRedirect
next env.redirect "/api/v1/videos/#{ex.message}"
rescue ex
@ -570,6 +572,8 @@ get "/api/v1/channels/:ucid" do |env|
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "newest"
begin
author, ucid, auto_generated = get_about_info(ucid)
@ -580,7 +584,7 @@ get "/api/v1/channels/:ucid" do |env|
page = 1
begin
videos, count = get_60_videos(ucid, page, auto_generated)
videos, count = get_60_videos(ucid, page, auto_generated, sort_by)
rescue ex
error_message = {"error" => ex.message}.to_json
halt env, status_code: 500, response: error_message
@ -717,6 +721,8 @@ end
ucid = env.params.url["ucid"]
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "newest"
begin
author, ucid, auto_generated = get_about_info(ucid)
@ -726,7 +732,7 @@ end
end
begin
videos, count = get_60_videos(ucid, page, auto_generated)
videos, count = get_60_videos(ucid, page, auto_generated, sort_by)
rescue ex
error_message = {"error" => ex.message}.to_json
halt env, status_code: 500, response: error_message
@ -1178,10 +1184,11 @@ get "/api/manifest/dash/id/:id" do |env|
local = env.params.query["local"]?.try &.== "true"
id = env.params.url["id"]
region = env.params.query["region"]?
client = make_client(YT_URL)
begin
video = fetch_video(id, proxies)
video = fetch_video(id, proxies, region)
rescue ex : VideoRedirect
next env.redirect "/api/manifest/dash/id/#{ex.message}"
rescue ex

View File

@ -163,7 +163,7 @@ def fetch_channel(ucid, client, db, pull_all_videos = true)
return channel
end
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil)
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest")
if auto_generated
seed = Time.unix(1525757349)
@ -190,6 +190,16 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil)
meta += page.size.to_u8.unsafe_chr
meta += page
case sort_by
when "newest"
# Empty tags can be omitted
# meta += "\x18\x00"
when "popular"
meta += "\x18\x01"
when "oldest"
meta += "\x18\x02"
end
meta = Base64.urlsafe_encode(meta)
meta = URI.escape(meta)
@ -254,14 +264,14 @@ def get_about_info(ucid)
return {author, ucid, auto_generated, sub_count}
end
def get_60_videos(ucid, page, auto_generated)
def get_60_videos(ucid, page, auto_generated, sort_by = "newest")
count = 0
videos = [] of SearchVideo
client = make_client(YT_URL)
2.times do |i|
url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
response = client.get(url)
json = JSON.parse(response.body)

View File

@ -16,6 +16,7 @@ class Config
hmac_key: String?,
full_refresh: Bool,
geo_bypass: Bool,
domain: String?,
})
end
@ -36,55 +37,6 @@ def login_req(login_form, f_req)
return HTTP::Params.encode(data)
end
def generate_captcha(key)
minute = Random::Secure.rand(12)
minute_angle = minute * 30
minute = minute * 5
hour = Random::Secure.rand(12)
hour_angle = hour * 30 + minute_angle.to_f / 12
if hour == 0
hour = 12
end
clock_svg = <<-END_SVG
<svg viewBox="0 0 100 100" width="200px">
<circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle>
<text x="69" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text>
<text x="82.909" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text>
<text x="88" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text>
<text x="82.909" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text>
<text x="69" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text>
<text x="50" y="91" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text>
<text x="31" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text>
<text x="17.091" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text>
<text x="12" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text>
<text x="17.091" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text>
<text x="31" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text>
<text x="50" y="15" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text>
<circle cx="50" cy="50" r="3" fill="black"></circle>
<line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line>
<line id="hour" transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line>
</svg>
END_SVG
challenge = ""
convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true,
input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc|
challenge = proc.output.gets_to_end
challenge = Base64.strict_encode(challenge)
challenge = "data:image/png;base64,#{challenge}"
end
answer = "#{hour}:#{minute.to_s.rjust(2, '0')}"
token = OpenSSL::HMAC.digest(:sha256, key, answer)
token = Base64.urlsafe_encode(token)
return {challenge: challenge, token: token}
end
def html_to_content(description_html)
if !description_html
description = ""
@ -296,55 +248,3 @@ def extract_items(nodeset, ucid = nil)
return items
end
def create_response(user_id, operation, key, expire = 6.hours)
expire = Time.now + expire
nonce = Random::Secure.hex(4)
challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}"
token = OpenSSL::HMAC.digest(:sha256, key, challenge)
challenge = Base64.urlsafe_encode(challenge)
token = Base64.urlsafe_encode(token)
return challenge, token
end
def validate_response(challenge, token, user_id, operation, key)
if !challenge
raise "Hidden field \"challenge\" is a required field"
end
if !token
raise "Hidden field \"token\" is a required field"
end
challenge = Base64.decode_string(challenge)
if challenge.split("-").size == 4
expire, nonce, challenge_user_id, challenge_operation = challenge.split("-")
expire = expire.to_i?
expire ||= 0
else
raise "Invalid challenge"
end
challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge)
challenge = Base64.urlsafe_encode(challenge)
if challenge != token
raise "Invalid token"
end
if challenge_operation != operation
raise "Invalid token"
end
if challenge_user_id != user_id
raise "Invalid token"
end
if expire < Time.now.to_unix
raise "Token is expired, please try again"
end
end

View File

@ -18,16 +18,28 @@ def elapsed_text(elapsed)
"#{(millis * 1000).round(2)}µs"
end
def make_client(url)
def make_client(url, proxies = {} of String => Array({ip: String, port: Int32}), region = nil)
context = OpenSSL::SSL::Context::Client.new
context.add_options(
OpenSSL::SSL::Options::ALL |
OpenSSL::SSL::Options::NO_SSL_V2 |
OpenSSL::SSL::Options::NO_SSL_V3
)
client = HTTP::Client.new(url, context)
client = HTTPClient.new(url, context)
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
if region
proxies[region]?.try &.each do |proxy|
begin
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
client.set_proxy(proxy)
break
rescue ex
end
end
end
return client
end

View File

@ -70,7 +70,11 @@ class Preferences
JSON.mapping({
video_loop: Bool,
autoplay: Bool,
listen: {
continue: {
type: Bool,
default: false,
},
listen: {
type: Bool,
default: false,
},
@ -195,3 +199,112 @@ def create_user(sid, email, password)
return user
end
def create_response(user_id, operation, key, db, expire = 6.hours)
expire = Time.now + expire
nonce = Random::Secure.hex(16)
db.exec("INSERT INTO nonces VALUES ($1) ON CONFLICT DO NOTHING", nonce)
challenge = "#{expire.to_unix}-#{nonce}-#{user_id}-#{operation}"
token = OpenSSL::HMAC.digest(:sha256, key, challenge)
challenge = Base64.urlsafe_encode(challenge)
token = Base64.urlsafe_encode(token)
return challenge, token
end
def validate_response(challenge, token, user_id, operation, key, db)
if !challenge
raise "Hidden field \"challenge\" is a required field"
end
if !token
raise "Hidden field \"token\" is a required field"
end
challenge = Base64.decode_string(challenge)
if challenge.split("-").size == 4
expire, nonce, challenge_user_id, challenge_operation = challenge.split("-")
expire = expire.to_i?
expire ||= 0
else
raise "Invalid challenge"
end
challenge = OpenSSL::HMAC.digest(:sha256, HMAC_KEY, challenge)
challenge = Base64.urlsafe_encode(challenge)
if db.query_one?("SELECT EXISTS (SELECT true FROM nonces WHERE nonce = $1)", nonce, as: Bool)
db.exec("DELETE FROM nonces * WHERE nonce = $1", nonce)
else
raise "Invalid token"
end
if challenge != token
raise "Invalid token"
end
if challenge_operation != operation
raise "Invalid token"
end
if challenge_user_id != user_id
raise "Invalid user"
end
if expire < Time.now.to_unix
raise "Token is expired, please try again"
end
end
def generate_captcha(key, db)
minute = Random::Secure.rand(12)
minute_angle = minute * 30
minute = minute * 5
hour = Random::Secure.rand(12)
hour_angle = hour * 30 + minute_angle.to_f / 12
if hour == 0
hour = 12
end
clock_svg = <<-END_SVG
<svg viewBox="0 0 100 100" width="200px">
<circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle>
<text x="69" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text>
<text x="82.909" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text>
<text x="88" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text>
<text x="82.909" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text>
<text x="69" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text>
<text x="50" y="91" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text>
<text x="31" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text>
<text x="17.091" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text>
<text x="12" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text>
<text x="17.091" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text>
<text x="31" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text>
<text x="50" y="15" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text>
<circle cx="50" cy="50" r="3" fill="black"></circle>
<line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line>
<line id="hour" transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line>
</svg>
END_SVG
image = ""
convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true,
input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc|
image = proc.output.gets_to_end
image = Base64.strict_encode(image)
image = "data:image/png;base64,#{image}"
end
answer = "#{hour}:#{minute.to_s.rjust(2, '0')}"
answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer)
challenge, token = create_response(answer, "sign_in", key, db)
return {image: image, challenge: challenge, token: token}
end

View File

@ -507,14 +507,14 @@ end
class VideoRedirect < Exception
end
def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true)
def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true, region = nil)
if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool)
video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
# If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours)
if refresh && Time.now - video.updated > 10.minutes
begin
video = fetch_video(id, proxies)
video = fetch_video(id, proxies, region)
video_array = video.to_a
args = arg_array(video_array[1..-1], 2)
@ -529,7 +529,7 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32})
end
end
else
video = fetch_video(id, proxies)
video = fetch_video(id, proxies, region)
video_array = video.to_a
args = arg_array(video_array)
@ -540,12 +540,12 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32})
return video
end
def fetch_video(id, proxies)
def fetch_video(id, proxies, region)
html_channel = Channel(XML::Node | String).new
info_channel = Channel(HTTP::Params).new
spawn do
client = make_client(YT_URL)
client = make_client(YT_URL, proxies, region)
html = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
if md = html.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/)
@ -557,7 +557,7 @@ def fetch_video(id, proxies)
end
spawn do
client = make_client(YT_URL)
client = make_client(YT_URL, proxies, region)
info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
info = HTTP::Params.parse(info.body)
@ -628,9 +628,9 @@ def fetch_video(id, proxies)
end
proxy = {ip: proxy.proxy_host, port: proxy.proxy_port}
region = proxies.select { |region, list| list.includes? proxy }
if !region.empty?
info["region"] = region.keys[0]
region_proxies = proxies.select { |region, list| list.includes? proxy }
if !region_proxies.empty?
info["region"] = region_proxies.keys[0]
end
break
@ -677,7 +677,8 @@ def fetch_video(id, proxies)
wilson_score = ci_lower_bound(likes, likes + dislikes)
published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).not_nil!["content"]
published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"]
published ||= Time.now.to_s("%Y-%m-%d")
published = Time.parse(published, "%Y-%m-%d", Time::Location.local)
allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",")
@ -685,15 +686,16 @@ def fetch_video(id, proxies)
is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True"
is_family_friendly ||= true
genre = html.xpath_node(%q(//meta[@itemprop="genre"])).not_nil!["content"]
genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"]
genre ||= ""
genre_url = html.xpath_node(%(//a[text()="#{genre}"])).try &.["href"]
case genre
when "Movies"
genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g"
when "Education"
# Education channel is linked but does not exist
# genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g"
genre_url = ""
genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g"
end
genre_url ||= ""
@ -730,15 +732,19 @@ end
def process_video_params(query, preferences)
autoplay = query["autoplay"]?.try &.to_i?
continue = query["continue"]?.try &.to_i?
listen = query["listen"]? && (query["listen"] == "true" || query["listen"] == "1").to_unsafe
preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase }
quality = query["quality"]?
region = query["region"]?
speed = query["speed"]?.try &.to_f?
video_loop = query["loop"]?.try &.to_i?
volume = query["volume"]?.try &.to_i?
if preferences
# region ||= preferences.region
autoplay ||= preferences.autoplay.to_unsafe
continue ||= preferences.continue.to_unsafe
listen ||= preferences.listen.to_unsafe
preferred_captions ||= preferences.captions
quality ||= preferences.quality
@ -748,6 +754,7 @@ def process_video_params(query, preferences)
end
autoplay ||= 0
continue ||= 0
listen ||= 0
preferred_captions ||= [] of String
quality ||= "hd720"
@ -756,6 +763,7 @@ def process_video_params(query, preferences)
volume ||= 100
autoplay = autoplay == 1
continue = continue == 1
listen = listen == 1
video_loop = video_loop == 1
@ -786,11 +794,13 @@ def process_video_params(query, preferences)
params = {
autoplay: autoplay,
continue: continue,
controls: controls,
listen: listen,
preferred_captions: preferred_captions,
quality: quality,
raw: raw,
region: region,
speed: speed,
video_end: video_end,
video_loop: video_loop,