From 86d0de4b0e6d77fb1e5772815acf67639176c0cc Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 31 May 2019 10:29:17 -0500 Subject: [PATCH 001/146] Fix typo in post webhook --- src/invidious.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index e89448e4d..fa4ac44d1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2867,7 +2867,7 @@ post "/feed/webhook/:token" do |env| video_array = video.to_a args = arg_array(video_array) - PG_DB.exec("INSERT INTO channel_videos (id, title, published, updated, ucid, author, length_seconds, live_now, premiere_timestamp) VALUES (#{args}) \ + PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ updated = $4, ucid = $5, author = $6, length_seconds = $7, \ live_now = $8, premiere_timestamp = $9, views = $10", video_array) From 701b5ea56134f2c8d8e8f54036b9f6467095b47b Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 09:51:31 -0500 Subject: [PATCH 002/146] Remove watched videos from notifications --- src/invidious.cr | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index fa4ac44d1..de1349473 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -350,6 +350,7 @@ get "/watch" do |env| if user subscriptions = user.subscriptions watched = user.watched + notifications = user.notifications end subscriptions ||= [] of String @@ -377,6 +378,12 @@ get "/watch" do |env| PG_DB.exec("UPDATE users SET watched = watched || $1 WHERE email = $2", [id], user.as(User).email) end + if notifications && notifications.includes? id + PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + env.get("user").as(User).notifications.delete(id) + notifications.delete(id) + end + if nojs if preferences source = preferences.comments[0] @@ -558,6 +565,7 @@ get "/embed/:id" do |env| if user subscriptions = user.subscriptions watched = user.watched + notifications = user.notifications end subscriptions ||= [] of String @@ -580,6 +588,12 @@ get "/embed/:id" do |env| PG_DB.exec("UPDATE users SET watched = watched || $1 WHERE email = $2", [id], user.as(User).email) end + if notifications && notifications.includes? id + PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + env.get("user").as(User).notifications.delete(id) + notifications.delete(id) + end + fmt_stream = video.fmt_stream(decrypt_function) adaptive_fmts = video.adaptive_fmts(decrypt_function) @@ -1696,10 +1710,10 @@ post "/subscription_ajax" do |env| when .starts_with? "action_create" if !user.subscriptions.includes? channel_id get_channel(channel_id, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", channel_id, email) + PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) end when .starts_with? "action_remove" - PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", channel_id, email) + PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) end payload = { @@ -4534,7 +4548,7 @@ delete "/api/v1/auth/subscriptions/:ucid" do |env| ucid = env.params.url["ucid"] - PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", ucid, user.email) + PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) payload = { "email" => user.email, "action" => "refresh", From 18d66dddedb4c6c14975a0ab47377258fea59e2e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 10:19:18 -0500 Subject: [PATCH 003/146] Add 'needs_update' column for scheduling feed refresh --- config/migrate-scripts/migrate-db-701b5ea.sh | 3 + config/sql/users.sql | 1 + src/invidious.cr | 60 +++++------------- src/invidious/channels.cr | 29 ++++----- src/invidious/helpers/helpers.cr | 1 - src/invidious/helpers/jobs.cr | 66 ++------------------ src/invidious/users.cr | 11 ++-- 7 files changed, 46 insertions(+), 125 deletions(-) create mode 100755 config/migrate-scripts/migrate-db-701b5ea.sh diff --git a/config/migrate-scripts/migrate-db-701b5ea.sh b/config/migrate-scripts/migrate-db-701b5ea.sh new file mode 100755 index 000000000..429531a2a --- /dev/null +++ b/config/migrate-scripts/migrate-db-701b5ea.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +psql invidious kemal -c "ALTER TABLE users ADD COLUMN feed_needs_update boolean" diff --git a/config/sql/users.sql b/config/sql/users.sql index 536508a45..0f2cdba27 100644 --- a/config/sql/users.sql +++ b/config/sql/users.sql @@ -12,6 +12,7 @@ CREATE TABLE public.users password text, token text, watched text[], + feed_needs_update boolean, CONSTRAINT users_email_key UNIQUE (email) ); diff --git a/src/invidious.cr b/src/invidious.cr index de1349473..822f7b85c 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1710,18 +1710,12 @@ post "/subscription_ajax" do |env| when .starts_with? "action_create" if !user.subscriptions.includes? channel_id get_channel(channel_id, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) end when .starts_with? "action_remove" - PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) end - payload = { - "email" => user.email, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") - if redirect env.redirect referer else @@ -1884,7 +1878,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) end if body["watch_history"]? @@ -1906,7 +1900,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_freetube" user.subscriptions += body.scan(/"channelId":"(?[a-zA-Z0-9_-]{24})"/).map do |md| md["channel_id"] @@ -1915,7 +1909,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe_subscriptions" body = JSON.parse(body) user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| @@ -1939,7 +1933,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe" Zip::Reader.open(IO::Memory.new(body)) do |file| file.each_entry do |entry| @@ -1958,7 +1952,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) db.close tempfile.delete @@ -1967,12 +1961,6 @@ post "/data_control" do |env| end end end - - payload = { - "email" => user.email, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") end env.redirect referer @@ -2874,7 +2862,7 @@ post "/feed/webhook/:token" do |env| views: video.views, ) - users = PG_DB.query_all("UPDATE users SET notifications = notifications || $1 \ + emails = PG_DB.query_all("UPDATE users SET notifications = notifications || $1 \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email", video.id, video.published, video.ucid, as: String) @@ -2886,13 +2874,14 @@ post "/feed/webhook/:token" do |env| updated = $4, ucid = $5, author = $6, length_seconds = $7, \ live_now = $8, premiere_timestamp = $9, views = $10", video_array) - users.each do |user| - payload = { - "email" => user, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") + # Update all users affected by insert + if emails.empty? + values = "'{}'" + else + values = "VALUES #{emails.map { |id| %(('#{id}')) }.join(",")}" end + + PG_DB.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY($1)", emails) end end @@ -4490,7 +4479,6 @@ post "/api/v1/auth/preferences" do |env| PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) env.response.status_code = 204 - "" end get "/api/v1/auth/subscriptions" do |env| @@ -4525,13 +4513,7 @@ post "/api/v1/auth/subscriptions/:ucid" do |env| if !user.subscriptions.includes? ucid get_channel(ucid, PG_DB, false, false) - PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) - - payload = { - "email" => user.email, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) end # For Google accounts, access tokens don't have enough information to @@ -4539,7 +4521,6 @@ post "/api/v1/auth/subscriptions/:ucid" do |env| # YouTube. env.response.status_code = 204 - "" end delete "/api/v1/auth/subscriptions/:ucid" do |env| @@ -4548,15 +4529,9 @@ delete "/api/v1/auth/subscriptions/:ucid" do |env| ucid = env.params.url["ucid"] - PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) - payload = { - "email" => user.email, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") + PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) env.response.status_code = 204 - "" end get "/api/v1/auth/tokens" do |env| @@ -4663,7 +4638,6 @@ post "/api/v1/auth/tokens/unregister" do |env| end env.response.status_code = 204 - "" end get "/api/manifest/dash/id/videoplayback" do |env| diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index d0eb7dd3d..d33cd9c38 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -184,7 +184,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) views: views, ) - users = db.query_all("UPDATE users SET notifications = notifications || $1 \ + emails = db.query_all("UPDATE users SET notifications = notifications || $1 \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email", video.id, video.published, ucid, as: String) @@ -198,13 +198,14 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) updated = $4, ucid = $5, author = $6, length_seconds = $7, \ live_now = $8, views = $10", video_array) - users.each do |user| - payload = { - "email" => user, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") + # Update all users affected by insert + if emails.empty? + values = "'{}'" + else + values = "VALUES #{emails.map { |id| %(('#{id}')) }.join(",")}" end + + db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY($1)", emails) end if pull_all_videos @@ -252,7 +253,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.now - video.published > 1.minute - users = db.query_all("UPDATE users SET notifications = notifications || $1 \ + emails = db.query_all("UPDATE users SET notifications = notifications || $1 \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email", video.id, video.published, video.ucid, as: String) @@ -266,13 +267,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) live_now = $8, views = $10", video_array) # Update all users affected by insert - users.each do |user| - payload = { - "email" => user, - "action" => "refresh", - }.to_json - PG_DB.exec("NOTIFY feeds, E'#{payload}'") + if emails.empty? + values = "'{}'" + else + values = "VALUES #{emails.map { |id| %(('#{id}')) }.join(",")}" end + + db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY($1)", emails) end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 3155cb676..476038c7d 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -105,7 +105,6 @@ struct Config hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) - use_feed_events: {type: Bool | Int32, default: false}, # Update feeds on receiving notifications default_home: {type: String, default: "Top"}, feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending", "Subscriptions"]}, top_enabled: {type: Bool, default: true}, diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index b9f9a86fd..1dd81cf5f 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -43,66 +43,6 @@ def refresh_channels(db, logger, config) end def refresh_feeds(db, logger, config) - # Spawn thread to handle feed events - if config.use_feed_events - case config.use_feed_events - when Bool - max_feed_event_threads = config.use_feed_events.as(Bool).to_unsafe - when Int32 - max_feed_event_threads = config.use_feed_events.as(Int32) - end - max_feed_event_channel = Channel(Int32).new - - spawn do - queue = Deque(String).new(30) - PG.connect_listen(PG_URL, "feeds") do |event| - if !queue.includes? event.payload - queue << event.payload - end - end - - max_threads = max_feed_event_channel.receive - active_threads = 0 - active_channel = Channel(Bool).new - - loop do - until queue.empty? - event = queue.shift - - if active_threads >= max_threads - if active_channel.receive - active_threads -= 1 - end - end - - active_threads += 1 - - spawn do - begin - feed = JSON.parse(event) - email = feed["email"].as_s - action = feed["action"].as_s - - view_name = "subscriptions_#{sha256(email)}" - - case action - when "refresh" - db.exec("REFRESH MATERIALIZED VIEW #{view_name}") - end - rescue ex - end - - active_channel.send(true) - end - end - - sleep 5.seconds - end - end - - max_feed_event_channel.send(max_feed_event_threads.as(Int32)) - end - max_channel = Channel(Int32).new spawn do max_threads = max_channel.receive @@ -110,7 +50,7 @@ def refresh_feeds(db, logger, config) active_channel = Channel(Bool).new loop do - db.query("SELECT email FROM users") do |rs| + db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs| rs.each do email = rs.read(String) view_name = "subscriptions_#{sha256(email)}" @@ -135,6 +75,7 @@ def refresh_feeds(db, logger, config) end db.exec("REFRESH MATERIALIZED VIEW #{view_name}") + db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) rescue ex # Rename old views begin @@ -152,6 +93,7 @@ def refresh_feeds(db, logger, config) SELECT * FROM channel_videos WHERE \ ucid = ANY ((SELECT subscriptions FROM users WHERE email = E'#{email.gsub("'", "\\'")}')::text[]) \ ORDER BY published DESC;") + db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex logger.write("REFRESH #{email} : #{ex.message}\n") @@ -164,7 +106,7 @@ def refresh_feeds(db, logger, config) end end - sleep 1.minute + sleep 5.seconds end end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 298d6b0d7..ceaac9f11 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -20,9 +20,10 @@ struct User type: Preferences, converter: PreferencesConverter, }, - password: String?, - token: String, - watched: Array(String), + password: String?, + token: String, + watched: Array(String), + feed_needs_update: Bool?, }) end @@ -205,7 +206,7 @@ def fetch_user(sid, headers, db) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.now, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String) + user = User.new(Time.now, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true) return user, sid end @@ -213,7 +214,7 @@ def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.now, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String) + user = User.new(Time.now, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true) return user, sid end From b3788bc1431aea47b7a9ffb325984f4a58c21125 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 11:19:01 -0500 Subject: [PATCH 004/146] Fix typo for feed_needs_update --- src/invidious.cr | 2 +- src/invidious/channels.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 822f7b85c..86df17758 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2881,7 +2881,7 @@ post "/feed/webhook/:token" do |env| values = "VALUES #{emails.map { |id| %(('#{id}')) }.join(",")}" end - PG_DB.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY($1)", emails) + PG_DB.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})") end end diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index d33cd9c38..0f760dada 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -205,7 +205,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) values = "VALUES #{emails.map { |id| %(('#{id}')) }.join(",")}" end - db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY($1)", emails) + db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})") end if pull_all_videos @@ -273,7 +273,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) values = "VALUES #{emails.map { |id| %(('#{id}')) }.join(",")}" end - db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY($1)", emails) + db.exec("UPDATE users SET feed_needs_update = true WHERE email = ANY(#{values})") end end From 0338fd42e15ee9803068e6d6eeb04d78b94f321c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 5 May 2019 07:46:01 -0500 Subject: [PATCH 005/146] Add support for Web notifications --- assets/js/embed.js | 4 +- assets/js/notifications.js | 139 +++++++++++++++++++ assets/js/sse.js | 200 ++++++++++++++++++++++++++++ assets/js/subscribe_widget.js | 12 +- locales/ar.json | 3 + locales/de.json | 3 + locales/el.json | 3 + locales/en-US.json | 3 + locales/eo.json | 3 + locales/es.json | 5 +- locales/eu.json | 3 + locales/fr.json | 3 + locales/it.json | 3 + locales/nb_NO.json | 3 + locales/nl.json | 5 +- locales/pl.json | 3 + locales/ru.json | 3 + locales/uk.json | 3 + src/invidious/views/embed.ecr | 36 ++--- src/invidious/views/licenses.ecr | 28 ++++ src/invidious/views/preferences.ecr | 7 + src/invidious/views/template.ecr | 14 +- 22 files changed, 456 insertions(+), 30 deletions(-) create mode 100644 assets/js/notifications.js create mode 100644 assets/js/sse.js diff --git a/assets/js/embed.js b/assets/js/embed.js index cbf21a585..12530fbee 100644 --- a/assets/js/embed.js +++ b/assets/js/embed.js @@ -1,4 +1,4 @@ -function get_playlist(plid, timeouts = 0) { +function get_playlist(plid, timeouts) { if (timeouts > 10) { console.log('Failed to pull playlist'); return; @@ -53,7 +53,7 @@ function get_playlist(plid, timeouts = 0) { xhr.ontimeout = function () { console.log('Pulling playlist timed out.'); - get_playlist(plid, timeouts + 1); + get_playlist(plid, timeouts++); } } diff --git a/assets/js/notifications.js b/assets/js/notifications.js new file mode 100644 index 000000000..5c9847de8 --- /dev/null +++ b/assets/js/notifications.js @@ -0,0 +1,139 @@ +var notifications, delivered; + +function get_subscriptions(callback, failures) { + if (failures >= 10) { + return + } + + var xhr = new XMLHttpRequest(); + xhr.responseType = 'json'; + xhr.timeout = 20000; + xhr.open('GET', '/api/v1/auth/subscriptions', true); + xhr.send(null); + + xhr.onreadystatechange = function () { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + subscriptions = xhr.response; + callback(subscriptions); + } else { + console.log('Pulling subscriptions failed... ' + failures + '/10'); + get_subscriptions(callback, failures++) + } + } + } + + xhr.ontimeout = function () { + console.log('Pulling subscriptions failed... ' + failures + '/10'); + get_subscriptions(callback, failures++); + } +} + +function create_notification_stream(subscriptions) { + notifications = new SSE( + '/api/v1/auth/notifications?fields=videoId,title,author,authorId,publishedText,published,authorThumbnails,liveNow', { + withCredentials: true, + payload: 'topics=' + subscriptions.map(function (subscription) { return subscription.authorId }).join(','), + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + }); + delivered = []; + + var start_time = Math.round(new Date() / 1000); + + notifications.onmessage = function (event) { + if (!event.id) { + return + } + + var notification = JSON.parse(event.data); + console.log('Got notification:', notification); + + if (start_time < notification.published && !delivered.includes(notification.videoId)) { + if (Notification.permission === 'granted') { + var system_notification = + new Notification((notification.liveNow ? notification_data.live_now_text : notification_data.upload_text).replace('`x`', notification.author), { + body: notification.title, + icon: '/ggpht' + new URL(notification.authorThumbnails[2].url).pathname, + img: '/ggpht' + new URL(notification.authorThumbnails[4].url).pathname, + tag: notification.videoId + }); + + system_notification.onclick = function (event) { + window.open('/watch?v=' + event.currentTarget.tag, '_blank'); + } + } + + delivered.push(notification.videoId); + localStorage.setItem('notification_count', parseInt(localStorage.getItem('notification_count') || '0') + 1); + var notification_ticker = document.getElementById('notification_ticker'); + + if (parseInt(localStorage.getItem('notification_count')) > 0) { + notification_ticker.innerHTML = + '' + localStorage.getItem('notification_count') + ' '; + } else { + notification_ticker.innerHTML = + ''; + } + } + } + + notifications.onerror = function (event) { + console.log('Something went wrong with notifications, trying to reconnect...'); + notifications.close(); + get_subscriptions(create_notification_stream); + } + + notifications.ontimeout = function (event) { + console.log('Something went wrong with notifications, trying to reconnect...'); + notifications.close(); + get_subscriptions(create_notification_stream); + } + + notifications.stream(); +} + +window.addEventListener('storage', function (e) { + if (e.key === 'stream' && !e.newValue) { + if (notifications) { + localStorage.setItem('stream', true); + } else { + setTimeout(function () { + if (!localStorage.getItem('stream')) { + get_subscriptions(create_notification_stream); + localStorage.setItem('stream', true); + } + }, Math.random() * 1000 + 10); + } + } else if (e.key === 'notification_count') { + var notification_ticker = document.getElementById('notification_ticker'); + + if (parseInt(e.newValue) > 0) { + notification_ticker.innerHTML = + '' + e.newValue + ' '; + } else { + notification_ticker.innerHTML = + ''; + } + } +}); + +window.addEventListener('load', function (e) { + localStorage.setItem('notification_count', document.getElementById('notification_count') ? document.getElementById('notification_count').innerText : '0'); + + if (localStorage.getItem('stream')) { + localStorage.removeItem('stream'); + } else { + setTimeout(function () { + if (!localStorage.getItem('stream')) { + get_subscriptions(create_notification_stream); + localStorage.setItem('stream', true); + } + }, Math.random() * 1000 + 10); + } +}); + +window.addEventListener('unload', function (e) { + if (notifications) { + localStorage.removeItem('stream'); + } +}); diff --git a/assets/js/sse.js b/assets/js/sse.js new file mode 100644 index 000000000..3601b5af5 --- /dev/null +++ b/assets/js/sse.js @@ -0,0 +1,200 @@ +/** + * Copyright (C) 2016 Maxime Petazzoni . + * All rights reserved. + */ + +var SSE = function (url, options) { + if (!(this instanceof SSE)) { + return new SSE(url, options); + } + + this.INITIALIZING = -1; + this.CONNECTING = 0; + this.OPEN = 1; + this.CLOSED = 2; + + this.url = url; + + options = options || {}; + this.headers = options.headers || {}; + this.payload = options.payload !== undefined ? options.payload : ''; + this.method = options.method || (this.payload && 'POST' || 'GET'); + + this.FIELD_SEPARATOR = ':'; + this.listeners = {}; + + this.xhr = null; + this.readyState = this.INITIALIZING; + this.progress = 0; + this.chunk = ''; + + this.addEventListener = function(type, listener) { + if (this.listeners[type] === undefined) { + this.listeners[type] = []; + } + + if (this.listeners[type].indexOf(listener) === -1) { + this.listeners[type].push(listener); + } + }; + + this.removeEventListener = function(type, listener) { + if (this.listeners[type] === undefined) { + return; + } + + var filtered = []; + this.listeners[type].forEach(function(element) { + if (element !== listener) { + filtered.push(element); + } + }); + if (filtered.length === 0) { + delete this.listeners[type]; + } else { + this.listeners[type] = filtered; + } + }; + + this.dispatchEvent = function(e) { + if (!e) { + return true; + } + + e.source = this; + + var onHandler = 'on' + e.type; + if (this.hasOwnProperty(onHandler)) { + this[onHandler].call(this, e); + if (e.defaultPrevented) { + return false; + } + } + + if (this.listeners[e.type]) { + return this.listeners[e.type].every(function(callback) { + callback(e); + return !e.defaultPrevented; + }); + } + + return true; + }; + + this._setReadyState = function (state) { + var event = new CustomEvent('readystatechange'); + event.readyState = state; + this.readyState = state; + this.dispatchEvent(event); + }; + + this._onStreamFailure = function(e) { + this.dispatchEvent(new CustomEvent('error')); + this.close(); + } + + this._onStreamProgress = function(e) { + if (this.xhr.status !== 200) { + this._onStreamFailure(e); + return; + } + + if (this.readyState == this.CONNECTING) { + this.dispatchEvent(new CustomEvent('open')); + this._setReadyState(this.OPEN); + } + + var data = this.xhr.responseText.substring(this.progress); + this.progress += data.length; + data.split(/(\r\n|\r|\n){2}/g).forEach(function(part) { + if (part.trim().length === 0) { + this.dispatchEvent(this._parseEventChunk(this.chunk.trim())); + this.chunk = ''; + } else { + this.chunk += part; + } + }.bind(this)); + }; + + this._onStreamLoaded = function(e) { + this._onStreamProgress(e); + + // Parse the last chunk. + this.dispatchEvent(this._parseEventChunk(this.chunk)); + this.chunk = ''; + }; + + /** + * Parse a received SSE event chunk into a constructed event object. + */ + this._parseEventChunk = function(chunk) { + if (!chunk || chunk.length === 0) { + return null; + } + + var e = {'id': null, 'retry': null, 'data': '', 'event': 'message'}; + chunk.split(/\n|\r\n|\r/).forEach(function(line) { + line = line.trimRight(); + var index = line.indexOf(this.FIELD_SEPARATOR); + if (index <= 0) { + // Line was either empty, or started with a separator and is a comment. + // Either way, ignore. + return; + } + + var field = line.substring(0, index); + if (!(field in e)) { + return; + } + + var value = line.substring(index + 1).trimLeft(); + if (field === 'data') { + e[field] += value; + } else { + e[field] = value; + } + }.bind(this)); + + var event = new CustomEvent(e.event); + event.data = e.data; + event.id = e.id; + return event; + }; + + this._checkStreamClosed = function() { + if (this.xhr.readyState === XMLHttpRequest.DONE) { + this._setReadyState(this.CLOSED); + } + }; + + this.stream = function() { + this._setReadyState(this.CONNECTING); + + this.xhr = new XMLHttpRequest(); + this.xhr.addEventListener('progress', this._onStreamProgress.bind(this)); + this.xhr.addEventListener('load', this._onStreamLoaded.bind(this)); + this.xhr.addEventListener('readystatechange', this._checkStreamClosed.bind(this)); + this.xhr.addEventListener('error', this._onStreamFailure.bind(this)); + this.xhr.addEventListener('abort', this._onStreamFailure.bind(this)); + this.xhr.open(this.method, this.url); + for (var header in this.headers) { + this.xhr.setRequestHeader(header, this.headers[header]); + } + this.xhr.send(this.payload); + }; + + this.close = function() { + if (this.readyState === this.CLOSED) { + return; + } + + this.xhr.abort(); + this.xhr = null; + this._setReadyState(this.CLOSED); + }; +}; + +// Export our SSE module for npm.js +if (typeof exports !== 'undefined') { + exports.SSE = SSE; +} diff --git a/assets/js/subscribe_widget.js b/assets/js/subscribe_widget.js index 25c5f2a6d..f875d505d 100644 --- a/assets/js/subscribe_widget.js +++ b/assets/js/subscribe_widget.js @@ -7,8 +7,8 @@ if (subscribe_button.getAttribute('data-type') === 'subscribe') { subscribe_button.onclick = unsubscribe; } -function subscribe(timeouts = 0) { - if (timeouts > 10) { +function subscribe(timeouts) { + if (timeouts >= 10) { console.log('Failed to subscribe.'); return; } @@ -37,12 +37,12 @@ function subscribe(timeouts = 0) { xhr.ontimeout = function () { console.log('Subscribing timed out.'); - subscribe(timeouts + 1); + subscribe(timeouts++); } } -function unsubscribe(timeouts = 0) { - if (timeouts > 10) { +function unsubscribe(timeouts) { + if (timeouts >= 10) { console.log('Failed to subscribe'); return; } @@ -71,6 +71,6 @@ function unsubscribe(timeouts = 0) { xhr.ontimeout = function () { console.log('Unsubscribing timed out.'); - unsubscribe(timeouts + 1); + unsubscribe(timeouts++); } } diff --git a/locales/ar.json b/locales/ar.json index 9e0772428..9619043d0 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "فقط اظهر اخر فيديو لم يتم رؤيتة من القناة: ", "Only show unwatched: ": "فقط اظهر الذى لم يتم رؤيتة: ", "Only show notifications (if there are any): ": "إظهار الإشعارات فقط (إذا كان هناك أي): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "إعدادات التفضيلات", "Clear watch history": "حذف سجل المشاهدة", "Import/export data": "إضافة\\إستخراج البيانات", diff --git a/locales/de.json b/locales/de.json index 4e243c4e5..d1b2601ec 100644 --- a/locales/de.json +++ b/locales/de.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Nur neueste ungesehene Videos des Kanals anzeigen: ", "Only show unwatched: ": "Nur ungesehene anzeigen: ", "Only show notifications (if there are any): ": "Nur Benachrichtigungen anzeigen (wenn es welche gibt): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Dateneinstellungen", "Clear watch history": "Verlauf löschen", "Import/export data": "Daten im- exportieren", diff --git a/locales/el.json b/locales/el.json index 7a12d2df2..54d514cb1 100644 --- a/locales/el.json +++ b/locales/el.json @@ -91,6 +91,9 @@ "Only show latest unwatched video from channel: ": "Προβολή μόνο του τελευταίου μη-προβεβλημένου βίντεο του καναλιού: ", "Only show unwatched: ": "Προβολή μόνο μη-προβεβλημένων: ", "Only show notifications (if there are any): ": "Προβολή μόνο ειδοποιήσεων (αν υπάρχουν): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Προτιμήσεις δεδομένων", "Clear watch history": "Εκκαθάριση ιστορικού προβολής", "Import/export data": "Εισαγωγή/εξαγωγή δεδομένων", diff --git a/locales/en-US.json b/locales/en-US.json index 5f6245f53..1ca2b9704 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -91,6 +91,9 @@ "Only show latest unwatched video from channel: ": "Only show latest unwatched video from channel: ", "Only show unwatched: ": "Only show unwatched: ", "Only show notifications (if there are any): ": "Only show notifications (if there are any): ", + "Enable web notifications": "Enable web notifications", + "`x` uploaded a video": "`x` uploaded a video", + "`x` is live": "`x` is live", "Data preferences": "Data preferences", "Clear watch history": "Clear watch history", "Import/export data": "Import/export data", diff --git a/locales/eo.json b/locales/eo.json index 3f06c7904..f14ef4667 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Nur montri pli novan malviditan videon el kanalo: ", "Only show unwatched: ": "Nur montri malviditajn: ", "Only show notifications (if there are any): ": "Nur montri sciigojn (se estas): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Datumagordoj", "Clear watch history": "Forigi vidohistorion", "Import/export data": "Importi/Eksporti datumojn", diff --git a/locales/es.json b/locales/es.json index 2f6d8560c..e0fac8a27 100644 --- a/locales/es.json +++ b/locales/es.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Mostrar solo el último vídeo sin ver del canal: ", "Only show unwatched: ": "Mostrar solo los no vistos: ", "Only show notifications (if there are any): ": "Mostrar solo notificaciones (si hay alguna): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Preferencias de los datos", "Clear watch history": "Borrar el historial de reproducción", "Import/export data": "Importar/Exportar datos", @@ -312,4 +315,4 @@ "Videos": "Vídeos", "Playlists": "Listas de reproducción", "Current version: ": "Versión actual: " -} +} \ No newline at end of file diff --git a/locales/eu.json b/locales/eu.json index aae43603a..60fa6f6d8 100644 --- a/locales/eu.json +++ b/locales/eu.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "", "Only show unwatched: ": "", "Only show notifications (if there are any): ": "", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "", "Clear watch history": "", "Import/export data": "", diff --git a/locales/fr.json b/locales/fr.json index 9f042480f..e2d586aec 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Afficher uniquement la dernière vidéo de la chaîne non regardée : ", "Only show unwatched: ": "Afficher uniquement les vidéos non regardées : ", "Only show notifications (if there are any): ": "Afficher uniquement les notifications (s'il y en a) : ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Préférences liées aux données", "Clear watch history": "Supprimer l'historique des vidéos regardées", "Import/export data": "Importer/exporter les données", diff --git a/locales/it.json b/locales/it.json index 10527f9f6..ce7800c3c 100644 --- a/locales/it.json +++ b/locales/it.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Mostra solo il video più recente non guardato del canale: ", "Only show unwatched: ": "Mostra solo i video non guardati: ", "Only show notifications (if there are any): ": "Mostra solo le notifiche (se presenti): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Preferenze dati", "Clear watch history": "Cancella la cronologia dei video guardati", "Import/export data": "Importazione/esportazione dati", diff --git a/locales/nb_NO.json b/locales/nb_NO.json index 634922450..83f97570b 100644 --- a/locales/nb_NO.json +++ b/locales/nb_NO.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Kun vis siste usette video fra kanal: ", "Only show unwatched: ": "Kun vis usette: ", "Only show notifications (if there are any): ": "Kun vis merknader (hvis det er noen): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Datainnstillinger", "Clear watch history": "Tøm visningshistorikk", "Import/export data": "Importer/eksporter data", diff --git a/locales/nl.json b/locales/nl.json index 5da8548a5..50fe85d8b 100644 --- a/locales/nl.json +++ b/locales/nl.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Alleen nieuwste niet-bekeken video van kanaal tonen: ", "Only show unwatched: ": "Alleen niet-bekeken videos tonen: ", "Only show notifications (if there are any): ": "Alleen meldingen tonen (als die er zijn): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Gegevensinstellingen", "Clear watch history": "Kijkgeschiedenis wissen", "Import/export data": "Gegevens im-/exporteren", @@ -312,4 +315,4 @@ "Videos": "Video's", "Playlists": "Afspeellijsten", "Current version: ": "Huidige versie: " -} +} \ No newline at end of file diff --git a/locales/pl.json b/locales/pl.json index 621bdd76e..fa4ec965a 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Pokazuj tylko najnowszy nie obejrzany film z kanału: ", "Only show unwatched: ": "Pokazuj tylko nie obejrzane: ", "Only show notifications (if there are any): ": "Pokazuj tylko powiadomienia (jeśli są): ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Preferencje danych", "Clear watch history": "Wyczyść historię", "Import/export data": "Import/Eksport danych", diff --git a/locales/ru.json b/locales/ru.json index f9c562044..e603b98f2 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Показывать только непросмотренные видео с каналов: ", "Only show unwatched: ": "Показывать только непросмотренные видео: ", "Only show notifications (if there are any): ": "Показывать только оповещения, если они есть: ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Настройки данных", "Clear watch history": "Очистить историю просмотров", "Import/export data": "Импорт/Экспорт данных", diff --git a/locales/uk.json b/locales/uk.json index e666e2802..319f22d7e 100644 --- a/locales/uk.json +++ b/locales/uk.json @@ -85,6 +85,9 @@ "Only show latest unwatched video from channel: ": "Показувати тільки непереглянуті відео з каналів: ", "Only show unwatched: ": "Показувати тільки непереглянуті відео: ", "Only show notifications (if there are any): ": "Показувати лише сповіщення, якщо вони є: ", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", "Data preferences": "Налаштування даних", "Clear watch history": "Очистити історію переглядів", "Import/export data": "Імпорт і експорт даних", diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index 32abd6269..b6307b9c6 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -2,24 +2,24 @@ "> - - - - <%= rendered "components/player_sources" %> - - <%= HTML.escape(video.title) %> - Invidious - + + + + <%= rendered "components/player_sources" %> + + <%= HTML.escape(video.title) %> - Invidious + diff --git a/src/invidious/views/licenses.ecr b/src/invidious/views/licenses.ecr index 6b10fb996..0f92d86e5 100644 --- a/src/invidious/views/licenses.ecr +++ b/src/invidious/views/licenses.ecr @@ -23,6 +23,20 @@ + + + notifications.js + + + + AGPL-3.0 + + + + <%= translate(locale, "source") %> + + + player.js @@ -51,6 +65,20 @@ + + + sse.js + + + + Expat + + + + <%= translate(locale, "source") %> + + + subscribe_widget.js diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index d0747b59d..e9d2d84c4 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -165,6 +165,13 @@ function update_value(element) { checked<% end %>> + + <% # Conditions for supporting web notifications %> + <% if CONFIG.use_pubsub_feeds && (Kemal.config.ssl || config.https_only) %> +
+ <%= translate(locale, "Enable web notifications") %> +
+ <% end %> <% end %> <% if env.get?("user") && config.admins.includes? env.get?("user").as(User).email %> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 6b6f74fa5..0d8c99249 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -51,10 +51,10 @@
- " href="/feed/subscriptions" class="pure-menu-heading"> + " href="/feed/subscriptions" class="pure-menu-heading"> <% notification_count = env.get("user").as(User).notifications.size %> <% if notification_count > 0 %> - <%= notification_count %> + <%= notification_count %> <% else %> <% end %> @@ -151,6 +151,16 @@
+ <% if env.get? "user" %> + + + + <% end %> From 8cecce75707a31f9aa31d5ecbdb2999afd2dae70 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 16:26:18 -0500 Subject: [PATCH 006/146] Fix audio mode for raw URLs --- src/invidious.cr | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 86df17758..3d69cef1a 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -470,11 +470,23 @@ get "/watch" do |env| thumbnail = "/vi/#{video.id}/maxres.jpg" if params.raw - url = fmt_stream[0]["url"] + if params.listen + url = audio_streams[0]["url"] - fmt_stream.each do |fmt| - if fmt["label"].split(" - ")[0] == params.quality - url = fmt["url"] + audio_streams.each do |fmt| + pp fmt["bitrate"] + pp params.quality.rchop("k") + if fmt["bitrate"] == params.quality.rchop("k") + url = fmt["url"] + end + end + else + url = fmt_stream[0]["url"] + + fmt_stream.each do |fmt| + if fmt["label"].split(" - ")[0] == params.quality + url = fmt["url"] + end end end From 4e111c84f3d5432863c973dbf1d0ea62978b4db7 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 17:18:34 -0500 Subject: [PATCH 007/146] Fix typo in '/watch' --- src/invidious.cr | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 3d69cef1a..9f79a82f8 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -474,8 +474,6 @@ get "/watch" do |env| url = audio_streams[0]["url"] audio_streams.each do |fmt| - pp fmt["bitrate"] - pp params.quality.rchop("k") if fmt["bitrate"] == params.quality.rchop("k") url = fmt["url"] end From e23bab0103ee7dccd94b086ee6f48dd8970388dc Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 17:38:49 -0500 Subject: [PATCH 008/146] Only add notification event listener after onload --- assets/js/embed.js | 2 +- assets/js/notifications.js | 52 +++++++++++++++++------------------ assets/js/subscribe_widget.js | 4 +-- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/assets/js/embed.js b/assets/js/embed.js index 12530fbee..d2116b2ef 100644 --- a/assets/js/embed.js +++ b/assets/js/embed.js @@ -1,4 +1,4 @@ -function get_playlist(plid, timeouts) { +function get_playlist(plid, timeouts = 0) { if (timeouts > 10) { console.log('Failed to pull playlist'); return; diff --git a/assets/js/notifications.js b/assets/js/notifications.js index 5c9847de8..90b8c4f02 100644 --- a/assets/js/notifications.js +++ b/assets/js/notifications.js @@ -1,6 +1,6 @@ var notifications, delivered; -function get_subscriptions(callback, failures) { +function get_subscriptions(callback, failures = 1) { if (failures >= 10) { return } @@ -92,31 +92,6 @@ function create_notification_stream(subscriptions) { notifications.stream(); } -window.addEventListener('storage', function (e) { - if (e.key === 'stream' && !e.newValue) { - if (notifications) { - localStorage.setItem('stream', true); - } else { - setTimeout(function () { - if (!localStorage.getItem('stream')) { - get_subscriptions(create_notification_stream); - localStorage.setItem('stream', true); - } - }, Math.random() * 1000 + 10); - } - } else if (e.key === 'notification_count') { - var notification_ticker = document.getElementById('notification_ticker'); - - if (parseInt(e.newValue) > 0) { - notification_ticker.innerHTML = - '' + e.newValue + ' '; - } else { - notification_ticker.innerHTML = - ''; - } - } -}); - window.addEventListener('load', function (e) { localStorage.setItem('notification_count', document.getElementById('notification_count') ? document.getElementById('notification_count').innerText : '0'); @@ -130,6 +105,31 @@ window.addEventListener('load', function (e) { } }, Math.random() * 1000 + 10); } + + window.addEventListener('storage', function (e) { + if (e.key === 'stream' && !e.newValue) { + if (notifications) { + localStorage.setItem('stream', true); + } else { + setTimeout(function () { + if (!localStorage.getItem('stream')) { + get_subscriptions(create_notification_stream); + localStorage.setItem('stream', true); + } + }, Math.random() * 1000 + 10); + } + } else if (e.key === 'notification_count') { + var notification_ticker = document.getElementById('notification_ticker'); + + if (parseInt(e.newValue) > 0) { + notification_ticker.innerHTML = + '' + e.newValue + ' '; + } else { + notification_ticker.innerHTML = + ''; + } + } + }); }); window.addEventListener('unload', function (e) { diff --git a/assets/js/subscribe_widget.js b/assets/js/subscribe_widget.js index f875d505d..8f055e26f 100644 --- a/assets/js/subscribe_widget.js +++ b/assets/js/subscribe_widget.js @@ -7,7 +7,7 @@ if (subscribe_button.getAttribute('data-type') === 'subscribe') { subscribe_button.onclick = unsubscribe; } -function subscribe(timeouts) { +function subscribe(timeouts = 0) { if (timeouts >= 10) { console.log('Failed to subscribe.'); return; @@ -41,7 +41,7 @@ function subscribe(timeouts) { } } -function unsubscribe(timeouts) { +function unsubscribe(timeouts = 0) { if (timeouts >= 10) { console.log('Failed to subscribe'); return; From 576067c1e5d3586b0773035f935addc0e0724396 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 1 Jun 2019 18:06:44 -0500 Subject: [PATCH 009/146] Fix preference for web notifications --- src/invidious/views/preferences.ecr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index e9d2d84c4..232ce2244 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -166,8 +166,8 @@ function update_value(element) { checked<% end %>> - <% # Conditions for supporting web notifications %> - <% if CONFIG.use_pubsub_feeds && (Kemal.config.ssl || config.https_only) %> + <% # Web notifications are only supported over HTTPS %> + <% if Kemal.config.ssl || config.https_only %>
<%= translate(locale, "Enable web notifications") %>
From 71bf8b6b4d66a615a84b86446ca2a1c350868965 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 2 Jun 2019 07:41:53 -0500 Subject: [PATCH 010/146] Refactor connect_listen for notifications --- src/invidious.cr | 31 +++++- src/invidious/helpers/helpers.cr | 174 +++++++++++++++---------------- 2 files changed, 115 insertions(+), 90 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 9f79a82f8..296db08a4 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -186,6 +186,13 @@ spawn do end end +notification_channels = [] of Channel(PQ::Notification) +PG.connect_listen(PG_URL, "notifications") do |event| + notification_channels.each do |channel| + channel.send(event) + end +end + proxies = PROXY_LIST before_all do |env| @@ -4457,17 +4464,37 @@ get "/api/v1/mixes/:rdid" do |env| end get "/api/v1/auth/notifications" do |env| + env.response.content_type = "text/event-stream" + topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) topics ||= [] of String - create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics) + notification_channel = Channel(PQ::Notification).new + notification_channels << notification_channel + + begin + create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics, notification_channel) + rescue ex + ensure + notification_channels.delete(notification_channel) + end end post "/api/v1/auth/notifications" do |env| + env.response.content_type = "text/event-stream" + topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) topics ||= [] of String - create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics) + notification_channel = Channel(PQ::Notification).new + notification_channels << notification_channel + + begin + create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics, notification_channel) + rescue ex + ensure + notification_channels.delete(notification_channel) + end end get "/api/v1/auth/preferences" do |env| diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 476038c7d..539db2504 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -661,89 +661,23 @@ def copy_in_chunks(input, output, chunk_size = 4096) end end -def create_notification_stream(env, proxies, config, kemal_config, decrypt_function, topics) +def create_notification_stream(env, proxies, config, kemal_config, decrypt_function, topics, notification_channel) locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "text/event-stream" - since = env.params.query["since"]?.try &.to_i? + id = 0 - begin - id = 0 - - if topics.includes? "debug" - spawn do - loop do - time_span = [0, 0, 0, 0] - time_span[rand(4)] = rand(30) + 5 - published = Time.now - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) - video_id = TEST_IDS[rand(TEST_IDS.size)] - - video = get_video(video_id, PG_DB, proxies) - video.published = published - response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function)) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush - - id += 1 - - sleep 1.minute - end - end - end - + if topics.includes? "debug" spawn do - if since - topics.try &.each do |topic| - case topic - when .match(/UC[A-Za-z0-9_-]{22}/) - PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", - topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| - response = JSON.parse(video.to_json(locale, config, Kemal.config)) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush - - id += 1 - end - else - # TODO - end - end - end - - PG.connect_listen(PG_URL, "notifications") do |event| - notification = JSON.parse(event.payload) - topic = notification["topic"].as_s - video_id = notification["videoId"].as_s - published = notification["published"].as_i64 + loop do + time_span = [0, 0, 0, 0] + time_span[rand(4)] = rand(30) + 5 + published = Time.now - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) + video_id = TEST_IDS[rand(TEST_IDS.size)] video = get_video(video_id, PG_DB, proxies) - video.published = Time.unix(published) - response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function)) + video.published = published + response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function)) if fields_text = env.params.query["fields"]? begin @@ -754,24 +688,88 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct end end - if topics.try &.includes? topic - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush - id += 1 + id += 1 + + sleep 1.minute + end + end + end + + spawn do + if since + topics.try &.each do |topic| + case topic + when .match(/UC[A-Za-z0-9_-]{22}/) + PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", + topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| + response = JSON.parse(video.to_json(locale, config, Kemal.config)) + + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush + + id += 1 + end + else + # TODO end end end + end - # Send heartbeat + spawn do loop do - env.response.puts ":keepalive #{Time.now.to_unix}" - env.response.puts - env.response.flush - sleep (20 + rand(11)).seconds + event = notification_channel.receive + + notification = JSON.parse(event.payload) + topic = notification["topic"].as_s + video_id = notification["videoId"].as_s + published = notification["published"].as_i64 + + video = get_video(video_id, PG_DB, proxies) + video.published = Time.unix(published) + response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function)) + + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + + if topics.try &.includes? topic + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush + + id += 1 + end end - rescue + end + + # Send heartbeat + loop do + env.response.puts ":keepalive #{Time.now.to_unix}" + env.response.puts + env.response.flush + sleep (20 + rand(11)).seconds end end From 108648b427398aa6b9139603e034cff1807701c0 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 2 Jun 2019 11:48:18 -0500 Subject: [PATCH 011/146] Optimize query for creating subscription feeds --- src/invidious.cr | 6 +++--- src/invidious/helpers/jobs.cr | 11 ++++++++--- src/invidious/users.cr | 12 ++++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 296db08a4..0049bfdc9 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1224,9 +1224,9 @@ post "/login" do |env| view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE \ - ucid = ANY ((SELECT subscriptions FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}')::text[]) \ - ORDER BY published DESC;") + SELECT * FROM channel_videos WHERE + ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}') + ORDER BY published DESC") if Kemal.config.ssl || config.https_only secure = true diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index 1dd81cf5f..63f7c16f8 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -74,6 +74,11 @@ def refresh_feeds(db, logger, config) end end + if db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "ucid = ANY" + logger.write("Materialized view #{view_name} is out-of-date, recreating...\n") + db.exec("DROP MATERIALIZED VIEW #{view_name}") + end + db.exec("REFRESH MATERIALIZED VIEW #{view_name}") db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) rescue ex @@ -90,9 +95,9 @@ def refresh_feeds(db, logger, config) if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) logger.write("CREATE #{view_name}\n") db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE \ - ucid = ANY ((SELECT subscriptions FROM users WHERE email = E'#{email.gsub("'", "\\'")}')::text[]) \ - ORDER BY published DESC;") + SELECT * FROM channel_videos WHERE + ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{email.gsub("'", "\\'")}') + ORDER BY published DESC") db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex diff --git a/src/invidious/users.cr b/src/invidious/users.cr index ceaac9f11..a8764a142 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -149,9 +149,9 @@ def get_user(sid, headers, db, refresh = true) begin view_name = "subscriptions_#{sha256(user.email)}" db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE \ - ucid = ANY ((SELECT subscriptions FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}')::text[]) \ - ORDER BY published DESC;") + SELECT * FROM channel_videos WHERE + ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}') + ORDER BY published DESC") rescue ex end end @@ -171,9 +171,9 @@ def get_user(sid, headers, db, refresh = true) begin view_name = "subscriptions_#{sha256(user.email)}" db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE \ - ucid = ANY ((SELECT subscriptions FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}')::text[]) \ - ORDER BY published DESC;") + SELECT * FROM channel_videos WHERE + ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}') + ORDER BY published DESC") rescue ex end end From 84b2583973ad615b729faae01d934b82a1ad00d6 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 2 Jun 2019 15:45:29 -0500 Subject: [PATCH 012/146] Fix insert for empty descriptions --- src/invidious/videos.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 8c0595043..6d87d8e56 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1159,7 +1159,7 @@ def fetch_video(id, proxies, region) info["avg_rating"] = "#{avg_rating}" description = html.xpath_node(%q(//p[@id="eow-description"])) - description = description ? description.to_xml(options: XML::SaveOptions::NO_DECL) : "" + description = description ? description.to_xml(options: XML::SaveOptions::NO_DECL) : %q(

) wilson_score = ci_lower_bound(likes, likes + dislikes) From d892ba6aa5f83fab35d353e2f2c6f1ab16d0b832 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 3 Jun 2019 13:12:06 -0500 Subject: [PATCH 013/146] Refactor connection channel for delivering notifications --- src/invidious.cr | 39 +++---- src/invidious/helpers/helpers.cr | 182 +++++++++++++++++-------------- 2 files changed, 117 insertions(+), 104 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 0049bfdc9..e1e9af2a8 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -186,10 +186,21 @@ spawn do end end -notification_channels = [] of Channel(PQ::Notification) -PG.connect_listen(PG_URL, "notifications") do |event| - notification_channels.each do |channel| - channel.send(event) +connection_channel = Channel({Bool, Channel(PQ::Notification)}).new +spawn do + connections = [] of Channel(PQ::Notification) + + PG.connect_listen(PG_URL, "notifications") { |event| connections.each { |connection| connection.send(event) } } + + loop do + action, connection = connection_channel.receive + + case action + when true + connections << connection + when false + connections.delete(connection) + end end end @@ -4469,15 +4480,7 @@ get "/api/v1/auth/notifications" do |env| topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) topics ||= [] of String - notification_channel = Channel(PQ::Notification).new - notification_channels << notification_channel - - begin - create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics, notification_channel) - rescue ex - ensure - notification_channels.delete(notification_channel) - end + create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics, connection_channel) end post "/api/v1/auth/notifications" do |env| @@ -4486,15 +4489,7 @@ post "/api/v1/auth/notifications" do |env| topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) topics ||= [] of String - notification_channel = Channel(PQ::Notification).new - notification_channels << notification_channel - - begin - create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics, notification_channel) - rescue ex - ensure - notification_channels.delete(notification_channel) - end + create_notification_stream(env, proxies, config, Kemal.config, decrypt_function, topics, connection_channel) end get "/api/v1/auth/preferences" do |env| diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 539db2504..699fc3748 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -661,7 +661,10 @@ def copy_in_chunks(input, output, chunk_size = 4096) end end -def create_notification_stream(env, proxies, config, kemal_config, decrypt_function, topics, notification_channel) +def create_notification_stream(env, proxies, config, kemal_config, decrypt_function, topics, connection_channel) + connection = Channel(PQ::Notification).new + connection_channel.send({true, connection}) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? since = env.params.query["since"]?.try &.to_i? @@ -669,15 +672,87 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct if topics.includes? "debug" spawn do + begin + loop do + time_span = [0, 0, 0, 0] + time_span[rand(4)] = rand(30) + 5 + published = Time.now - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) + video_id = TEST_IDS[rand(TEST_IDS.size)] + + video = get_video(video_id, PG_DB, proxies) + video.published = published + response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function)) + + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush + + id += 1 + + sleep 1.minute + end + rescue ex + end + end + end + + spawn do + begin + if since + topics.try &.each do |topic| + case topic + when .match(/UC[A-Za-z0-9_-]{22}/) + PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", + topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| + response = JSON.parse(video.to_json(locale, config, Kemal.config)) + + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush + + id += 1 + end + else + # TODO + end + end + end + end + end + + spawn do + begin loop do - time_span = [0, 0, 0, 0] - time_span[rand(4)] = rand(30) + 5 - published = Time.now - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) - video_id = TEST_IDS[rand(TEST_IDS.size)] + event = connection.receive + + notification = JSON.parse(event.payload) + topic = notification["topic"].as_s + video_id = notification["videoId"].as_s + published = notification["published"].as_i64 video = get_video(video_id, PG_DB, proxies) - video.published = published - response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function)) + video.published = Time.unix(published) + response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function)) if fields_text = env.params.query["fields"]? begin @@ -688,88 +763,31 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct end end - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush + if topics.try &.includes? topic + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush - id += 1 - - sleep 1.minute - end - end - end - - spawn do - if since - topics.try &.each do |topic| - case topic - when .match(/UC[A-Za-z0-9_-]{22}/) - PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", - topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| - response = JSON.parse(video.to_json(locale, config, Kemal.config)) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush - - id += 1 - end - else - # TODO + id += 1 end end + rescue ex + ensure + connection_channel.send({false, connection}) end end - spawn do + begin + # Send heartbeat loop do - event = notification_channel.receive - - notification = JSON.parse(event.payload) - topic = notification["topic"].as_s - video_id = notification["videoId"].as_s - published = notification["published"].as_i64 - - video = get_video(video_id, PG_DB, proxies) - video.published = Time.unix(published) - response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function)) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - if topics.try &.includes? topic - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush - - id += 1 - end + env.response.puts ":keepalive #{Time.now.to_unix}" + env.response.puts + env.response.flush + sleep (20 + rand(11)).seconds end - end - - # Send heartbeat - loop do - env.response.puts ":keepalive #{Time.now.to_unix}" - env.response.puts - env.response.flush - sleep (20 + rand(11)).seconds + rescue ex + ensure + connection_channel.send({false, connection}) end end From d19749734985c1cc057a27494bbaf64a54bcf348 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 3 Jun 2019 13:36:34 -0500 Subject: [PATCH 014/146] Add 'type' field to ChannelVideo and Video --- src/invidious/channels.cr | 2 ++ src/invidious/videos.cr | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 0f760dada..bf897e7d8 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -11,6 +11,8 @@ end struct ChannelVideo def to_json(locale, config, kemal_config, json : JSON::Builder) json.object do + json.field "type", "shortVideo" + json.field "title", self.title json.field "videoId", self.id json.field "videoThumbnails" do diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 6d87d8e56..a58414910 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -276,6 +276,8 @@ struct Video def to_json(locale, config, kemal_config, decrypt_function) JSON.build do |json| json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id json.field "videoThumbnails" do From d6ec441c8ef418ebb8be625e4ee7738cdda62597 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 3 Jun 2019 13:36:49 -0500 Subject: [PATCH 015/146] Add buffer for notification channels --- src/invidious.cr | 2 +- src/invidious/helpers/helpers.cr | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index e1e9af2a8..24dcaf7e7 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -186,7 +186,7 @@ spawn do end end -connection_channel = Channel({Bool, Channel(PQ::Notification)}).new +connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) spawn do connections = [] of Channel(PQ::Notification) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 699fc3748..2dd50d422 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -662,7 +662,7 @@ def copy_in_chunks(input, output, chunk_size = 4096) end def create_notification_stream(env, proxies, config, kemal_config, decrypt_function, topics, connection_channel) - connection = Channel(PQ::Notification).new + connection = Channel(PQ::Notification).new(8) connection_channel.send({true, connection}) locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -750,6 +750,10 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct video_id = notification["videoId"].as_s published = notification["published"].as_i64 + if !topics.try &.includes? topic + next + end + video = get_video(video_id, PG_DB, proxies) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function)) @@ -763,14 +767,12 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct end end - if topics.try &.includes? topic - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush - id += 1 - end + id += 1 end rescue ex ensure From 352e409a6e5df40bb62b00e4053c4294390dd5a3 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 4 Jun 2019 19:58:56 -0500 Subject: [PATCH 016/146] Fix toggle_theme when visiting preferences with JS disabled --- src/invidious.cr | 2 +- src/invidious/helpers/utils.cr | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 24dcaf7e7..e8621b233 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1487,7 +1487,7 @@ end get "/toggle_theme" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - referer = get_referer(env) + referer = get_referer(env, unroll: false) redirect = env.params.query["redirect"]? redirect ||= "true" diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 9bdb6a7e5..fcccb7f9b 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -243,7 +243,7 @@ def make_host_url(config, kemal_config) return "#{scheme}#{host}#{port}" end -def get_referer(env, fallback = "/") +def get_referer(env, fallback = "/", unroll = true) referer = env.params.query["referer"]? referer ||= env.request.headers["referer"]? referer ||= fallback @@ -251,16 +251,18 @@ def get_referer(env, fallback = "/") referer = URI.parse(referer) # "Unroll" nested referrers - loop do - if referer.query - params = HTTP::Params.parse(referer.query.not_nil!) - if params["referer"]? - referer = URI.parse(URI.unescape(params["referer"])) + if unroll + loop do + if referer.query + params = HTTP::Params.parse(referer.query.not_nil!) + if params["referer"]? + referer = URI.parse(URI.unescape(params["referer"])) + else + break + end else break end - else - break end end From d876fd7f5b193ce444e8575da7dc70ce5386b364 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 4 Jun 2019 20:54:38 -0500 Subject: [PATCH 017/146] Add 'unique_res' option to '/api/manifest/dash/id' --- src/invidious.cr | 16 +++++++++++----- src/invidious/videos.cr | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index e8621b233..f94ce5765 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -4692,15 +4692,18 @@ get "/api/manifest/dash/id/:id" do |env| id = env.params.url["id"] region = env.params.query["region"]? + # Since some implementations create playlists based on resolution regardless of different codecs, + # we can opt to only add a source to a representation if it has a unique height + unique_res = env.params.query["unique_res"]? && (env.params.query["unique_res"] == "true" || env.params.query["unique_res"] == "1") + client = make_client(YT_URL) begin video = get_video(id, PG_DB, proxies, region: region) rescue ex : VideoRedirect url = "/api/manifest/dash/id/#{ex.message}" - if local - url += "?local=true" + if env.params.query + url += "?#{env.params.query}" end - next env.redirect url rescue ex env.response.status_code = 403 @@ -4737,7 +4740,7 @@ get "/api/manifest/dash/id/:id" do |env| XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", - "profiles": "urn:mpeg:dash:profile:isoff-live:2011", minBufferTime: "PT1.5S", type: "static", + "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static", mediaPresentationDuration: "PT#{video.info["length_seconds"]}S") do xml.element("Period") do i = 0 @@ -4746,7 +4749,7 @@ get "/api/manifest/dash/id/:id" do |env| xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do audio_streams.select { |stream| stream["type"].starts_with? mime_type }.each do |fmt| codecs = fmt["type"].split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"] + bandwidth = fmt["bitrate"].to_i * 1000 itag = fmt["itag"] url = fmt["url"] @@ -4765,6 +4768,7 @@ get "/api/manifest/dash/id/:id" do |env| end {"video/mp4", "video/webm"}.each do |mime_type| + heights = [] of Int32 xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do video_streams.select { |stream| stream["type"].starts_with? mime_type }.each do |fmt| codecs = fmt["type"].split("codecs=")[1].strip('"') @@ -4775,6 +4779,8 @@ get "/api/manifest/dash/id/:id" do |env| # Resolutions reported by YouTube player (may not accurately reflect source) height = [4320, 2160, 1440, 1080, 720, 480, 360, 240, 144].sort_by { |i| (height - i).abs }[0] + next if unique_res && heights.includes? height + heights << height xml.element("Representation", id: itag, codecs: codecs, width: width, height: height, startWithSAP: "1", maxPlayoutRate: "1", diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index a58414910..99c6df5c0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1235,11 +1235,11 @@ def process_video_params(query, preferences) continue = query["continue"]?.try &.to_i? continue_autoplay = query["continue_autoplay"]?.try &.to_i? listen = query["listen"]? && (query["listen"] == "true" || query["listen"] == "1").to_unsafe - local = query["local"]? && (query["local"] == "true").to_unsafe + local = query["local"]? && (query["local"] == "true" || query["listen"] == "1").to_unsafe preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? region = query["region"]? - related_videos = query["related_videos"]? + related_videos = query["related_videos"]? && (query["related_videos"] == "true" || query["related_videos"] == "1").to_unsafe speed = query["speed"]?.try &.to_f? video_loop = query["loop"]?.try &.to_i? volume = query["volume"]?.try &.to_i? From 8ba45808be779ead6298526f19f446919d3ccf8c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 4 Jun 2019 21:14:57 -0500 Subject: [PATCH 018/146] Fix typo in '/api/manifest/dash/id' --- src/invidious.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index f94ce5765..b1ec27055 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -4767,8 +4767,8 @@ get "/api/manifest/dash/id/:id" do |env| i += 1 end + heights = [] of Int32 {"video/mp4", "video/webm"}.each do |mime_type| - heights = [] of Int32 xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do video_streams.select { |stream| stream["type"].starts_with? mime_type }.each do |fmt| codecs = fmt["type"].split("codecs=")[1].strip('"') From 8521f0408709b26ce57e707f179721d9f78a6807 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 5 Jun 2019 11:10:23 -0500 Subject: [PATCH 019/146] Use short URL for sharing videos --- assets/js/player.js | 3 ++- src/invidious/videos.cr | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/assets/js/player.js b/assets/js/player.js index 060400c90..823721855 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -25,12 +25,13 @@ if (player_data.aspect_ratio) { var embed_url = new URL(location); embed_url.searchParams.delete('v'); +short_url = location.origin + '/' + video_data.id + embed_url.search; embed_url = location.origin + '/embed/' + video_data.id + embed_url.search; var shareOptions = { socials: ["fbFeed", "tw", "reddit", "email"], - url: window.location.href, + url: short_url, title: player_data.title, description: player_data.description, image: player_data.thumbnail, diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 99c6df5c0..92787fdca 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1240,7 +1240,7 @@ def process_video_params(query, preferences) quality = query["quality"]? region = query["region"]? related_videos = query["related_videos"]? && (query["related_videos"] == "true" || query["related_videos"] == "1").to_unsafe - speed = query["speed"]?.try &.to_f? + speed = query["speed"]?.try &.rchop("x").to_f? video_loop = query["loop"]?.try &.to_i? volume = query["volume"]?.try &.to_i? From 7e0cd0ab600edec014ec98559745ccfff9606709 Mon Sep 17 00:00:00 2001 From: ssantos Date: Wed, 29 May 2019 15:29:27 +0000 Subject: [PATCH 020/146] Update German translation --- locales/de.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/de.json b/locales/de.json index d1b2601ec..6092fd94c 100644 --- a/locales/de.json +++ b/locales/de.json @@ -6,7 +6,7 @@ "Unsubscribe": "Abbestellen", "Subscribe": "Abonnieren", "View channel on YouTube": "Kanal auf YouTube anzeigen", - "View playlist on YouTube": "", + "View playlist on YouTube": "Wiedergabeliste auf YouTube anzeigen", "newest": "neueste", "oldest": "älteste", "popular": "beliebt", @@ -315,4 +315,4 @@ "Videos": "Videos", "Playlists": "Wiedergabelisten", "Current version: ": "Aktuelle Version: " -} \ No newline at end of file +} From cb6f97a831d487a816a68c3b0de358228a0f974c Mon Sep 17 00:00:00 2001 From: Jorge Maldonado Ventura Date: Mon, 27 May 2019 20:54:17 +0000 Subject: [PATCH 021/146] Update Esperanto translation --- locales/eo.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/eo.json b/locales/eo.json index f14ef4667..6c34fd734 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -6,7 +6,7 @@ "Unsubscribe": "Malaboni", "Subscribe": "Aboni", "View channel on YouTube": "Vidi kanalon en YouTube", - "View playlist on YouTube": "", + "View playlist on YouTube": "Vidi ludliston en YouTube", "newest": "pli novaj", "oldest": "pli malnovaj", "popular": "popularaj", @@ -315,4 +315,4 @@ "Videos": "Videoj", "Playlists": "Ludlistoj", "Current version: ": "Nuna versio: " -} \ No newline at end of file +} From 48de136e9dcc942ef97d4c499be0ebdedda6b64c Mon Sep 17 00:00:00 2001 From: Jorge Maldonado Ventura Date: Sat, 1 Jun 2019 21:38:07 +0000 Subject: [PATCH 022/146] Update Esperanto translation --- locales/eo.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/locales/eo.json b/locales/eo.json index 6c34fd734..bdc6c0bf4 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -85,9 +85,9 @@ "Only show latest unwatched video from channel: ": "Nur montri pli novan malviditan videon el kanalo: ", "Only show unwatched: ": "Nur montri malviditajn: ", "Only show notifications (if there are any): ": "Nur montri sciigojn (se estas): ", - "Enable web notifications": "", - "`x` uploaded a video": "", - "`x` is live": "", + "Enable web notifications": "Ebligi retejajn sciigojn", + "`x` uploaded a video": "`x` alŝutis videon", + "`x` is live": "`x` estas nuna", "Data preferences": "Datumagordoj", "Clear watch history": "Forigi vidohistorion", "Import/export data": "Importi/Eksporti datumojn", From 51799844c9d0825829194aede6eb3600f7399064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Allan=20Nordh=C3=B8y?= Date: Sat, 1 Jun 2019 21:34:07 +0000 Subject: [PATCH 023/146] =?UTF-8?q?Update=20Norwegian=20Bokm=C3=A5l=20tran?= =?UTF-8?q?slation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- locales/nb_NO.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/locales/nb_NO.json b/locales/nb_NO.json index 83f97570b..acde88b67 100644 --- a/locales/nb_NO.json +++ b/locales/nb_NO.json @@ -6,7 +6,7 @@ "Unsubscribe": "Opphev abonnement", "Subscribe": "Abonner", "View channel on YouTube": "Vis kanal på YouTube", - "View playlist on YouTube": "", + "View playlist on YouTube": "Vis spilleliste på YouTube", "newest": "nyeste", "oldest": "eldste", "popular": "populært", @@ -85,9 +85,9 @@ "Only show latest unwatched video from channel: ": "Kun vis siste usette video fra kanal: ", "Only show unwatched: ": "Kun vis usette: ", "Only show notifications (if there are any): ": "Kun vis merknader (hvis det er noen): ", - "Enable web notifications": "", - "`x` uploaded a video": "", - "`x` is live": "", + "Enable web notifications": "Skru på nettmerknader", + "`x` uploaded a video": "`x` lastet opp en video", + "`x` is live": "`x` er pålogget", "Data preferences": "Datainnstillinger", "Clear watch history": "Tøm visningshistorikk", "Import/export data": "Importer/eksporter data", @@ -315,4 +315,4 @@ "Videos": "Videoer", "Playlists": "Spillelister", "Current version: ": "Nåværende versjon: " -} \ No newline at end of file +} From 89725df3dc7e285adc5d3a80956f22265a858f6c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 5 Jun 2019 23:08:16 -0500 Subject: [PATCH 024/146] Update CHANGELOG and bump version --- CHANGELOG.md | 64 ++++++++++++++++++++++++++++ screenshots/native_notification.png | Bin 0 -> 22500 bytes shard.yml | 2 +- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 screenshots/native_notification.png diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fcff74af..4804a3e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,67 @@ +# 0.18.0 (2019-06-06) + +# Version 0.18.0: Native Notifications and Optimizations + +Hope everyone has been doing well. This past month there have been [97 commits](https://github.com/omarroth/invidious/compare/0.17.0...0.18.0) from 10 contributors. For the most part changes this month have been on optimizing various parts of the site, mainly subscription feeds and support for serving images and other assets. + +I'm quite happy to mention that support for Greek (`el`) has been added, which I hope will continue to make the site accessible for more users. + +Subscription feeds will now only update when necessary, rather than periodically. This greatly lightens the load on DB as well as making the feeds generally more responsive when changing subscriptions, importing data, and when receiving new uploads. + +Caching for images and other assets should be greatly improved with [#456](https://github.com/omarroth/invidious/issues/456). JavaScript has been pulled out into separate files where possible to take advantage of this, which should result in lighter pages and faster load times. + +This past month several people have encountered issues with downloads and watching high quality video through the site, see [#532](https://github.com/omarroth/invidious/issues/532) and [#562](https://github.com/omarroth/invidious/issues/562). For this coming month I've allocated some more hardware which should help with this, and I'm also looking into optimizing how videos are currently served. + +## For Developers + +`viewCount` is now available for `/api/v1/popular` and all videos returned from `/api/v1/auth/notifications`. Both also now provide `"type"` for indicating available information for each object. + +An `/authorize_token` page is now available for more easily creating new tokens for use in applications, see [this comment](https://github.com/omarroth/invidious/issues/473#issuecomment-496230812) in [#473](https://github.com/omarroth/invidious/issues/473) for more details. + +A POST `/api/v1/auth/notifications` endpoint is also now available for correctly returning notifications for 150+ channels. + +## For Administrators + +There are two new schema changes for administrators: `views` for adding view count to the popular page, and `feed_needs_update` for tracking feed changes. + +As always the relevant migration scripts are provided which should run when following instructions for [updating](https://github.com/omarroth/invidious/wiki/Updating). Otherwise, adding `check_tables: true` to your config will automatically make the required changes. + +## Native Notifications + +[](https://omar.yt/81c3ae1839831bd9300d75e273b6552a86dc2352/native_notification.png "Example of native notification, available in repository under screnshots/native_notification.png") + +It is now possible to receive [Web notifications](https://developer.mozilla.org/en-US/docs/Web/API/Notifications_API) from subscribed channels. + +You can enable notifications by clicking "Enable web notifications" in your preferences. Generally they appear within 20-60 seconds of a new video being uploaded, and I've found them to be an enormous quality of life improvement. + +Although it has been fairly stable, please feel free to report any issues you find [here](https://github.com/omarroth/invidious/issues) or emailing me directly at omarroth@protonmail.com. + +Important to note for administrators is that instances require [`use_pubsub_feeds`](https://github.com/omarroth/invidious/wiki/Configuration) and must be served over HTTPS in order to correctly send web notifications. + +## Finances + +### Donations + +- [Patreon](https://www.patreon.com/omarroth) : \$49.73 +- [Liberapay](https://liberapay.com/omarroth) : \$100.57 +- Crypto : ~\$11.12 (converted from BCH, BTC) +- Total : \$161.42 + +### Expenses + +- invidious-load1 (nyc1) : \$10.00 (load balancer) +- invidious-update1 (s-1vcpu-1gb) : \$5.00 (updates feeds) +- invidious-node1 (s-1vcpu-1gb) : \$5.00 (web server) +- invidious-node2 (s-1vcpu-1gb) : \$5.00 (web server) +- invidious-node3 (s-1vcpu-1gb) : \$5.00 (web server) +- invidious-node4 (s-1vcpu-1gb) : \$5.00 (web server) +- invidious-node5 (s-1vcpu-1gb) : \$5.00 (web server) +- invidious-node6 (s-1vcpu-1gb) : \$5.00 (web server) +- invidious-db1 (s-4vcpu-8gb) : \$40.00 (database) +- Total : \$85.00 + +See you all next month! + # 0.17.0 (2019-05-06) # Version 0.17.0: Player and Authentication API diff --git a/screenshots/native_notification.png b/screenshots/native_notification.png new file mode 100644 index 0000000000000000000000000000000000000000..be246d1dcb7f7e4766f4ead74e9e661de0c01e83 GIT binary patch literal 22500 zcmXtAWk6g#(}l&|-JRkN#oda#ySux)LyMQYew zA~&nZyTa2VFu#am&W-Q;Jq3Fsjk~oQoPYU*o=wwZ?sY$zv*PUC6Zq#~5_!$8`NxPf zIypQ#Gy`rZqjqhzPJ`KIr*Aoruy9ac7)(4KYaW;Fmr;vnP&nG2+mp6&V0wBwO!n>V zZAb_NPfl0hpZlA`>EQbnYkCf+^)|6^SfPh?cTVf6YRxLG={%ua9uaPC&AO#x6B`?w z_qSJy)XdC>?P#*8Y)%CcLdr%vzJT(FU8Zs=T6}{Z3 z%daIcydT^T9*5_`5U)Op9|8IjEs!lpv|RFVx_1bCqp8pt-pX42Ll=&kW2w_ z4TE9dyjoI1eaY&HJmhR)QKCYR9Y5G~n?fO>Rco+Pp-#s~a5Ta1d7_Xfn7vtw`6-iy zgwLfw!fJLyIPe*ngult=Ep?1S@}(q&?hX?=&l8%E%jPmUuu8QUmKcVMn6>BSx?HvR z$dCat3>Jfo&!yL_q4&$=1|Nbli_~g6PwV5K%|MNb!2i3PSQ5*iJ0Lsm1E1R-{L7%r;|?AScx>Ze2BOh; zZRa9b-o{ybdM(8+a20!zljGyZCngqbxV|grTV*ga7?^(SM68O{ko*^A#OM&rPRsSC z6Itx2Pq!@P_sJq}q6#ofW|yqYAKmswq~dWY$|!kwTEGLn0(~(5-qS_z*kW94vg6QMN`B7jZFx#UPgG8fgc+hQUO3GfE4iGry+3g=wuCW zCK#&!np^@g$c1<4f)He(z+Z=2lR%0Cr=Lkgr}Dw~^Vxy-2^NJ5YhrKaR7yx#@!!$T z#wbR~s+BEN zNefHrk_MWXPJD=NKSo4EEZ4n7!0`8Aj%5SS8*zw$xY5W8OTP2XE~K9jB<5bEjVlRq zaeX#AQBYbWd`(u9UriKJNZ>sdD>WAc`?Jo!OGKk)T^%kN2XFlJWbFDD4HZ@3sgM^N zmW4)I>C#kR?*jfjOO>><-C`7TVh5L)_*7+V-Y_nPeFdF1Q6kaD)Msqah{|la-^5gb zCE9r6Hi45u^7MG0`WozNILJ%T>z@bb^)`ys(W!4Av9Pft?)F82?Ck7WVu}puX6eOE zO*BK2W zl8iuJ6eEt{$?TjPBZ2#pDr6bHvmi)gb^*i~AcnPA4q+0;)oZdr_(M#J^)MYFiwU0g z(4pV4A#dZhH(XAaRTZ3*c@Y&H98AO?l!paWv25Uw-8xY&cK}G{r~&=`=f+T!a^UzA zT!arf7xgO;MklvHw~&QlY717|q%gAztdGp@pjP}ZLSdlmL9!Lf^8%8{-fUeM`fUq% z;n>vRc+WF}u7`(L443uspai!{uB7h&7qlVdy^{uvYXu(&(1C$}aA*l@Ep+@s> z4wz7iQch&qq;P@XU2ci>s>{>}SSv*$cTgRFMh2S!QT{d48w>}@#tnN-v5NdXLKXkW zk&!Ns8G}O{NFy%!?`hw$0U+aqnE`xIzc@GA%XovY%_J_8w1jLaN4k7up*PON$fhm> z0u&az1$uZJJJ0$ZPR`RA7-EoC9P0}(Fi6t#g1 z^fD{}pCVy)+QdRxnHU{S>2}~_wxzj)Jaoj2vEzIG7fSz46oGk>jdWNb?upE1RW}a;P9I z-Irk%3suOm;oB%F&~UkjuZ>hdp;i^=-l2hDY5?cC-}k&OFnI62pe`+os02;(6%R?TvNz!6pEsG0l z+b;~v?Ylpz_&=St-|h)WSCRgy4_djjUJf86PGQfS++BGeBQh@fyHC+00mi$Hh23ri z;Lu1InlS_Zlx--x4^wS7#ih)kW3yhM9fA9IqCu96lkCi4gXHFZeeI^`*QIOEG@))? zW|4)Wm72GIxLc)a@5aGVD&C+*SxbzmM#b>M&4Q+6A5*oY$9b2qfjCi`BK;WIsyMq; z%btlgUb=X4Y!t`kNxGQg?U_4P6rLekIh!Zcu_8K0{piy~z+u-znL^Hi#_5B|8~*L} z_yR@Qa*47?F1K~kJms`FZGf1N-`#33sJG|dBKYN)u4yKIGQ1_dO3SwdJ@?M@WU0sJ z_=`-b0TQRPEZnH$S}U*J9FXEB_JZ{34?LmtE3;mw#q5_snPigO$5FZ*9_K<13Xm!l zMnzO@5MAo$<2$8Wd}2;jbYT>FSIX0_YB~v^jYu1Q&S*bIJ@YW?nS>3288ugyv2N8; z_Jlzm1gD>s^?{CpRkq~U{M=IXxPkQUv#&gA6=@Bhx9KexZNDy5Zr0l+L`+VNj?Uvx z5ax<8Vd_s<#gCSZYP)|t#Trhspr&)L;UQ!9dpSy++}HW&z{g}j{?UJmSP;{FWBl0MYY@_t5SCq;oX3*cwP^^udMM$f>wOiI5&ZZLko@*XtAY1P&`(;FWV;Bc`dj`p6cPv7v?H z5;p7YRQ=H5hVybF5_~aa3;=q4XTyhNh2RQT;^T^KN7~W(VS{0&drnQwWc1Smcr+XQ zcAqofi}zEb<}3G=%*9-+k&xfau{9^V-vT`z9cn5;e*W2NvparwoV+6Ix&alltIMLgJcqOrMAH; zy+mDsblf;}7!&=snin2zTGOXRd3&CtX13@+OH4k8NhVV9sCFi$0(nht!3phNP8~-K z96{md?dtmY4$m{Dj7vpabq5cR_sy=I)bmYwB~&yNo0T@Pjb(h1bb#r|%5lZa&|!V$ zAdtLVksh?w=GVn0vUTvyCt#p^()iR(LPbW$#%8~~{Qhh$nB9>OH>u0(m0uGu)}6;* z?AhTpsn_K5`>fS@t6Om&?;&#BL!HZZ^JV#Jw|L9H&b86X@TaGKy$gjz$8A4WGd}iZv5!Otja2J)c@XrKwgvjdex1%|NQdrR1=;##Rk z=%wZ&gNcO%so1f_hLL5-mTKR4dPggXeXBiJ#0hz@e>r_R>r4ZGTHrN>ABW!D3PC|b z`b;W*?FHi=1!DVv2hVKVI&pIXjEcJD<(gHt_)Q@fUtYg1l&{uk&g0Hv6q*H_;Y%5J_vHevwze zt&!~wzZ>8A`f&C(U(Vc%E^s?%I+Hu84{Xo{F2}Xhw-F;p0x2Ga=pkL=G$@}eC%=03 zyo52{j#}sAXwTRR2N>0V=6!!U1R`Md+K?mZA(!#?_b=7xHd<`=M?^%lN+%TPhGcwT zhZHEXOJ{Jmn&hwdzM#XR6+JxWv@TaGClt;g)2i0FnOBA%h{_EA#dE)#+y_&oRlVHe z%PGDRos2IgQsv|RjX#iZ3$CZu5J`{V{+GW0{aKEH|NR9K4|&Av{QGON8sMEa?-I{A zz;(LTaK7hp=djsc47w?Zp~3oMlT5oV@9J*p&%>#1r;Y0OBobGO7UvaA4AQr4I5LB7 zm+d9vpwJR~&D8G-eVztauMZb(TumgEL$v~zZ$F$iytjhi-~2BeH#&XifU!PMli9q< zYMR9L1^u%Z-9(1%*JUDaI()%zwHqC&U%niz#zWojjiw$s;J?g$5@>NE5pg2!H1)u+ zDr%FrC4k0hwqNu5-Qwsm)j}d*R;3wk95y9pw(`RgynDP|-%PhUtuB$`%P4)cw2+mWc1c~tIsC#jx0UaYA ze$s6#By{k#`qbYWv+mWsJ!tp90lIw!c0mZ=_^{~ngal|sztwsA8#mH+gy+dZ zA!?u6P`G^Qme1?mgUAcI6%r}Fl^EXf`Nq$?`i?IRJ51H%7a7bsY(br8D1Bz3khX5w zFDuq{ozDzfc+N8gq2Vq1txkQulM~R!-oIDVr+nWEu?P+Ve0+Qi42ZuiTwPt!Dd*}n z`g!KAHd8#!X0Zp82}t9FidW|$iI)`7&XVHW2a^fO9|Knc??>r$vXJ^gE3DlwZgT`IVhy34y^8=-s zw*(wd7L*Kss?@kao_;kBvYqXGs*;En#~fEoBK(NC1*bGUXj`f<`u6(Vw>f;6)YBt! zzMR|PmFV-wu?%9)eyPU)CXsiw#WvjI!l=u|*m1ox`^yJZ$MvT__ZPz`>UB>92`F5!+doEe$V(Y)l>tzVpjg=PxF_-lSCo9p`9+($V#0M*V->D7 z!b4CT7LKG)Homl*?v@YWHe;X$e3q@7VjY=OD}R}Fj)sFZLRcg3_%h@H930+tN5+PV zb6EVb|L`eO%>Y5PO0!COpfY7#EX)V4O&Q3heZ13bzbM|uHPoZtIV1=TN2UT$$D=Wu z*?(zwx22Jt$oiPNIVhS%^C>fr&-aIDjdafZ{QTWLx1}>3wJ61WyrB?J#&wWEe7wXJ zMP0W%W1n0hc{7q$sSbVzNJT+Ysu{O~qXN5U;n+AiUgC4lTC)Ty4^v6Z1v-P`dl=wy z`{m^mYqQ-VBAQoYnc}g;M1;L&8?l1v!#qtt64#8BQjcq4J9r}2M?o0A`rXAr@vwiW z&|9p$r96RrdlOBtOGh$Mt^D*|$z#`*`G|{C)u2>?0G*#6e-mMb{~PjS+VPXmcULVh z9**ooO_I9aXh{R&RiAjib+)^c#kJ>|L7+o#4`(8grZXGAV@{3t)Hz-hGcqLp;4Y@}~X_vTB9&fZv%SAj~GgJ!5{CEsM4rAkRtqVHsh$;=WLiE!piW#lq z&*MM#O1{$q4dZKdwW-x9L4Z0aZ%~y%`*oP_&85e7p&PkwgU)GE)Q&|6ATZ{6FEle` z%(R=-WEFz_`Km0Q91*;XKHo`I>y_n79pl35B{ zF~dve02X{#zSz`nafHHf-fDnS(hP89OIox3`Qt%#K08@2#<~K(Nldg!Ft?zfz`HA_ z&Hv&2wEd<;ZM36!3jqOvrPyt4xOFkizjqrpYBel&*ESj2>w}tBu&^e(ceg|-J?bXI zd7d{q%Z8H_7P}MPdfftZPG)B2=Q!IGNv%AHa3(%n#^I4K@IW>n6ltm{gAq=6O~i_1 z1IqL(DNI~i81WH;l;(OLas6@jaOkqyfvk_=a?z&Vu;H2v>NYxdvB|_=q#HkQQNGNH zXum*rXWQD+?~>tG+#-P*+V9uK?>*W#Mdgkv+?myX7GHVYq%!#5ooMn!LP0@Q7-nrZ zDOEbBP^$Ac0-@W8-r!P5G|@fW-wRENaBw1v%foo$wxguaW)Q+rd+=uv)+&Jvaag_4 z{ppO_VPRo$SOquR`j4Qlhv&QFa8fBgdZL9owB+=a+at!gK+Ps4>dVgU&{?+Uwkx4N zY zFV%{0m-p^K?P>$x{S!@K{sileQ&d+1QsT)Gn|vFI4a#64$}k3@%Ts=^ODMhRt(;c# z7K4ZG1#NGCv2;?%R7Jk>*+!MO>OTJ7KDw@|M${DK;>dnn=enS3eaw_3Ab+8Z=<3A} z#X`MIHOO?izrboY(w!*+6DdMryK{a0+UVQnEV8MVF{!POZ-cAJ+K(FD;|eBJvRrGk zSol7BX?#@5421VKhZ?cu7`V^kr&u6Xb!hngH*w7Zm6Eg%fXM(D{o`RY?^8z_m0 z?rsOtEjzZVI|5>sqD^!O2QAlqhQcg!_QQjNC;C8L10yFRB^RBDU>AI1;1N~R6YZ(K z@3&$Be(%%O$U518!Q$Ze@A<)`{1)(rh`ECCxUAB_g!E9VC@%6dC353!N}vgl3v`DV zGOIg1B%Zp|rGXfQIMvLD=+)WO`{{JX1q#8q8vWKNPV51e^U{)n0TPsAf!_v8vSfj; z%}b)t#tQkU)?7+^m!GMQz7{8|A$SmxKL2J#HtNOWHNpoeKAO71_sG1x5!LiZ8G37c zv>9TVB5~=vi^Uq&EuKDkdAd<(z}$|6re3NcRwM#E(^r?_KkyHad5W%n4iuCtkf%-;^j#qwrlzarG{C{pl!}i z9r=$+Gqv6>p{C#$`))cse{1PUxY`T@fxT^c_k(z#E%0Hi2UBqlp)sdNUvMx0kA+cS zyf1Fpy6GYhD$W<_X!9d=+7|P}M%Pwoq$>I#i7;zt&@UWE3@QuAn@=F_$Pq3D`2Oq&t$tDW%>vhh?9(_H0K5P?=f=c;9q2FWUI++q$Oa}2X#~BG{cW9aQr4}YHF9SeG9NGR_AKAJOivU`vv4fG@^2B&>5Fn^z(<#1o|3HIli*Np_iFRT;V+nx#az{ zXe$#Apw83;&k;l2X>ljbVYLY(0lJka=y)|A%j!V8Ny;JT71=x< zBWh?nh6dAzsu#dvoUUXIs`rh=mGCm{rZ;xsWT)U3O0?WISBc86`{CJ0i*mAHn!^W+ zUYX_0_FI8TqB}1=CB@?{KO%;F%kJ|waZm<}an9#df|JAP)IY;p-2-1nBk)NL3m{|s za#u}$cnb8Y^+xgNy@jhmuCZ0mGFwg=s0vj^ z>R`aT=lhJZX;%2SZF%VDVUGXnvb7 zh?q4{-{6wCG35?_Wq*FZzQ?7Cy^R4ee$cj?^ zLpmxdieBM7beagsCmK^`Y8EoiW^)y|XjW}-V%_C(2U~^C`KdlaynB;+O4=oSsJbIH zkomPbb^rxA0sTXflg)x$ECNHe`0020iT$n443~2&z|vUyr$u5*tq>Kp4_uP8csVNv z*oKA4l#+!iS5KJsMxil-=}%AAR+i6I0gW8<%(Pvi^bLACF{I(TZw^jopKVJ2gv&wv zj|*`A@^n-y_JY{m4oY*u559Bg7oU3WNrVs{tkS3mDq@RV^&+3}n@T8NP@ z!w!0G)uWTN3eHPLOa%%O5@Wnfzq$$Irp!x0B#$z4G?sqH+j(xsqXQ!?W=8o7huS$j zU$<7c9?hWTXKF5yo&c!xN*~05ajtMufn{1F;Jgqtwx1PDZ+ZYY-Hv@kq;R)V=ATjR8(_ z=W!n8r#&!#AvBl*P@o<0(03q(Mhc+Epe>kR{gnqtQGjL~I>3N)x{ug0$M7xYj4mOy zq$Mls=rI2x*kefvHIu9wyOI4ON<5(ep*s?9bY}q>6d6MeV9ABH*Q%_izn9SH-rB`V(pWZv&YQb3E#ylDMQJgzbof8n@rx&YJSERq zscfwevy_i7O~Jz{WVn>DV3!%WpZ_E!jt~`c#BHY_+jrnjDgII-td=gS)h=%toS(7| zVQJz5A4aRfd!Q2Tk6lBK+weOloR|jw{-jG1oyA&0}nsj+ZBIK4uD9uh^xUSAT8-H zuzs{x&_XWa1Tz<2UI3p%SaGgDe=Z(vta}~3%Y?0u1XL#j4_pWFt*je8*|ng5`yDm& z=l*hVFsRB-+H|L!@#FPlH7F7BS^!+Pnd1ErySG_Ow z=)_jmR*hT-6BB~Q?Zn%(fXD?iZdY`GHM=<`Lc$^h5z(DisA!hHN>&vwuj5LkQrAy~ zKbb9K#~CTm|2?o8yapGBdJEfNIG;6V4o$k;4I2=kE^4w(ohw$%8wVp8slZ}#<%2ol zVA~rY%H}dZH-9Uw=qLX}$dz^R-s=Wx3N!8PN&6&HW`}(8q~NbNwOY6y!S}mfl`loiqNB7 zxIaanoEju)S+`JnpbUtMo@-J3a@?aV(jDVdW zvSy_(u+ZOVw)(M~#+Ya%9XE!VKU~h9&Ey23jR_q@-g|oyU|#dHnTl(uAf`TK22_73 z%d|~_gbHVV_z7AbzCjUCsF`X^f2+9+_<@lgnu08B%#s$R9XB<(blk90t^yp2BTbOI z$a0Dc9PA(qu5cCQLlm&t0FaV)ayc4*&qXqN7+5M%v zf2AO)Ym>txqg5L6ws_?}i#Jdk7sGUXWtX0sVChZ65rmvJ)+}J@Wgvmihk$EjON7x* zDO|kx81Ve^`c05ZKjdjsFnG7K>Etxh-I`}?QhvNCVGB*K zPu9qCdCTP&m<$!jeDkL1lPFSge|XNX7iAw`lzx{pyd`Uex*vAFT86ojm`es|*R=|J zh5=hvvPS=7(UFp?%u2q@kn@=D$fFmn*hmMk#naVj)l~pxJYSOU@ZZj;b{^?Vvl5_? ztTz=E7A$ZW*!;^xd&e0xp8mkN`DA-zF~?M)yUeAg-icwRE0me)?xiyjDB>dj3QI ziDW@h?6P$CJ%Tse9cso22oW966xUV3p_YbCP+Z$)gg48*3Dz<+18u8z6CW4dXCItu zC&6#R+K1#T9jyLwAISe)d`HQoezkC24q3gFF zp#vj^@h`+s%8Dfp<%uU|1CK{p{f30`D!%@pVwsZ(*ml}1O#@`7{IR+jfxHtxAn6S3 z5q($8C&d1Ogu0Pk()q_;*QJqoSp6Tn5Mw?YK)r(b7j*I-p-dFP`v@OiaRSjh$Ot zFZF)I6z*Q4>6iC33sfL>t)-n=Qr^Cx1J22t6%tHsvjyjd8ILr^eZWT`m!9=!(>ZfZ^X@{Q&SymV zaKSHiqC1bv##POFjbY@rW$!`&A^U6M08x0@;b{-dTnU$1v*tf(uS7`oDVHRiuU(WGkh2Fum@A#>tp)8R?7-ZN}!Zoi>#e#z#|Rf%5HX#b09W2C;d{(@0sSFR?0LOUl@ct3M@R zcv5z!#rKipPnkBj_cdU90jii__`7)*y z{M9B{E*;|dtxdrqOpP7CRH}(pee|s@f%mHvy`&&yCTFK5%>e~N*O|Ou-KtpHcYxg* z@X@j7+5}^|sDI*{XnVb=^}5@W@CZp6f29d2jV*Xd1XV~B6lKa6K@^>uy{z(|v(XSP z;T|KYwIBiuHSMXb$_9rbXv#|e#7Q!xO7-rVv7cl39EMyPY zM3g9!0 zsiV{{afbeg1h2HTfN$nxvfecqeD5ma@_FZ?0S99gEjSR_IdvK-sx~E^ICcq-=E$dI zK+vY5Ih(Uxx3I!f%sItCI8iPJe5S6A{!7|w(JS@5|B~p`W1@C2U!LHPbgcM4g-lO> zsUYR#(DGdxbLLGtkcp8v2`D(*c(nDiBL~?FFIUpjo2DGrg`X$IMrvmZm>06xZP|yve*KC;j13&VL{T6B znjI2g2=#I3)&Npdc<9aoz1WnFd*<@F0suAU0f)qGFjzPfM+M~)nmw9yUqt(8<2|1m zB%_O$&(!IB)=7t=tbMpG2`1Y4tL&+e-Dty556CbuDEIQ^?;=-8hA18W(6 z5T9mB>Z9_~pOB%1ht$?a2}`o)ysF)nt-9|}cTQlBNMw2J z_9T_qFVw|8))&?2BjpyO>0kH*nnk*=cG6F)%Yxj>a7M5&0|c+OOLp&Tq;xngV$Xqg za9a~h5u&xf_%)Ns_zJDpTATz=a&XS%2cDQ1tsZPQOfvU5F|xtP^@isY{i53TVS>N4ehpg z9x(0*K|$YP#vaiJ5OlZN3{+#6fiTZDADO$RzRp@+)pw4#&t)N-+V zu9}o4$xq(TbU^w@6$WruHbI!*NugTMBIcm}y98YF18@|9xTHik#y?CE8=qo^6dBpb ze!2-**brIfQyR{&=JI1(kXbTAi~q3 z`Dvm|JM4g=i(WvRDp!KpphG{R{@W+uv-pSPIAkQ`OZ7^PtRei=%D%GZAJrzN5jaT8 z;u*7Q1^Z`-(?T2*qv#2y*a)d^9vk|qjk3igNo(gL;IudlC% zN29hgzsb(Lsvj-2I)mvFWa+nR3_?O(O0Um4P5Wn`se*M|QT4tDUJX`%Z22s<(BeOx zr)?B?w{-V=%lD!y+PCHX=K7|?<9F2OTX3zR+x$@Jpm1l-C$;NkjPWA)(bOP7P~B?3 zQnS@*%jy8kBkqldN2k1}%u1@nJYUy+8bC{GpSqv@eZ;hu;_JY6;*AV7lwXYVS%j`i%>+Ht+f7FQ)6zYC4YUrKdyp9h^19BTOFotYlE&k+{Zj2630uKmhDYZUZCmo#^u$C!@I;oD$Jb`+PlIk71k3()otXghL1OvBt-#TIKh*nziQN z3(bsnOn!)IQ$46;=Q9;pOp107BU{0(XETtTUxWC4ulon%D<*abVgLN}uQRe?d%<~7 zQ=>O6xO%QPx>@pudAq%!3WdNhB;CE$_7ni-6LTMDDBof}F zownqmM2Ra2I8@Ng1EbbU;BC1?9~2x~?ep>XK_tAl*Ck_=lO++am$q<-uC-$ShfpXu zuivXUi3Ge}OxN)Y!v>!3IUJV15(sRDqC9`EHaqICe`s=AMC@Z|GWK7)KkM?l-@77> z6vtg+!_MXnek*eBkqS2zPJVwhZ-N$1#h{W2J4PkrZ?fATo78XVB`nofqA63#4ufWq z2lEF%hd`BRj%O@T@Bsz9PT~CtI-s8K23XRQw=V1JkBx(TUADSuknqBM)#v9@5OAeE zO$!Ts+zr}%Fsz&qmyM@z+B1>jLW9<1ZZEg%W3Dm-9$4By z8?(h?F5BDNxdLA69nSgT9DBFN8y$9wmGtlo)fyFF!bM#>l+p(ZVBJHZ#dOH2H1rHy zEBP;3p_}xW4W=(qIV0ok@nvyQhIoa!kV!c9u0`H^Qz1HVJPTh(BKH40SJiQhrm5-o z7zt7LCb(dbhstRJz(r_@oEC>r<1qQ%B9o}!oM@B!BkFGv%gZ{!!S8AQ4=y_cG6^cZ z)96??Jr(N12_ynuFOMT5Bj94Z>P2__5_9Qy%XnN~r_F#`RjO~YiGFPPAs z@x7_Kg##UjDx|CRQ6Bk5U6&-Uwuyv~weJ>l)C+*9xEig*TGWG+z$lq7Q{Sx&i@bB` zNb~9Pf)=)s2kE@5q+u;h?t^4q*tGjlRTj(Tu*STf{yg6iWM`*E%4%YO6(yaulF0$MIaeAs@vf!DQ4HX5*xZ1CQ` zoq54Vcj_=rjgN;rhw5UK)r?Jync2C27WTcS{^cxYZ<`$y_zY5qC64s4}=wLg|_igdBwKHoV-z15|{vhb|T(4XV&B40pQf5fKYqp2`-$ zGoG)pTLNo~7l?d&us7;*a$nGjj^;Wj?liy@bi zqut!Mg0*E~n_%G}1iKG&kABOuhMuMZ8}#xD1(Fy^xW?{26|44ax00VUvQ zfLOo$bAiWd7=~>0F-BSxKuXGo0=gGl1ozMI^=A6@&z-xYnKShHi~iJE4N!518rKbPytWg=llA0n2#Q%Z?)pRRkKwO zY0YkSdxV!CoOl1kU+%fQ(QPhMES+d$%^rw7xScCHK*(bg0*?}bpD*N%N|Mk7!Q@06 zxJ&ravyRFOzbnb(19%@wBLi0%#busy<$P_VeME?XkABQ(`;v5E0DVa&@i&fkKlGE3 zHGaxu_Fr^56PB8!Rs454!VgV47$oee$hri(>knpn9hc%69nEwHwrfeIVf{Vt2BW8` zi%{+=Z2=KcxBux;G01a0H>J~#sQ%$RQcBU7Wr^MxKJd-J$e0I=ExHovXJ!6rvjo99 ztA^+hjK8AF5OR6zS@0lYj_OBb4*c1@@d;$VUoaEs5;p7r3MKWg(3P1T4bsix& zDb1g?yNr@)Mv{m>BFnYnR?;g`js6E|l@S1GQ~;)gr;dNED7z5b1n9O&D5Vv%(tg=+ zQHJ6Hj_?u0f#(Jglz*|O;2FG%B=QLqQh0p))QsT{f?s0H73}5BO}JuGa}T{Vt)v0v zv*cg5J))6|Vz4ixwXo^nTO1wB`6kAwvR4jA(YTcNf(_;wAyX>n;xZ#9= z5j7%2w#I)tPXh|jWI2NNU5fd?wjf<86R>7NO1VopU)Y1TA;y+oHtgT1r6BZ5P_aWzp%(KR%3A(#o=S z@K~=@q|>z2Wh)O1ItCA&xWg6~4*$Ykc^2o1e-+zs-6c|DXmD78@d0wGc%jF_{_Tiy zXvir3ah3Qdz|*x5(f#QD_mruT>9}g=agVO2&CfNTbYw;s5MyL-6(#goKpiamHN{}n zFYXsfH1K0zQe*UzZ@-+L!V6%yFQ4igda2Yj)K_B+eTmf}a(Mt8YFEOqImFg4$H~I9 zGz`17cXlc#pu74~Dct@YG@Q`hYn-iUo(xsohSn2x_((=A7P6adnN2mFZ=~YDK?;fM zzlGrM2%!rL{utj*D7$$-CJyhHl&sf4V9OG_!h|SP{`JQg31sog=^Z~9B8)m*{ah_V z?0UK{Xc`_IX4ZYqm88t(v6OdmVo4spxw~W5bLH{PxrKnh3;O$i0i+JV)Z6#=tLa@j zC9^21+qfubI9~f-r(m33rB$($ZUwR~OHUZ9##se#c4CJm5pboh#caPZqps6?<~mlroD6%`dHmLkIC z!vh+8xn`qPwAUFpVuttKgdC0a^iIC{B2DE9g@+{)@xC1mpoNEp32KUuE`6Y#!-B|A zqR<@E*Dhpoq^~`FgM$0@`poEL72c=vk)hIftGg}wDgcL0`H)P+b!R$}XgXIUKQ)yK zBi??o@(v6zzyh5jm~u%iuY0gWiA2!nGRTiVZ{?7XoA7>!h(?&|`}DWJ%7G?^CYeGN zprlNz{n~xxVLo2`;B6t8Ow3}~g505)!=($R|9f;xm`(r9J(6XtOiV!FWRf`P=kg#G z9mfa#Co~U=-B}93rMI8>xW9kOl*-c?Tx;ISvJUpcp^~01Hqrq~TF;h4Z&Sp=1-#Gq zrbu>G4bS0sGhVF@5;pwryLmj0stGv3$(Jg_lhbvfM6@v(Pqlw*EtW{0%23p{9fcF| zi__8oY~l`Z&9XSV z*iR3kTBkdmhjyK+#Tu~wW|GxjB20q6$@yWb{Qm-u6>;iOcpgB(&;~kAPftI0?)
KM8#f$?Ck9SO`ZDJ;ln8C=j$6C z9i5h%dhGad{Jrqr(AP>7b4LKr zHB76!cki-T+ue~bql{DH+`pYj6xE>b|;T=17x~Y8b`~{smb^PeVF(Kgi zkYFS_b?i8I{(Lu!Q(`cLslVb%l_*eL!J=28R;`+)y9&isQR;kHsbFC|P%tF-?cV+8 zZ@=x_wyi<^`ry+=ix<04tJCQ&Ub@tvL4!4`S06fX0I$8nhYpP&I~GM2FI@`qrOQs3 zI1wz|yH~FxhYtO|VZ(ua`=(8uiYI8&C!Zjn7Ar0R?%uL#6aIGZ*m3Ik@x==l!i-41 zdw1&RpF0%67Zo0p?K>WXG&mf^3*hggM61z`4$(9_2w`_RJac z=FC~NWXY^KbKN;#7)gT%4SuA8#TqV~lnkDTSBDO5 z+O#Ps=C8m0qB7xzx_13~WMm}B7cX5Zb3ng-coP?DG^>94sW|;k^KtjKY2AAMoH;lO zR&{7zhxYA}zoi(n7w>fdjyJSZ&1K013-|2PclDYzty{PH^pi>WGW+ZBk^Tdphi&du zEr=L*b?JzB&%V9*3OW4u-$*!BUNqA?MC_D>aV9YA@nV1()v9HSU_ggXoxrNJ80ZyeLGMxtd>tZ@lbcKJp0hjFTeoblF+(}vFRRs!DYNfSQazi0R4 zPd|Nm&>+0k=FXm#o0s>;pBsx-L0vdt>yCv1-?zixorF8k3D*V^v}e3%EzF!U`H0~~ zn)KdceyVJc3Rz2Z0*Zt0U*i z)vJAa_eSE<<;%Bk-$nwaE_KL3p5CQQB6y9d9Li`74tD%-M3)^h@1>;R)eGLIi@TWs zkwKC+v8ht{as*>s{^uXCheIQ8+_-_+9$*eT6mhe<62&@q>co_Rfr#Doj0ba5(1&0_ zVgr5QGuIs=2EWl|sl<+5J3+y04yRJ3$R6%7-eV7jgSs;A;xTJbzX5t+j~+eRty|aN zprB+AKeWrzfD&;H;QNkQk(+6mOg83&57w+*10sf7(B56(W>FSmB+xqtX@6fI z80lc))btE=*11!jA`5h>y+7td^lGmEWo>Avt;dhv;EupwkM7;jq`C9wz52!*3++@@ z7M7uvdzzqVv}h@>a5U87tN;CX-z(i!KyiwNw`|=?wXiH)jJF*S#_JgyGYA2iHTJ2; z5eq@pf-ekont`=zy7aN)R)%vKMVdEnP8Af}yZan`2-G1|iUbaMO2lW+p2e6nygVW+ zRiac;zZSC@Z+G-P!V3YjlzH=P-m(QUc&QkSwtkoL-cnLhQAK2gtplIA(JTy$ih7qY zkGhlY(y22we-I2%cjO`C_|4*GF*n8h{Cs^>s>GxunN)^*s3N~Z(_l#2Vb4fZDzL_V()0bIz>U4lK;9F+4nc?(8|edi2<{ zZyz(iBGVvTr+;2-LzBkOUA%M|3O&6F4lE2hhG~q<(zOjQc*>Ko_i?mAk?S{XfX>3g zLP|5P%dQ4K-sWh z6B;hciA3)fO`y*P1#tyG{p1t+D7X)EV)|zKM$zH<#fAqQ8ys|a!!M-Q)Z3@T~bpl(iHUh=*BI)h#$5JZH9 z_$b}=^c_EOeA?$CTTTi3 z6i%H!Lv>xce-)gvFr~08T#T_BH*enj>F1w6ojjRtUh`(n(0zxVM)wCyB_$<|e(yb) zy|ZV|WPUH=X8v~GJj8mUUykmY!zb)lUw(-&WYkD|lj+(W5+E!p7!EcH228;iM9>A^ z84hCfE{1#HWw7tSfyo*TI%K`*_C5i4{pc+4qV7%U93!s`cy z2=zvxWr|AxQrZ_$x)(_~5-}Qg{=#_#B#ay};{N?~_zJ4W+WKlMW^2~08B94Q5#w(D zWa6ZvyoNxN7R{TvXcg!c=Yn5&ifwO9+dI<$cLloXupyHw3tleHQg)%d=pkHw*eu_C z^%V}GA6zPx7L_TQgLg$r+WnOK8NmVm)uN+>u-Vz{-2D86qhICzZ>>Hx-5hn4(iP-D3OP?l{fPeGbMGL>4J{`MmUIiB{OzA8O7mph| zwr9_t`}glhrwiP*O`0@m)v_f#t z2-<>y+_QUka4~QIW5Q$LR41K@2My83^lpg>nAqr)gX;rhM&K02J&(aD7>`#@5yZ^) zoH})S_}e48ckA}K{T)Lc!b9!A!syS$xw|><8p6~tc`;NaCMUVsJI8Di(l#PKb^7$~ z-Mc4$_8DGlUre7q=%tqc2ciye|1sYf4kfXR-cS4?f$%lFoE-uu5IIo4UOja0zzMr^ z*Dl-vEn5`(hAgVQj5z}b40z+sH+%Q$O$Sck!?37M^avwV{>LAFD0+y_K|BqZ7laF{ zg=+$)v7L>rRNyd7VDK-P+I?G?CK1GG77d)h-H%R;MhzRH$2KD)vsSH|bW2<$;%_Bb ztd@E;9veiLd@l_wtdQBL9KR7^!O5vU+sc7o> z_;~mG)-YCW^r(?g#pGg~CwdPMQhws($-{pi!2|_G<8SD8cs)_AYE^X3{%`7(@4x$w zM%|VPTln1Nt5h56*Q&15>0PwP>7l<4|21{m7l^?? z%(d%YAg9I4DU)_h(0p3zid|tu zFs5$E;KA-!M2$9XQw~wRXhW06jp3h#mjEvb_$6S+P*$D$&GS@}VqrvZ;d7HRn<$G4 zSK00se&CVF%Y$z(A6FAZjIG34K(dHe+q2i!aqWJ*nuA&!5#yU3e@LKKdX^2mIUPh@ z?>gmTxGawyJGN!pR?P4Ma&fJ}>qjSyA3yrtci$d462m%ZyO^#8_c2%)?q5tdL1(&| zu(>37JiI^g@}lV2{!T4Tx42Ul0IOnrGh82dla<&cghn7n4YsfY3!|V|D1=9%VVwsy zb-ub#BodaW&_Mv6;+ZqP#tCAyD@MLL9DP!|W#?aJ%bi{YJe+s#B;rE5a}}7{DjIPK)53wq(&F40L|} zr57={qix%^ls65N{=gyi^P06~ob(5uEjpc<8p?r30+y`PW|Wxezyjzch>yR4p&a;_ zD9WuxwcXFm^!M>``VAEx8hktPZe;iaA2B=?l_SH7Y9Kj4!0A;84D=5PaqB8nB!=j7 zLj(neN+C@3_19jbtHv0%egg->KEPBD@E{`}mfc$;M!={?WI9ulF5)zSF&D6fXU(1s zg8<{uqz?x@I~6WVW}jZYK*YcQ@duRj>C>kys!A+-?RuOo>{_eEpnM16P8~bo!vfD8 zoWfvX%$^9x6un{276dC3^Zjs=^~- za53&AbdUb==bs->oJhqB!(Kq|9Q?wrJrX5WWOrEz;64Tm<4S;$ODs~H0FwpK=9)FD zJ2*p%D~SC3g2KuX&gm8I;OI)>@C(Bu0b3X>?52a5S>2TXP5yYoB+50x%zsj8+9~1n z29)J0c)c+W3J1OXG9D-nmx~YigoFefMF-5BS+j7r(2}=q--bDiU5vG(hd>K#`e6yN ziy0RI7sP=6{V=}-5~u|aEZ#A2D&nID!NLFy{EJh@1{0X3V5*I~0rQrE ze3=by+otWo{cV{!o~FI%M)0C4r)N>>tKd|d7c)g-G3HT&Ld1Z=Q;(1OE}c8uZikFebclRSXMha|1?IAP*U3r+H1sypU_m9Z40sZ@e##55OEHI^8< zS7O855nyXH&aaR=g^3J7fFM8+_?IB?`IpmLw`|p-MN7&HieFg1Xd6YrNP}vzfr7EY zfkWBC@JL|b23~29r|27>0^nl!`jCKi{O6`WN51Vin}pX-*#pbV>va=_1Ob8oLBLxG z$fZ*F9x)LLSQQ(-RYAevUu?_-xEMM3r7}Ud@STE%v7rbGf`91^lice(8Qxv$b0I`r zYUoHof&f9_Uxa|l*Ec0K1vAIeG)gCE@&yHhaPbSy#V<2K`HPW*Qt*?&C`KuaT~gX7 z^DOW4Bt%@^2owDT0fIn9ARrM-^-_IOa*`J-gGKD;R}mdugowR()rcyB072l1A#gt< zvq__dFloKmg7C2LQ}*G7UexJS6(M4$u#ilG072j>Lm(n7^viE%QgbwWtLd2N?# From e4a0669da8cd428dc577f91f915644298f97667e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 6 Jun 2019 21:31:10 -0500 Subject: [PATCH 025/146] Fix typo in video param --- src/invidious/videos.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 92787fdca..8f0fda462 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1235,7 +1235,7 @@ def process_video_params(query, preferences) continue = query["continue"]?.try &.to_i? continue_autoplay = query["continue_autoplay"]?.try &.to_i? listen = query["listen"]? && (query["listen"] == "true" || query["listen"] == "1").to_unsafe - local = query["local"]? && (query["local"] == "true" || query["listen"] == "1").to_unsafe + local = query["local"]? && (query["local"] == "true" || query["local"] == "1").to_unsafe preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? region = query["region"]? From fda619f704a442970157407c8446341ccdb84977 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 6 Jun 2019 21:32:39 -0500 Subject: [PATCH 026/146] Fix 'unique_res' to keep resolutions unique within a representation --- src/invidious.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index b1ec27055..c60e5537a 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -4693,7 +4693,7 @@ get "/api/manifest/dash/id/:id" do |env| region = env.params.query["region"]? # Since some implementations create playlists based on resolution regardless of different codecs, - # we can opt to only add a source to a representation if it has a unique height + # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]? && (env.params.query["unique_res"] == "true" || env.params.query["unique_res"] == "1") client = make_client(YT_URL) @@ -4767,8 +4767,8 @@ get "/api/manifest/dash/id/:id" do |env| i += 1 end - heights = [] of Int32 {"video/mp4", "video/webm"}.each do |mime_type| + heights = [] of Int32 xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do video_streams.select { |stream| stream["type"].starts_with? mime_type }.each do |fmt| codecs = fmt["type"].split("codecs=")[1].strip('"') From 317d8703ca506394af1adf5f72b1c66c83bb704b Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 6 Jun 2019 21:33:30 -0500 Subject: [PATCH 027/146] Optimize query for pulling popular videos --- src/invidious/helpers/jobs.cr | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index 63f7c16f8..ee468d1ad 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -200,13 +200,10 @@ end def pull_popular_videos(db) loop do - subscriptions = db.query_all("SELECT channel FROM \ - (SELECT UNNEST(subscriptions) AS channel FROM users) AS d \ - GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40", as: String) - - videos = db.query_all("SELECT DISTINCT ON (ucid) * FROM \ - channel_videos WHERE ucid IN (#{arg_array(subscriptions)}) \ - ORDER BY ucid, published DESC", subscriptions, as: ChannelVideo).sort_by { |video| video.published }.reverse + videos = db.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE ucid IN \ + (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d \ + GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) \ + ORDER BY ucid, published DESC", as: ChannelVideo).sort_by { |video| video.published }.reverse yield videos sleep 1.minute From 3f791b65b5c4974f8fe1bb66ead0267a153e5d9a Mon Sep 17 00:00:00 2001 From: Esmail EL BoB Date: Fri, 7 Jun 2019 04:46:46 +0000 Subject: [PATCH 028/146] Update ar.json --- locales/ar.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/locales/ar.json b/locales/ar.json index 9619043d0..683802363 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -6,7 +6,7 @@ "Unsubscribe": "إلغاء الإشتراك", "Subscribe": "إشتراك", "View channel on YouTube": "زيارة القناة على موقع يوتيوب", - "View playlist on YouTube": "", + "View playlist on YouTube": "عرض قائمة التشغيل على اليوتيوب", "newest": "الأجدد", "oldest": "الأقدم", "popular": "الاكثر شعبية", @@ -85,9 +85,9 @@ "Only show latest unwatched video from channel: ": "فقط اظهر اخر فيديو لم يتم رؤيتة من القناة: ", "Only show unwatched: ": "فقط اظهر الذى لم يتم رؤيتة: ", "Only show notifications (if there are any): ": "إظهار الإشعارات فقط (إذا كان هناك أي): ", - "Enable web notifications": "", - "`x` uploaded a video": "", - "`x` is live": "", + "Enable web notifications": "تفعيل إشعارات المتصفح", + "`x` uploaded a video": "`x` رفع فيديو", + "`x` is live": "`x` فى بث مباشر", "Data preferences": "إعدادات التفضيلات", "Clear watch history": "حذف سجل المشاهدة", "Import/export data": "إضافة\\إستخراج البيانات", @@ -315,4 +315,4 @@ "Videos": "الفيديوهات", "Playlists": "قوائم التشغيل", "Current version: ": "الإصدار الحالى" -} \ No newline at end of file +} From 9d23f1298deae78bcb5bd993295a5dced8ec011b Mon Sep 17 00:00:00 2001 From: Heimen Stoffels Date: Fri, 7 Jun 2019 12:29:03 +0200 Subject: [PATCH 029/146] Updated Dutch translation --- locales/nl.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/locales/nl.json b/locales/nl.json index 50fe85d8b..2cc716efd 100644 --- a/locales/nl.json +++ b/locales/nl.json @@ -85,9 +85,9 @@ "Only show latest unwatched video from channel: ": "Alleen nieuwste niet-bekeken video van kanaal tonen: ", "Only show unwatched: ": "Alleen niet-bekeken videos tonen: ", "Only show notifications (if there are any): ": "Alleen meldingen tonen (als die er zijn): ", - "Enable web notifications": "", - "`x` uploaded a video": "", - "`x` is live": "", + "Enable web notifications": "Systemmeldingen inschakelen", + "`x` uploaded a video": "`x` heeft een video geüpload", + "`x` is live": "`x` zendt nu live uit", "Data preferences": "Gegevensinstellingen", "Clear watch history": "Kijkgeschiedenis wissen", "Import/export data": "Gegevens im-/exporteren", @@ -117,13 +117,13 @@ "`x` unseen notifications": "`x` ongelezen meldingen", "search": "zoeken", "Log out": "Uitloggen", - "Released under the AGPLv3 by Omar Roth.": "Uitgegeven onder de AGPLv3-licentie door Omar Roth.", + "Released under the AGPLv3 by Omar Roth.": "Uitgebracht onder de AGPLv3-licentie, door Omar Roth.", "Source available here.": "De broncode is hier beschikbaar.", "View JavaScript license information.": "JavaScript-licentieinformatie tonen.", "View privacy policy.": "Privacybeleid tonen", "Trending": "Uitgelicht", "Unlisted": "Verborgen", - "Watch on YouTube": "Bekijk video op YouTube", + "Watch on YouTube": "Video bekijken op YouTube", "Hide annotations": "Annotaties verbergen", "Show annotations": "Annotaties tonen", "Genre: ": "Genre: ", @@ -315,4 +315,4 @@ "Videos": "Video's", "Playlists": "Afspeellijsten", "Current version: ": "Huidige versie: " -} \ No newline at end of file +} From ab3980cd38575f310730c5871f25589d060ba0b3 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 11:28:58 -0500 Subject: [PATCH 030/146] Enforce maximum email length --- src/invidious.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index c60e5537a..1882c4ff5 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -897,7 +897,8 @@ post "/login" do |env| next templated "error" end - email = env.params.body["email"]?.try &.downcase + # https://stackoverflow.com/a/574698 + email = env.params.body["email"]?.try &.downcase.byte_slice(0, 254) password = env.params.body["password"]? account_type = env.params.query["type"]? From 27e032d10dce0bd657e7b705d3dc8b24f1b55178 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 12:39:12 -0500 Subject: [PATCH 031/146] Add '/api/v1/auth/feeds' --- src/invidious.cr | 318 +++++++++----------------------------- src/invidious/channels.cr | 42 +++++ src/invidious/search.cr | 51 ++++++ src/invidious/users.cr | 98 ++++++++++++ 4 files changed, 263 insertions(+), 246 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 1882c4ff5..990f0b705 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -40,18 +40,19 @@ PG_URL = URI.new( path: CONFIG.db.dbname, ) -PG_DB = DB.open PG_URL -ARCHIVE_URL = URI.parse("https://archive.org") -LOGIN_URL = URI.parse("https://accounts.google.com") -PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com") -YT_URL = URI.parse("https://www.youtube.com") -CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" -TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} -CURRENT_BRANCH = {{ "#{`git branch | sed -n '/\* /s///p'`.strip}" }} -CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} -CURRENT_VERSION = {{ "#{`git describe --tags --abbrev=0`.strip}" }} +PG_DB = DB.open PG_URL +ARCHIVE_URL = URI.parse("https://archive.org") +LOGIN_URL = URI.parse("https://accounts.google.com") +PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") +REDDIT_URL = URI.parse("https://www.reddit.com") +TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com") +YT_URL = URI.parse("https://www.youtube.com") +CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" +TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} +CURRENT_BRANCH = {{ "#{`git branch | sed -n '/\* /s///p'`.strip}" }} +CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} +CURRENT_VERSION = {{ "#{`git describe --tags --abbrev=0`.strip}" }} +MAX_ITEMS_PER_PAGE = 1000 # This is used to determine the `?v=` on the end of file URLs (for cache busting). We # only need to expire modified assets, so we can use this to find the last commit that changes @@ -2369,13 +2370,15 @@ get "/feed/subscriptions" do |env| sid = env.get? "sid" referer = get_referer(env) - if user + if !user + next env.redirect referer + end + user = user.as(User) sid = sid.as(String) - preferences = user.preferences token = user.token - if preferences.unseen_only + if user.preferences.unseen_only env.set "show_watched", true end @@ -2387,113 +2390,14 @@ get "/feed/subscriptions" do |env| user, sid = get_user(sid, headers, PG_DB) end - max_results = preferences.max_results + max_results = user.preferences.max_results max_results ||= env.params.query["max_results"]?.try &.to_i? max_results ||= 40 page = env.params.query["page"]?.try &.to_i? page ||= 1 - if max_results < 0 - limit = nil - offset = (page - 1) * 1 - else - limit = max_results - offset = (page - 1) * max_results - end - - notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, - as: Array(String)) - view_name = "subscriptions_#{sha256(user.email)}" - - if preferences.notifications_only && !notifications.empty? - # Only show notifications - - args = arg_array(notifications) - - notifications = PG_DB.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) - ORDER BY published DESC", notifications, as: ChannelVideo) - videos = [] of ChannelVideo - - notifications.sort_by! { |video| video.published }.reverse! - - case preferences.sort - when "alphabetically" - notifications.sort_by! { |video| video.title } - when "alphabetically - reverse" - notifications.sort_by! { |video| video.title }.reverse! - when "channel name" - notifications.sort_by! { |video| video.author } - when "channel name - reverse" - notifications.sort_by! { |video| video.author }.reverse! - end - else - if preferences.latest_only - if preferences.unseen_only - # Show latest video from a channel that a user hasn't watched - # "unseen_only" isn't really correct here, more accurate would be "unwatched_only" - - if user.watched.empty? - values = "'{}'" - else - values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" - end - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE \ - NOT id = ANY (#{values}) \ - ORDER BY ucid, published DESC", as: ChannelVideo) - else - # Show latest video from each channel - - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} \ - ORDER BY ucid, published DESC", as: ChannelVideo) - end - - videos.sort_by! { |video| video.published }.reverse! - else - if preferences.unseen_only - # Only show unwatched - - if user.watched.empty? - values = "'{}'" - else - values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" - end - videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE \ - NOT id = ANY (#{values}) \ - ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) - else - # Sort subscriptions as normal - - videos = PG_DB.query_all("SELECT * FROM #{view_name} \ - ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) - end - end - - case preferences.sort - when "published - reverse" - videos.sort_by! { |video| video.published } - when "alphabetically" - videos.sort_by! { |video| video.title } - when "alphabetically - reverse" - videos.sort_by! { |video| video.title }.reverse! - when "channel name" - videos.sort_by! { |video| video.author } - when "channel name - reverse" - videos.sort_by! { |video| video.author }.reverse! - end - - notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, - as: Array(String)) - - notifications = videos.select { |v| notifications.includes? v.id } - videos = videos - notifications - end - - if !limit - videos = videos[0..max_results] - end - - # Clear user's notifications and set updated to the current time. + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) # "updated" here is used for delivering new notifications, so if # we know a user has looked at their feed e.g. in the past 10 minutes, @@ -2505,10 +2409,7 @@ get "/feed/subscriptions" do |env| env.set "user", user templated "subscriptions" - else - env.redirect referer end -end get "/feed/history" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2519,10 +2420,13 @@ get "/feed/history" do |env| page = env.params.query["page"]?.try &.to_i? page ||= 1 - if user + if !user + next env.redirect referer + end + user = user.as(User) - limit = user.preferences.max_results + limit = user.preferences.max_results.clamp(0, MAX_ITEMS_PER_PAGE) if user.watched[(page - 1) * limit]? watched = user.watched.reverse[(page - 1) * limit, limit] else @@ -2530,10 +2434,7 @@ get "/feed/history" do |env| end templated "history" - else - env.redirect referer end -end get "/feed/channel/:ucid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2586,13 +2487,12 @@ get "/feed/channel/:ucid" do |env| end host_url = make_host_url(config, Kemal.config) - path = env.request.path - feed = XML.build(indent: " ", encoding: "UTF-8") do |xml| + XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{host_url}#{path}") + xml.element("link", rel: "self", href: "#{host_url}#{env.request.resource}") xml.element("id") { xml.text "yt:channel:#{ucid}" } xml.element("yt:channelId") { xml.text ucid } xml.element("title") { xml.text author } @@ -2604,50 +2504,11 @@ get "/feed/channel/:ucid" do |env| end videos.each do |video| - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{video.id}" } - xml.element("yt:videoId") { xml.text video.id } - xml.element("yt:channelId") { xml.text video.ucid } - xml.element("title") { xml.text video.title } - xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{video.id}") - - xml.element("author") do - if auto_generated - xml.element("name") { xml.text video.author } - xml.element("uri") { xml.text "#{host_url}/channel/#{video.ucid}" } - else - xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" } + video.to_xml(host_url, auto_generated, xml) end end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{host_url}/watch?v=#{video.id}") do - xml.element("img", src: "#{host_url}/vi/#{video.id}/mqdefault.jpg") end end - end - - xml.element("published") { xml.text video.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text video.title } - xml.element("media:thumbnail", url: "#{host_url}/vi/#{video.id}/mqdefault.jpg", - width: "320", height: "180") - xml.element("media:description") { xml.text video.description } - end - - xml.element("media:community") do - xml.element("media:statistics", views: video.views) - end - end - end - end - end - - feed -end get "/feed/private" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2667,103 +2528,30 @@ get "/feed/private" do |env| next end + max_results = user.preferences.max_results max_results = env.params.query["max_results"]?.try &.to_i? max_results ||= 40 page = env.params.query["page"]?.try &.to_i? page ||= 1 - if max_results < 0 - limit = nil - offset = (page - 1) * 1 - else - limit = max_results - offset = (page - 1) * max_results - end - - latest_only = env.params.query["latest_only"]?.try &.to_i? - latest_only ||= 0 - latest_only = latest_only == 1 - - sort = env.params.query["sort"]? - sort ||= "published" - - view_name = "subscriptions_#{sha256(user.email)}" - - if latest_only - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} \ - ORDER BY ucid, published DESC", as: ChannelVideo) - - videos.sort_by! { |video| video.published }.reverse! - else - videos = PG_DB.query_all("SELECT * FROM #{view_name} \ - ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) - end - - case sort - when "reverse_published" - videos.sort_by! { |video| video.published } - when "alphabetically" - videos.sort_by! { |video| video.title } - when "reverse_alphabetically" - videos.sort_by! { |video| video.title }.reverse! - when "channel_name" - videos.sort_by! { |video| video.author } - when "reverse_channel_name" - videos.sort_by! { |video| video.author }.reverse! - end - - if !limit - videos = videos[0..max_results] - end - + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) host_url = make_host_url(config, Kemal.config) - path = env.request.path - query = env.request.query.not_nil! - feed = XML.build(indent: " ", encoding: "UTF-8") do |xml| + XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", "xml:lang": "en-US") do xml.element("link", "type": "text/html", rel: "alternate", href: "#{host_url}/feed/subscriptions") - xml.element("link", "type": "application/atom+xml", rel: "self", href: "#{host_url}#{path}?#{query}") + xml.element("link", "type": "application/atom+xml", rel: "self", + href: "#{host_url}#{env.request.resource}") xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } videos.each do |video| - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{video.id}" } - xml.element("yt:videoId") { xml.text video.id } - xml.element("yt:channelId") { xml.text video.ucid } - xml.element("title") { xml.text video.title } - xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{video.id}") - - xml.element("author") do - xml.element("name") { xml.text video.author } - xml.element("uri") { xml.text "#{host_url}/channel/#{video.ucid}" } - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{host_url}/watch?v=#{video.id}") do - xml.element("img", src: "#{host_url}/vi/#{video.id}/mqdefault.jpg") - end - end - end - - xml.element("published") { xml.text video.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - xml.element("updated") { xml.text video.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text video.title } - xml.element("media:thumbnail", url: "#{host_url}/vi/#{video.id}/mqdefault.jpg", - width: "320", height: "180") - end - end + video.to_xml(locale, host_url, xml) end end end - - feed end get "/feed/playlist/:plid" do |env| @@ -4475,6 +4263,8 @@ get "/api/v1/mixes/:rdid" do |env| response end +# Authenticated endpoints + get "/api/v1/auth/notifications" do |env| env.response.content_type = "text/event-stream" @@ -4514,6 +4304,42 @@ post "/api/v1/auth/preferences" do |env| env.response.status_code = 204 end +get "/api/v1/auth/feed" do |env| + env.response.content_type = "application/json" + + user = env.get("user").as(User) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + max_results = user.preferences.max_results + max_results ||= env.params.query["max_results"]?.try &.to_i? + max_results ||= 40 + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + + JSON.build do |json| + json.object do + json.field "notifications" do + json.array do + notifications.each do |video| + video.to_json(locale, config, Kemal.config, json) + end + end + end + + json.field "videos" do + json.array do + videos.each do |video| + video.to_json(locale, config, Kemal.config, json) + end + end + end + end + end +end + get "/api/v1/auth/subscriptions" do |env| env.response.content_type = "application/json" user = env.get("user").as(User) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index bf897e7d8..55b8046e9 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -41,6 +41,48 @@ struct ChannelVideo end end + def to_xml(locale, host_url, xml : XML::Builder) + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}") + + xml.element("author") do + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" } + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do + xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg") + end + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + end + end + end + + def to_xml(locale, config, kemal_config, xml : XML::Builder | Nil = nil) + if xml + to_xml(locale, config, kemal_config, xml) + else + XML.build do |xml| + to_xml(locale, config, kemal_config, xml) + end + end + end + db_mapping({ id: String, title: String, diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 58ccb1645..12aa81b99 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -1,4 +1,55 @@ struct SearchVideo + def to_xml(host_url, auto_generated, xml : XML::Builder) + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}") + + xml.element("author") do + if auto_generated + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" } + else + xml.element("name") { xml.text author } + xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" } + end + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do + xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg") + end + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + xml.element("media:description") { xml.text self.description } + end + + xml.element("media:community") do + xml.element("media:statistics", views: self.views) + end + end + end + + def to_xml(host_url, auto_generated, xml : XML::Builder | Nil = nil) + if xml + to_xml(host_url, auto_generated, xml) + else + XML.build do |json| + to_xml(host_url, auto_generated, xml) + end + end + end + db_mapping({ title: String, id: String, diff --git a/src/invidious/users.cr b/src/invidious/users.cr index a8764a142..4ad11905e 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -321,3 +321,101 @@ def subscribe_ajax(channel_id, action, env_headers) client.post(post_url, headers, form: post_req) end end + +def get_subscription_feed(db, user, max_results = 40, page = 1) + limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) + offset = (page - 1) * limit + + notifications = db.query_one("SELECT notifications FROM users WHERE email = $1", user.email, + as: Array(String)) + view_name = "subscriptions_#{sha256(user.email)}" + + if user.preferences.notifications_only && !notifications.empty? + # Only show notifications + + args = arg_array(notifications) + + notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) + ORDER BY published DESC", notifications, as: ChannelVideo) + videos = [] of ChannelVideo + + notifications.sort_by! { |video| video.published }.reverse! + + case user.preferences.sort + when "alphabetically" + notifications.sort_by! { |video| video.title } + when "alphabetically - reverse" + notifications.sort_by! { |video| video.title }.reverse! + when "channel name" + notifications.sort_by! { |video| video.author } + when "channel name - reverse" + notifications.sort_by! { |video| video.author }.reverse! + end + else + if user.preferences.latest_only + if user.preferences.unseen_only + # Show latest video from a channel that a user hasn't watched + # "unseen_only" isn't really correct here, more accurate would be "unwatched_only" + + if user.watched.empty? + values = "'{}'" + else + values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" + end + videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE \ + NOT id = ANY (#{values}) \ + ORDER BY ucid, published DESC", as: ChannelVideo) + else + # Show latest video from each channel + + videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} \ + ORDER BY ucid, published DESC", as: ChannelVideo) + end + + videos.sort_by! { |video| video.published }.reverse! + else + if user.preferences.unseen_only + # Only show unwatched + + if user.watched.empty? + values = "'{}'" + else + values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" + end + videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE \ + NOT id = ANY (#{values}) \ + ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) + else + # Sort subscriptions as normal + + videos = PG_DB.query_all("SELECT * FROM #{view_name} \ + ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) + end + end + + case user.preferences.sort + when "published - reverse" + videos.sort_by! { |video| video.published } + when "alphabetically" + videos.sort_by! { |video| video.title } + when "alphabetically - reverse" + videos.sort_by! { |video| video.title }.reverse! + when "channel name" + videos.sort_by! { |video| video.author } + when "channel name - reverse" + videos.sort_by! { |video| video.author }.reverse! + end + + notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, + as: Array(String)) + + notifications = videos.select { |v| notifications.includes? v.id } + videos = videos - notifications + end + + if !limit + videos = videos[0..max_results] + end + + return videos, notifications +end From f065a21542fd9d7587b89c426327c3c83a24c2bc Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 12:42:07 -0500 Subject: [PATCH 032/146] Fix 404 handling for endpoints matching short URLs --- src/invidious.cr | 79 +++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 990f0b705..140001beb 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2374,42 +2374,42 @@ get "/feed/subscriptions" do |env| next env.redirect referer end - user = user.as(User) - sid = sid.as(String) - token = user.token + user = user.as(User) + sid = sid.as(String) + token = user.token if user.preferences.unseen_only - env.set "show_watched", true - end + env.set "show_watched", true + end - # Refresh account - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] + # Refresh account + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] - if !user.password - user, sid = get_user(sid, headers, PG_DB) - end + if !user.password + user, sid = get_user(sid, headers, PG_DB) + end max_results = user.preferences.max_results - max_results ||= env.params.query["max_results"]?.try &.to_i? - max_results ||= 40 + max_results ||= env.params.query["max_results"]?.try &.to_i? + max_results ||= 40 - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + page = env.params.query["page"]?.try &.to_i? + page ||= 1 videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - # "updated" here is used for delivering new notifications, so if - # we know a user has looked at their feed e.g. in the past 10 minutes, - # they've already seen a video posted 20 minutes ago, and don't need - # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.now, - user.email) - user.notifications = [] of String - env.set "user", user + # "updated" here is used for delivering new notifications, so if + # we know a user has looked at their feed e.g. in the past 10 minutes, + # they've already seen a video posted 20 minutes ago, and don't need + # to be notified. + PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.now, + user.email) + user.notifications = [] of String + env.set "user", user - templated "subscriptions" - end + templated "subscriptions" +end get "/feed/history" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2424,18 +2424,18 @@ get "/feed/history" do |env| next env.redirect referer end - user = user.as(User) + user = user.as(User) limit = user.preferences.max_results.clamp(0, MAX_ITEMS_PER_PAGE) - if user.watched[(page - 1) * limit]? - watched = user.watched.reverse[(page - 1) * limit, limit] - else - watched = [] of String - end - - templated "history" + if user.watched[(page - 1) * limit]? + watched = user.watched.reverse[(page - 1) * limit, limit] + else + watched = [] of String end + templated "history" +end + get "/feed/channel/:ucid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2505,10 +2505,10 @@ get "/feed/channel/:ucid" do |env| videos.each do |video| video.to_xml(host_url, auto_generated, xml) - end - end - end - end + end + end + end +end get "/feed/private" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -5058,6 +5058,11 @@ error 404 do |env| response = client.get(response.headers["Location"]) end + if response.body.empty? + env.response.headers["Location"] = "/" + halt env, status_code: 302 + end + html = XML.parse_html(response.body) ucid = html.xpath_node(%q(//meta[@itemprop="channelId"])) From 8c944815bcb1630739f0f5ba1994e051e67527e7 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 19:56:41 -0500 Subject: [PATCH 033/146] Minor refactor --- assets/js/embed.js | 6 +- assets/js/notifications.js | 11 ++-- assets/js/player.js | 28 +++++----- assets/js/subscribe_widget.js | 10 ++-- assets/js/watch.js | 30 +++++----- assets/js/watched_widget.js | 2 +- src/invidious.cr | 73 ++++++++++++++----------- src/invidious/channels.cr | 32 ++--------- src/invidious/comments.cr | 5 +- src/invidious/helpers/helpers.cr | 8 +-- src/invidious/helpers/jobs.cr | 4 +- src/invidious/helpers/logger.cr | 4 +- src/invidious/helpers/tokens.cr | 8 +-- src/invidious/helpers/utils.cr | 39 +++++++++++-- src/invidious/playlists.cr | 2 +- src/invidious/users.cr | 12 ++-- src/invidious/videos.cr | 6 +- src/invidious/views/components/item.ecr | 12 ++-- 18 files changed, 154 insertions(+), 138 deletions(-) diff --git a/assets/js/embed.js b/assets/js/embed.js index d2116b2ef..283bc06d3 100644 --- a/assets/js/embed.js +++ b/assets/js/embed.js @@ -1,5 +1,5 @@ -function get_playlist(plid, timeouts = 0) { - if (timeouts > 10) { +function get_playlist(plid, timeouts = 1) { + if (timeouts >= 10) { console.log('Failed to pull playlist'); return; } @@ -52,7 +52,7 @@ function get_playlist(plid, timeouts = 0) { } xhr.ontimeout = function () { - console.log('Pulling playlist timed out.'); + console.log('Pulling playlist timed out... ' + timeouts + '/10'); get_playlist(plid, timeouts++); } } diff --git a/assets/js/notifications.js b/assets/js/notifications.js index 90b8c4f02..7a1123500 100644 --- a/assets/js/notifications.js +++ b/assets/js/notifications.js @@ -1,7 +1,7 @@ var notifications, delivered; -function get_subscriptions(callback, failures = 1) { - if (failures >= 10) { +function get_subscriptions(callback, timeouts = 1) { + if (timeouts >= 10) { return } @@ -16,16 +16,13 @@ function get_subscriptions(callback, failures = 1) { if (xhr.status === 200) { subscriptions = xhr.response; callback(subscriptions); - } else { - console.log('Pulling subscriptions failed... ' + failures + '/10'); - get_subscriptions(callback, failures++) } } } xhr.ontimeout = function () { - console.log('Pulling subscriptions failed... ' + failures + '/10'); - get_subscriptions(callback, failures++); + console.log('Pulling subscriptions timed out... ' + timeouts + '/10'); + get_subscriptions(callback, timeouts++); } } diff --git a/assets/js/player.js b/assets/js/player.js index 823721855..2b546ff4a 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -1,20 +1,20 @@ var options = { - preload: "auto", + preload: 'auto', liveui: true, playbackRates: [0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0], controlBar: { children: [ - "playToggle", - "volumePanel", - "currentTimeDisplay", - "timeDivider", - "durationDisplay", - "progressControl", - "remainingTimeDisplay", - "captionsButton", - "qualitySelector", - "playbackRateMenuButton", - "fullscreenToggle" + 'playToggle', + 'volumePanel', + 'currentTimeDisplay', + 'timeDivider', + 'durationDisplay', + 'progressControl', + 'remainingTimeDisplay', + 'captionsButton', + 'qualitySelector', + 'playbackRateMenuButton', + 'fullscreenToggle' ] } } @@ -29,7 +29,7 @@ short_url = location.origin + '/' + video_data.id + embed_url.search; embed_url = location.origin + '/embed/' + video_data.id + embed_url.search; var shareOptions = { - socials: ["fbFeed", "tw", "reddit", "email"], + socials: ['fbFeed', 'tw', 'reddit', 'email'], url: short_url, title: player_data.title, @@ -38,7 +38,7 @@ var shareOptions = { embedCode: "" } -var player = videojs("player", options, function () { +var player = videojs('player', options, function () { this.hotkeys({ volumeStep: 0.1, seekStep: 5, diff --git a/assets/js/subscribe_widget.js b/assets/js/subscribe_widget.js index 8f055e26f..7a7f806d2 100644 --- a/assets/js/subscribe_widget.js +++ b/assets/js/subscribe_widget.js @@ -7,7 +7,7 @@ if (subscribe_button.getAttribute('data-type') === 'subscribe') { subscribe_button.onclick = unsubscribe; } -function subscribe(timeouts = 0) { +function subscribe(timeouts = 1) { if (timeouts >= 10) { console.log('Failed to subscribe.'); return; @@ -19,7 +19,7 @@ function subscribe(timeouts = 0) { xhr.responseType = 'json'; xhr.timeout = 20000; xhr.open('POST', url, true); - xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded"); + xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded'); xhr.send('csrf_token=' + subscribe_data.csrf_token); var fallback = subscribe_button.innerHTML; @@ -36,12 +36,12 @@ function subscribe(timeouts = 0) { } xhr.ontimeout = function () { - console.log('Subscribing timed out.'); + console.log('Subscribing timed out... ' + timeouts + '/10'); subscribe(timeouts++); } } -function unsubscribe(timeouts = 0) { +function unsubscribe(timeouts = 1) { if (timeouts >= 10) { console.log('Failed to subscribe'); return; @@ -70,7 +70,7 @@ function unsubscribe(timeouts = 0) { } xhr.ontimeout = function () { - console.log('Unsubscribing timed out.'); + console.log('Unsubscribing timed out... ' + timeouts + '/10'); unsubscribe(timeouts++); } } diff --git a/assets/js/watch.js b/assets/js/watch.js index c9cac43b4..80da3ee6e 100644 --- a/assets/js/watch.js +++ b/assets/js/watch.js @@ -109,10 +109,10 @@ function number_with_separator(val) { return val; } -function get_playlist(plid, timeouts = 0) { +function get_playlist(plid, timeouts = 1) { playlist = document.getElementById('playlist'); - if (timeouts > 10) { + if (timeouts >= 10) { console.log('Failed to pull playlist'); playlist.innerHTML = ''; return; @@ -175,18 +175,19 @@ function get_playlist(plid, timeouts = 0) { } xhr.ontimeout = function () { - console.log('Pulling playlist timed out.'); playlist = document.getElementById('playlist'); playlist.innerHTML = '

'; - get_playlist(plid, timeouts + 1); + + console.log('Pulling playlist timed out... ' + timeouts + '/10'); + get_playlist(plid, timeouts++); } } -function get_reddit_comments(timeouts = 0) { +function get_reddit_comments(timeouts = 1) { comments = document.getElementById('comments'); - if (timeouts > 10) { + if (timeouts >= 10) { console.log('Failed to pull comments'); comments.innerHTML = ''; return; @@ -238,7 +239,8 @@ function get_reddit_comments(timeouts = 0) { comments.children[0].children[1].children[0].onclick = swap_comments; } else { if (video_data.params.comments[1] === 'youtube') { - get_youtube_comments(timeouts + 1); + console.log('Pulling comments timed out... ' + timeouts + '/10'); + get_youtube_comments(timeouts++); } else { comments.innerHTML = fallback; } @@ -247,15 +249,15 @@ function get_reddit_comments(timeouts = 0) { } xhr.ontimeout = function () { - console.log('Pulling comments timed out.'); - get_reddit_comments(timeouts + 1); + console.log('Pulling comments timed out... ' + timeouts + '/10'); + get_reddit_comments(timeouts++); } } -function get_youtube_comments(timeouts = 0) { +function get_youtube_comments(timeouts = 1) { comments = document.getElementById('comments'); - if (timeouts > 10) { + if (timeouts >= 10) { console.log('Failed to pull comments'); comments.innerHTML = ''; return; @@ -303,7 +305,7 @@ function get_youtube_comments(timeouts = 0) { comments.children[0].children[1].children[0].onclick = swap_comments; } else { if (video_data.params.comments[1] === 'youtube') { - get_youtube_comments(timeouts + 1); + get_youtube_comments(timeouts++); } else { comments.innerHTML = ''; } @@ -312,10 +314,10 @@ function get_youtube_comments(timeouts = 0) { } xhr.ontimeout = function () { - console.log('Pulling comments timed out.'); comments.innerHTML = '

'; - get_youtube_comments(timeouts + 1); + console.log('Pulling comments timed out... ' + timeouts + '/10'); + get_youtube_comments(timeouts++); } } diff --git a/assets/js/watched_widget.js b/assets/js/watched_widget.js index 304a7688b..280da83a6 100644 --- a/assets/js/watched_widget.js +++ b/assets/js/watched_widget.js @@ -22,7 +22,7 @@ function mark_watched(target) { function mark_unwatched(target) { var tile = target.parentNode.parentNode.parentNode.parentNode.parentNode; - tile.style.display = "none"; + tile.style.display = 'none'; var count = document.getElementById('count') count.innerText = count.innerText - 1; diff --git a/src/invidious.cr b/src/invidious.cr index 140001beb..dd5bfd9b8 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -152,7 +152,7 @@ if config.statistics_enabled }, }, "metadata" => { - "updatedAt" => Time.now.to_unix, + "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => PG_DB.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0, }, } @@ -1119,7 +1119,7 @@ post "/login" do |env| if Crypto::Bcrypt::Password.new(user.password.not_nil!) == password.byte_slice(0, 55) sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.now) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) if Kemal.config.ssl || config.https_only secure = true @@ -1128,10 +1128,10 @@ post "/login" do |env| end if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.now + 2.years, + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.now + 2.years, + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, secure: secure, http_only: true) end else @@ -1233,7 +1233,7 @@ post "/login" do |env| args = arg_array(user_array) PG_DB.exec("INSERT INTO users VALUES (#{args})", user_array) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.now) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ @@ -1248,10 +1248,10 @@ post "/login" do |env| end if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.now + 2.years, + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.now + 2.years, + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, secure: secure, http_only: true) end @@ -1476,10 +1476,10 @@ post "/preferences" do |env| end if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.now + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.now + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -1513,10 +1513,10 @@ get "/toggle_theme" do |env| end if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.now + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.now + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -1719,9 +1719,9 @@ post "/subscription_ajax" do |env| end end - if env.params.query["action_create_subscription_to_channel"]? + if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 action = "action_create_subscription_to_channel" - elsif env.params.query["action_remove_subscriptions"]? + elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 action = "action_remove_subscriptions" else next env.redirect referer @@ -1737,12 +1737,12 @@ post "/subscription_ajax" do |env| email = user.email case action - when .starts_with? "action_create" + when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id get_channel(channel_id, PG_DB, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) end - when .starts_with? "action_remove" + when "action_remove_subscriptions" PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) end @@ -1885,7 +1885,7 @@ post "/data_control" do |env| env.response.flush loop do - env.response.puts %() + env.response.puts %() env.response.flush sleep (20 + rand(11)).seconds @@ -2403,7 +2403,7 @@ get "/feed/subscriptions" do |env| # we know a user has looked at their feed e.g. in the past 10 minutes, # they've already seen a video posted 20 minutes ago, and don't need # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.now, + PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, user.email) user.notifications = [] of String env.set "user", user @@ -2439,7 +2439,7 @@ end get "/feed/channel/:ucid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/atom+xml" + env.response.content_type = "text/xml; charset=UTF-8" ucid = env.params.url["ucid"] @@ -2513,7 +2513,7 @@ end get "/feed/private" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/atom+xml" + env.response.content_type = "text/xml; charset=UTF-8" token = env.params.query["token"]? @@ -2557,7 +2557,7 @@ end get "/feed/playlist/:plid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/atom+xml" + env.response.content_type = "text/xml; charset=UTF-8" plid = env.params.url["plid"] @@ -2608,17 +2608,21 @@ get "/feed/webhook/:token" do |env| topic = env.params.query["hub.topic"] challenge = env.params.query["hub.challenge"] - if verify_token.starts_with? "v1" + case verify_token + when .starts_with? "v1" _, time, nonce, signature = verify_token.split(":") data = "#{time}:#{nonce}" - else + when .starts_with? "v2" time, signature = verify_token.split(":") data = "#{time}" + else + env.response.status_code = 400 + next end # The hub will sometimes check if we're still subscribed after delivery errors, # so we reply with a 200 as long as the request hasn't expired - if Time.now.to_unix - time.to_i > 432000 + if Time.utc.to_unix - time.to_i > 432000 env.response.status_code = 400 next end @@ -2628,11 +2632,17 @@ get "/feed/webhook/:token" do |env| next end - ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"] - PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.now, ucid) + if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? + PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? + PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + else + env.response.status_code = 400 + next + end env.response.status_code = 200 - next challenge + challenge end post "/feed/webhook/:token" do |env| @@ -3217,11 +3227,10 @@ get "/api/v1/insights/:id" do |env| session_token = body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/).not_nil!["session_token"] post_req = { - "session_token" => session_token, + session_token: session_token, } - post_req = HTTP::Params.encode(post_req) - response = client.post("/insight_ajax?action_get_statistics_and_data=1&v=#{id}", headers, post_req).body + response = client.post("/insight_ajax?action_get_statistics_and_data=1&v=#{id}", headers, form: post_req).body response = XML.parse(response) html_content = XML.parse_html(response.xpath_node(%q(//html_content)).not_nil!.content) @@ -3265,16 +3274,14 @@ get "/api/v1/insights/:id" do |env| avg_view_duration_seconds = html_content.xpath_node(%q(//div[@id="stats-chart-tab-watch-time"]/span/span[2])).not_nil!.content avg_view_duration_seconds = decode_length_seconds(avg_view_duration_seconds) - response = { + { "viewCount" => view_count, "timeWatchedText" => time_watched, "subscriptionsDriven" => subscriptions_driven, "shares" => shares, "avgViewDurationSeconds" => avg_view_duration_seconds, "graphData" => graph_data, - } - - next response.to_json + }.to_json end get "/api/v1/annotations/:id" do |env| diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 55b8046e9..b7b6f5534 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -138,7 +138,7 @@ def get_channel(id, db, refresh = true, pull_all_videos = true) if db.query_one?("SELECT EXISTS (SELECT true FROM channels WHERE id = $1)", id, as: Bool) channel = db.query_one("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) - if refresh && Time.now - channel.updated > 10.minutes + if refresh && Time.utc - channel.updated > 10.minutes channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) channel_array = channel.to_a args = arg_array(channel_array) @@ -219,7 +219,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) id: video_id, title: title, published: published, - updated: Time.now, + updated: Time.utc, ucid: ucid, author: author, length_seconds: length_seconds, @@ -282,7 +282,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) id: video.id, title: video.title, published: video.published, - updated: Time.now, + updated: Time.utc, ucid: video.ucid, author: video.author, length_seconds: video.length_seconds, @@ -296,7 +296,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. - if Time.now - video.published > 1.minute + if Time.utc - video.published > 1.minute emails = db.query_all("UPDATE users SET notifications = notifications || $1 \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email", video.id, video.published, video.ucid, as: String) @@ -332,31 +332,11 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid) end - channel = InvidiousChannel.new(ucid, author, Time.now, false, nil) + channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil) return channel end -def subscribe_pubsub(ucid, key, config) - client = make_client(PUBSUB_URL) - time = Time.now.to_unix.to_s - nonce = Random::Secure.hex(4) - signature = "#{time}:#{nonce}" - - host_url = make_host_url(config, Kemal.config) - - body = { - "hub.callback" => "#{host_url}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}", - "hub.topic" => "https://www.youtube.com/xml/feeds/videos.xml?channel_id=#{ucid}", - "hub.verify" => "async", - "hub.mode" => "subscribe", - "hub.lease_seconds" => "432000", - "hub.secret" => key.to_s, - } - - return client.post("/subscribe", form: body) -end - def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) client = make_client(YT_URL) @@ -420,7 +400,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " if auto_generated seed = Time.unix(1525757349) - until seed >= Time.now + until seed >= Time.utc seed += 1.month end timestamp = seed - (page - 1).months diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index df8d5ca4c..a652f84a9 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -72,9 +72,8 @@ def fetch_youtube_comments(id, db, continuation, proxies, format, locale, thin_m end post_req = { - "session_token" => session_token, + session_token: session_token, } - post_req = HTTP::Params.encode(post_req) client = make_client(YT_URL, proxies, video.info["region"]?) headers = HTTP::Headers.new @@ -89,7 +88,7 @@ def fetch_youtube_comments(id, db, continuation, proxies, format, locale, thin_m headers["x-youtube-client-name"] = "1" headers["x-youtube-client-version"] = "2.20180719" - response = client.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, post_req) + response = client.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) response = JSON.parse(response.body) if !response["response"]["continuationContents"]? diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 2dd50d422..5c5d5bb19 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -146,7 +146,7 @@ def rank_videos(db, n) published = rs.read(Time) # Exponential decay, older videos tend to rank lower - temperature = wilson_score * Math.exp(-0.000005*((Time.now - published).total_minutes)) + temperature = wilson_score * Math.exp(-0.000005*((Time.utc - published).total_minutes)) top << {temperature, id} end end @@ -346,7 +346,7 @@ def extract_items(nodeset, ucid = nil, author_name = nil) published ||= Time.unix(metadata[0].xpath_node(%q(.//span)).not_nil!["data-timestamp"].to_i64) rescue ex end - published ||= Time.now + published ||= Time.utc begin view_count = metadata[0].content.rchop(" watching").delete(",").try &.to_i64? @@ -676,7 +676,7 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct loop do time_span = [0, 0, 0, 0] time_span[rand(4)] = rand(30) + 5 - published = Time.now - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) + published = Time.utc - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) video_id = TEST_IDS[rand(TEST_IDS.size)] video = get_video(video_id, PG_DB, proxies) @@ -783,7 +783,7 @@ def create_notification_stream(env, proxies, config, kemal_config, decrypt_funct begin # Send heartbeat loop do - env.response.puts ":keepalive #{Time.now.to_unix}" + env.response.puts ":keepalive #{Time.utc.to_unix}" env.response.puts env.response.flush sleep (20 + rand(11)).seconds diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index ee468d1ad..0a5fd7d9c 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -22,10 +22,10 @@ def refresh_channels(db, logger, config) begin channel = fetch_channel(id, db, config.full_refresh) - db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.now, channel.author, id) + db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id) rescue ex if ex.message == "Deleted or invalid channel" - db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.now, id) + db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) end logger.write("#{id} : #{ex.message}\n") end diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 5bb1eb408..8a458a45a 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -5,9 +5,9 @@ class Invidious::LogHandler < Kemal::BaseLogHandler end def call(context : HTTP::Server::Context) - time = Time.now + time = Time.utc call_next(context) - elapsed_text = elapsed_text(Time.now - time) + elapsed_text = elapsed_text(Time.utc - time) @io << time << ' ' << context.response.status_code << ' ' << context.request.method << ' ' << context.request.resource << ' ' << elapsed_text << '\n' diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index ba41cba3e..31b70c3be 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -1,6 +1,6 @@ def generate_token(email, scopes, expire, key, db) session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.now) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc) token = { "session" => session, @@ -18,7 +18,7 @@ def generate_token(email, scopes, expire, key, db) end def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = false) - expire = Time.now + expire + expire = Time.utc + expire token = { "session" => session, @@ -85,7 +85,7 @@ def validate_request(token, session, request, key, db, locale = nil) end if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) - if nonce[1] > Time.now + if nonce[1] > Time.utc db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.new(1990, 1, 1), nonce[0]) else raise translate(locale, "Erroneous token") @@ -100,7 +100,7 @@ def validate_request(token, session, request, key, db, locale = nil) end expire = token["expire"]?.try &.as_i - if expire.try &.< Time.now.to_unix + if expire.try &.< Time.utc.to_unix raise translate(locale, "Token is expired, please try again") end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index fcccb7f9b..37cc2eb84 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -110,9 +110,9 @@ def decode_date(string : String) case string when "today" - return Time.now + return Time.utc when "yesterday" - return Time.now - 1.day + return Time.utc - 1.day end # String matches format "20 hours ago", "4 months ago"... @@ -138,11 +138,11 @@ def decode_date(string : String) raise "Could not parse #{string}" end - return Time.now - delta + return Time.utc - delta end def recode_date(time : Time, locale) - span = Time.now - time + span = Time.utc - time if span.total_days > 365.0 span = translate(locale, "`x` years", (span.total_days.to_i / 365).to_s) @@ -327,3 +327,34 @@ def sha256(text) digest << text return digest.hexdigest end + +def subscribe_pubsub(topic, key, config) + case topic + when .match(/^UC[A-Za-z0-9_-]{22}$/) + topic = "channel_id=#{topic}" + when .match(/^(?:PL|LL|EC|UU|FL|UL|OLAK5uy_)[0-9A-Za-z-_]{10,}$/) + # There's a couple missing from the above regex, namely TL and RD, which + # don't have feeds + topic = "playlist_id=#{topic}" + else + # TODO + end + + client = make_client(PUBSUB_URL) + time = Time.utc.to_unix.to_s + nonce = Random::Secure.hex(4) + signature = "#{time}:#{nonce}" + + host_url = make_host_url(config, Kemal.config) + + body = { + "hub.callback" => "#{host_url}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}", + "hub.topic" => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}", + "hub.verify" => "async", + "hub.mode" => "subscribe", + "hub.lease_seconds" => "432000", + "hub.secret" => key.to_s, + } + + return client.post("/subscribe", form: body) +end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 92d9b9772..2b3f731e7 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -208,7 +208,7 @@ def fetch_playlist(plid, locale) if updated updated = decode_date(updated) else - updated = Time.now + updated = Time.utc end playlist = Playlist.new( diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 4ad11905e..37f3b4f1f 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -133,7 +133,7 @@ def get_user(sid, headers, db, refresh = true) if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) - if refresh && Time.now - user.updated > 1.minute + if refresh && Time.utc - user.updated > 1.minute user, sid = fetch_user(sid, headers, db) user_array = user.to_a @@ -144,7 +144,7 @@ def get_user(sid, headers, db, refresh = true) ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", user_array) db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.now) + ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) begin view_name = "subscriptions_#{sha256(user.email)}" @@ -166,7 +166,7 @@ def get_user(sid, headers, db, refresh = true) ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", user_array) db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.now) + ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) begin view_name = "subscriptions_#{sha256(user.email)}" @@ -206,7 +206,7 @@ def fetch_user(sid, headers, db) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.now, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true) + user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true) return user, sid end @@ -214,7 +214,7 @@ def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.now, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true) + user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true) return user, sid end @@ -314,7 +314,7 @@ def subscribe_ajax(channel_id, action, env_headers) headers["content-type"] = "application/x-www-form-urlencoded" post_req = { - "session_token" => session_token, + session_token: session_token, } post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}" diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 8f0fda462..6f3b4d438 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -852,7 +852,7 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}) video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours) - if (refresh && Time.now - video.updated > 10.minutes) || force_refresh + if (refresh && Time.utc - video.updated > 10.minutes) || force_refresh begin video = fetch_video(id, proxies, region) video_array = video.to_a @@ -1166,7 +1166,7 @@ def fetch_video(id, proxies, region) wilson_score = ci_lower_bound(likes, likes + dislikes) published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"] - published ||= Time.now.to_s("%Y-%m-%d") + published ||= Time.utc.to_s("%Y-%m-%d") published = Time.parse(published, "%Y-%m-%d", Time::Location.local) allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",") @@ -1218,7 +1218,7 @@ def fetch_video(id, proxies, region) author_thumbnail = "" end - video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description, + video = Video.new(id, info, Time.utc, title, views, likes, dislikes, wilson_score, published, description, nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail) return video diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index dc2acac9c..b73ce8a16 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -72,9 +72,9 @@

- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %> -
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %>
- <% elsif Time.now - item.published > 1.minute %> + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %> +
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
+ <% elsif Time.utc - item.published > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% else %>
@@ -121,9 +121,9 @@

- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %> -
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %>
- <% elsif Time.now - item.published > 1.minute %> + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %> +
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
+ <% elsif Time.utc - item.published > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% else %>
From 58995bb3a2f134f79b388a0c185015042c4fd665 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 20:07:55 -0500 Subject: [PATCH 034/146] Add support for log levels --- src/invidious.cr | 4 +-- src/invidious/helpers/helpers.cr | 19 +++++++++------ src/invidious/helpers/jobs.cr | 14 +++++------ src/invidious/helpers/logger.cr | 42 ++++++++++++++++++++++++++++++-- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index dd5bfd9b8..83bdc5be4 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -382,7 +382,7 @@ get "/watch" do |env| next env.redirect "/watch?v=#{ex.message}" rescue ex error_message = ex.message - logger.write("#{id} : #{ex.message}\n") + logger.puts("#{id} : #{ex.message}") next templated "error" end @@ -2653,7 +2653,7 @@ post "/feed/webhook/:token" do |env| signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) - logger.write("#{token} : Invalid signature\n") + logger.puts("#{token} : Invalid signature") env.response.status_code = 200 next end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 5c5d5bb19..ef33b736a 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -522,7 +522,7 @@ def analyze_table(db, logger, table_name, struct_type = nil) begin db.exec("SELECT * FROM #{table_name} LIMIT 0") rescue ex - logger.write("CREATE TABLE #{table_name}\n") + logger.puts("CREATE TABLE #{table_name}") db.using_connection do |conn| conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) @@ -546,7 +546,7 @@ def analyze_table(db, logger, table_name, struct_type = nil) if name != column_array[i]? if !column_array[i]? new_column = column_types.select { |line| line.starts_with? name }[0] - logger.write("ALTER TABLE #{table_name} ADD COLUMN #{new_column}\n") + logger.puts("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") next end @@ -564,26 +564,29 @@ def analyze_table(db, logger, table_name, struct_type = nil) # There's a column we didn't expect if !new_column - logger.write("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}\n") + logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") column_array = get_column_array(db, table_name) next end - logger.write("ALTER TABLE #{table_name} ADD COLUMN #{new_column}\n") + logger.puts("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - logger.write("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}\n") + + logger.puts("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - logger.write("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE\n") + + logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - logger.write("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}\n") + + logger.puts("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") column_array = get_column_array(db, table_name) end else - logger.write("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE\n") + logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") end end diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index 0a5fd7d9c..941f04810 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -27,7 +27,7 @@ def refresh_channels(db, logger, config) if ex.message == "Deleted or invalid channel" db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) end - logger.write("#{id} : #{ex.message}\n") + logger.puts("#{id} : #{ex.message}") end active_channel.send(true) @@ -68,14 +68,14 @@ def refresh_feeds(db, logger, config) column_array = get_column_array(db, view_name) ChannelVideo.to_type_tuple.each_with_index do |name, i| if name != column_array[i]? - logger.write("DROP MATERIALIZED VIEW #{view_name}\n") + logger.puts("DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}") raise "view does not exist" end end if db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "ucid = ANY" - logger.write("Materialized view #{view_name} is out-of-date, recreating...\n") + logger.puts("Materialized view #{view_name} is out-of-date, recreating...") db.exec("DROP MATERIALIZED VIEW #{view_name}") end @@ -87,13 +87,13 @@ def refresh_feeds(db, logger, config) legacy_view_name = "subscriptions_#{sha256(email)[0..7]}" db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0") - logger.write("RENAME MATERIALIZED VIEW #{legacy_view_name}\n") + logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}") db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}") rescue ex begin # While iterating through, we may have an email stored from a deleted account if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) - logger.write("CREATE #{view_name}\n") + logger.puts("CREATE #{view_name}") db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ SELECT * FROM channel_videos WHERE ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{email.gsub("'", "\\'")}') @@ -101,7 +101,7 @@ def refresh_feeds(db, logger, config) db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex - logger.write("REFRESH #{email} : #{ex.message}\n") + logger.puts("REFRESH #{email} : #{ex.message}") end end end @@ -151,7 +151,7 @@ def subscribe_to_feeds(db, logger, key, config) response = subscribe_pubsub(ucid, key, config) if response.status_code >= 400 - logger.write("#{ucid} : #{response.body}\n") + logger.puts("#{ucid} : #{response.body}") end rescue ex end diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 8a458a45a..52f0a22c4 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,7 +1,14 @@ require "logger" +enum LogLevel + Debug + Info + Warn + Error +end + class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT) + def initialize(@io : IO = STDOUT, @level = LogLevel::Warn) end def call(context : HTTP::Server::Context) @@ -18,7 +25,15 @@ class Invidious::LogHandler < Kemal::BaseLogHandler context end - def write(message : String) + def puts(message : String) + @io << message << '\n' + + if @io.is_a? File + @io.flush + end + end + + def write(message : String, level = @level) @io << message if @io.is_a? File @@ -26,6 +41,29 @@ class Invidious::LogHandler < Kemal::BaseLogHandler end end + def set_log_level(level : String) + case level.downcase + when "debug" + set_log_level(LogLevel::Debug) + when "info" + set_log_level(LogLevel::Info) + when "warn" + set_log_level(LogLevel::Warn) + when "error" + set_log_level(LogLevel::Error) + end + end + + def set_log_level(level : LogLevel) + @level = level + end + + {% for level in %w(debug info warn error) %} + def {{level.id}}(message : String) + puts(message, LogLevel::{{level.id.capitalize}}) + end + {% end %} + private def elapsed_text(elapsed) millis = elapsed.total_milliseconds return "#{millis.round(2)}ms" if millis >= 1 From 2febc268f784382b66373ad0b983d05eff959372 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 20:23:37 -0500 Subject: [PATCH 035/146] Fix warnings in Crystal 0.29 --- shard.yml | 3 ++- src/invidious.cr | 14 +++++++------- src/invidious/helpers/tokens.cr | 2 +- src/invidious/helpers/utils.cr | 16 ++++++++-------- src/invidious/playlists.cr | 6 +++--- src/invidious/videos.cr | 2 +- src/invidious/views/components/item.ecr | 2 +- src/invidious/views/watch.ecr | 2 +- 8 files changed, 24 insertions(+), 23 deletions(-) diff --git a/shard.yml b/shard.yml index 0d54a2f98..0e33c2fad 100644 --- a/shard.yml +++ b/shard.yml @@ -13,9 +13,10 @@ dependencies: github: kemalcr/kemal pg: github: will/crystal-pg + branch: cafe69e sqlite3: github: crystal-lang/crystal-sqlite3 -crystal: 0.28.0 +crystal: 0.29.0 license: AGPLv3 diff --git a/src/invidious.cr b/src/invidious.cr index 83bdc5be4..573855c7d 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1089,7 +1089,7 @@ post "/login" do |env| PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) cookie = env.request.cookies["PREFS"] - cookie.expires = Time.new(1990, 1, 1) + cookie.expires = Time.utc(1990, 1, 1) env.response.cookies << cookie end @@ -1117,7 +1117,7 @@ post "/login" do |env| next templated "error" end - if Crypto::Bcrypt::Password.new(user.password.not_nil!) == password.byte_slice(0, 55) + if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) @@ -1142,7 +1142,7 @@ post "/login" do |env| # Since this user has already registered, we don't want to overwrite their preferences if env.request.cookies["PREFS"]? cookie = env.request.cookies["PREFS"] - cookie.expires = Time.new(1990, 1, 1) + cookie.expires = Time.utc(1990, 1, 1) env.response.cookies << cookie end else @@ -1260,7 +1260,7 @@ post "/login" do |env| PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) cookie = env.request.cookies["PREFS"] - cookie.expires = Time.new(1990, 1, 1) + cookie.expires = Time.utc(1990, 1, 1) env.response.cookies << cookie end end @@ -1294,7 +1294,7 @@ post "/signout" do |env| PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) env.request.cookies.each do |cookie| - cookie.expires = Time.new(1990, 1, 1) + cookie.expires = Time.utc(1990, 1, 1) env.response.cookies << cookie end end @@ -2064,7 +2064,7 @@ post "/change_password" do |env| next templated "error" end - if Crypto::Bcrypt::Password.new(user.password.not_nil!) != password + if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password) error_message = translate(locale, "Incorrect password") next templated "error" end @@ -2120,7 +2120,7 @@ post "/delete_account" do |env| PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") env.request.cookies.each do |cookie| - cookie.expires = Time.new(1990, 1, 1) + cookie.expires = Time.utc(1990, 1, 1) env.response.cookies << cookie end end diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 31b70c3be..f946fc2c2 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -86,7 +86,7 @@ def validate_request(token, session, request, key, db, locale = nil) if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) if nonce[1] > Time.utc - db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.new(1990, 1, 1), nonce[0]) + db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) else raise translate(locale, "Erroneous token") end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 37cc2eb84..3ed067ad3 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -90,7 +90,7 @@ def decode_time(string) millis = /(?\d+)ms/.match(string).try &.["millis"].try &.to_f millis ||= 0 - time = hours * 3600 + minutes * 60 + seconds + millis / 1000 + time = hours * 3600 + minutes * 60 + seconds + millis // 1000 end return time @@ -99,7 +99,7 @@ end def decode_date(string : String) # String matches 'YYYY' if string.match(/^\d{4}/) - return Time.new(string.to_i, 1, 1) + return Time.utc(string.to_i, 1, 1) end # Try to parse as format Jul 10, 2000 @@ -145,11 +145,11 @@ def recode_date(time : Time, locale) span = Time.utc - time if span.total_days > 365.0 - span = translate(locale, "`x` years", (span.total_days.to_i / 365).to_s) + span = translate(locale, "`x` years", (span.total_days.to_i // 365).to_s) elsif span.total_days > 30.0 - span = translate(locale, "`x` months", (span.total_days.to_i / 30).to_s) + span = translate(locale, "`x` months", (span.total_days.to_i // 30).to_s) elsif span.total_days > 7.0 - span = translate(locale, "`x` weeks", (span.total_days.to_i / 7).to_s) + span = translate(locale, "`x` weeks", (span.total_days.to_i // 7).to_s) elsif span.total_hours > 24.0 span = translate(locale, "`x` days", (span.total_days.to_i).to_s) elsif span.total_minutes > 60.0 @@ -194,11 +194,11 @@ def number_to_short_text(number) text = text.rchop(".0") - if number / 1_000_000_000 != 0 + if number // 1_000_000_000 != 0 text += "B" - elsif number / 1_000_000 != 0 + elsif number // 1_000_000 != 0 text += "M" - elsif number / 1000 != 0 + elsif number // 1000 != 0 text += "K" end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 2b3f731e7..6eb0fd39e 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -6,7 +6,7 @@ struct PlaylistVideo ucid: String, length_seconds: Int32, published: Time, - playlists: Array(String), + plid: String, index: Int32, live_now: Bool, }) @@ -114,8 +114,8 @@ def extract_playlist(plid, nodeset, index) author: author, ucid: ucid, length_seconds: length_seconds, - published: Time.now, - playlists: [plid], + published: Time.utc, + plid: plid, index: index + offset, live_now: live_now ) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 6f3b4d438..5765d0c8d 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -973,7 +973,7 @@ def extract_polymer_config(body, html) if published params["published"] = Time.parse(published, "%b %-d, %Y", Time::Location.local).to_unix.to_s else - params["published"] = Time.new(1990, 1, 1).to_unix.to_s + params["published"] = Time.utc(1990, 1, 1).to_unix.to_s end params["description_html"] = "

" diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index b73ce8a16..780174243 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -52,7 +52,7 @@

<% when PlaylistVideo %> - + <% if !env.get("preferences").as(Preferences).thin_mode %>
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 5daf211ea..2f03190ae 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -148,7 +148,7 @@ var video_data = {

<%= translate(locale, "Engagement: ") %><%= engagement.round(2) %>%

<% if video.allowed_regions.size != REGIONS.size %>

- <% if video.allowed_regions.size < REGIONS.size / 2 %> + <% if video.allowed_regions.size < REGIONS.size // 2 %> <%= translate(locale, "Whitelisted regions: ") %><%= video.allowed_regions.join(", ") %> <% else %> <%= translate(locale, "Blacklisted regions: ") %><%= (REGIONS.to_a - video.allowed_regions).join(", ") %> From 0b1c57b39f66ca34f88bfeb96d2cd1cafb37a1c6 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 21:27:37 -0500 Subject: [PATCH 036/146] Add notifications to private feed --- src/invidious.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index 573855c7d..97de3fd04 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2547,7 +2547,7 @@ get "/feed/private" do |env| href: "#{host_url}#{env.request.resource}") xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } - videos.each do |video| + (notifications + videos).each do |video| video.to_xml(locale, host_url, xml) end end From 801dffd571ea9b0789da00fd0cfb2aae70e4e352 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 7 Jun 2019 21:39:32 -0500 Subject: [PATCH 037/146] Fix RSS content-type --- src/invidious.cr | 6 +++--- src/invidious/helpers/utils.cr | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 97de3fd04..63c441778 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2439,7 +2439,7 @@ end get "/feed/channel/:ucid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "text/xml; charset=UTF-8" + env.response.content_type = "application/atom+xml" ucid = env.params.url["ucid"] @@ -2513,7 +2513,7 @@ end get "/feed/private" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "text/xml; charset=UTF-8" + env.response.content_type = "application/atom+xml" token = env.params.query["token"]? @@ -2557,7 +2557,7 @@ end get "/feed/playlist/:plid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "text/xml; charset=UTF-8" + env.response.content_type = "application/atom+xml" plid = env.params.url["plid"] diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 3ed067ad3..bad2e3a39 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -332,7 +332,7 @@ def subscribe_pubsub(topic, key, config) case topic when .match(/^UC[A-Za-z0-9_-]{22}$/) topic = "channel_id=#{topic}" - when .match(/^(?:PL|LL|EC|UU|FL|UL|OLAK5uy_)[0-9A-Za-z-_]{10,}$/) + when .match(/^(PL|LL|EC|UU|FL|UL|OLAK5uy_)[0-9A-Za-z-_]{10,}$/) # There's a couple missing from the above regex, namely TL and RD, which # don't have feeds topic = "playlist_id=#{topic}" From ef8c9f093c0b6ea4b68b116d683e8d7f045ccd66 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 8 Jun 2019 10:18:45 -0500 Subject: [PATCH 038/146] Add premiere date to watch page --- assets/js/player.js | 33 +++++++++++++++++---------------- locales/ar.json | 1 + locales/de.json | 3 ++- locales/el.json | 1 + locales/en-US.json | 1 + locales/eo.json | 3 ++- locales/es.json | 1 + locales/fr.json | 1 + locales/nb_NO.json | 3 ++- locales/nl.json | 3 ++- locales/pl.json | 1 + locales/ru.json | 1 + locales/uk.json | 1 + src/invidious.cr | 2 +- src/invidious/videos.cr | 8 ++++++-- src/invidious/views/embed.ecr | 3 ++- src/invidious/views/watch.ecr | 13 +++++++++++-- 17 files changed, 53 insertions(+), 26 deletions(-) diff --git a/assets/js/player.js b/assets/js/player.js index 2b546ff4a..8854d859d 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -163,27 +163,28 @@ player.on('waiting', function () { } }); +if (video_data.premiere_timestamp && Math.round(new Date() / 1000) < video_data.premiere_timestamp) { + player.getChild('bigPlayButton').hide(); +} + if (video_data.params.autoplay) { var bpb = player.getChild('bigPlayButton'); + bpb.hide(); - if (bpb) { - bpb.hide(); + player.ready(function () { + new Promise(function (resolve, reject) { + setTimeout(() => resolve(1), 1); + }).then(function (result) { + var promise = player.play(); - player.ready(function () { - new Promise(function (resolve, reject) { - setTimeout(() => resolve(1), 1); - }).then(function (result) { - var promise = player.play(); - - if (promise !== undefined) { - promise.then(_ => { - }).catch(error => { - bpb.show(); - }); - } - }); + if (promise !== undefined) { + promise.then(_ => { + }).catch(error => { + bpb.show(); + }); + } }); - } + }); } if (!video_data.params.listen && video_data.params.quality === 'dash') { diff --git a/locales/ar.json b/locales/ar.json index 683802363..cc24da4a6 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -136,6 +136,7 @@ "Shared `x`": "شارك منذ `x`", "`x` views": "`x` مشاهدون", "Premieres in `x`": "يعرض فى `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "اهلا! يبدو ان الجافاسكريبت معطلة. اضغط هنا لعرض التعليقات, ضع فى إعتبارك انها ستأخذ وقت اطول للعرض.", "View YouTube comments": "عرض تعليقات اليوتيوب", "View more comments on Reddit": "عرض المزيد من التعليقات على\\من موقع Reddit", diff --git a/locales/de.json b/locales/de.json index 6092fd94c..e833ce694 100644 --- a/locales/de.json +++ b/locales/de.json @@ -136,6 +136,7 @@ "Shared `x`": "Geteilt `x`", "`x` views": "`x` Ansichten", "Premieres in `x`": "Premieren in `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Hallo! Anscheinend haben Sie JavaScript deaktiviert. Klicken Sie hier um Kommentare anzuzeigen, beachten sie dass es etwas länger dauern kann um sie zu laden.", "View YouTube comments": "YouTube Kommentare anzeigen", "View more comments on Reddit": "Mehr Kommentare auf Reddit anzeigen", @@ -315,4 +316,4 @@ "Videos": "Videos", "Playlists": "Wiedergabelisten", "Current version: ": "Aktuelle Version: " -} +} \ No newline at end of file diff --git a/locales/el.json b/locales/el.json index 54d514cb1..3ed49c67e 100644 --- a/locales/el.json +++ b/locales/el.json @@ -154,6 +154,7 @@ "": "`x` προβολές" }, "Premieres in `x`": "Πρώτη προβολή σε `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Γεια! Φαίνεται πως έχετε απενεργοποιήσει το JavaScript. Πατήστε εδώ για προβολή σχολίων, αλλά έχετε υπ'όψιν σας πως ίσως φορτώσουν πιο αργά. ", "View YouTube comments": "Προβολή σχολίων από το YouTube", "View more comments on Reddit": "Προβολή περισσότερων σχολίων στο Reddit", diff --git a/locales/en-US.json b/locales/en-US.json index 1ca2b9704..9b7862605 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -154,6 +154,7 @@ "": "`x` views" }, "Premieres in `x`": "Premieres in `x`", + "Premieres `x`": "Premieres `x`", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.", "View YouTube comments": "View YouTube comments", "View more comments on Reddit": "View more comments on Reddit", diff --git a/locales/eo.json b/locales/eo.json index bdc6c0bf4..bbaf128fd 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -136,6 +136,7 @@ "Shared `x`": "Konigita `x`", "`x` views": "`x` spektaĵoj", "Premieres in `x`": "Premieras en `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Saluton! Ŝajnas, ke vi havas Ĝavoskripton malebligitan. Klaku ĉi tie por vidi komentojn, memoru, ke la ŝargado povus daŭri iom pli.", "View YouTube comments": "Vidi komentojn de YouTube", "View more comments on Reddit": "Vidi pli komentoj en Reddit", @@ -315,4 +316,4 @@ "Videos": "Videoj", "Playlists": "Ludlistoj", "Current version: ": "Nuna versio: " -} +} \ No newline at end of file diff --git a/locales/es.json b/locales/es.json index e0fac8a27..10fbf5ca6 100644 --- a/locales/es.json +++ b/locales/es.json @@ -136,6 +136,7 @@ "Shared `x`": "Compartido `x`", "`x` views": "`x` visualizaciones", "Premieres in `x`": "Se estrena en `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "¡Hola! Parece que tiene JavaScript desactivado. Haga clic aquí para ver los comentarios, pero tenga en cuenta que pueden tardar un poco más en cargarse.", "View YouTube comments": "Ver los comentarios de YouTube", "View more comments on Reddit": "Ver más comentarios en Reddit", diff --git a/locales/fr.json b/locales/fr.json index e2d586aec..72b12e926 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -136,6 +136,7 @@ "Shared `x`": "Ajoutée le `x`", "`x` views": "`x` vues", "Premieres in `x`": "Première dans `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Il semblerait que JavaScript soit désactivé. Cliquez ici pour voir les commentaires sans. Gardez à l'esprit que le chargement peut prendre plus de temps.", "View YouTube comments": "Voir les commentaires YouTube", "View more comments on Reddit": "Voir plus de commentaires sur Reddit", diff --git a/locales/nb_NO.json b/locales/nb_NO.json index acde88b67..e33004cd9 100644 --- a/locales/nb_NO.json +++ b/locales/nb_NO.json @@ -136,6 +136,7 @@ "Shared `x`": "Delt `x`", "`x` views": "`x` visninger", "Premieres in `x`": "Premiere om `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Hei. Det ser ut til at du har JavaScript avslått. Klikk her for å vise kommentarer, ha i minnet at innlasting tar lengre tid.", "View YouTube comments": "Vis YouTube-kommentarer", "View more comments on Reddit": "Vis flere kommenterer på Reddit", @@ -315,4 +316,4 @@ "Videos": "Videoer", "Playlists": "Spillelister", "Current version: ": "Nåværende versjon: " -} +} \ No newline at end of file diff --git a/locales/nl.json b/locales/nl.json index 2cc716efd..d5a4907e3 100644 --- a/locales/nl.json +++ b/locales/nl.json @@ -136,6 +136,7 @@ "Shared `x`": "`x` gedeeld", "`x` views": "`x` weergaven", "Premieres in `x`": "Verschijnt over `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Hoi! Het lijkt erop dat je JavaScript hebt uitgeschakeld. Klik hier om de reacties te bekijken. Let op: het laden duurt wat langer.", "View YouTube comments": "YouTube-reacties tonen", "View more comments on Reddit": "Meer reacties bekijken op Reddit", @@ -315,4 +316,4 @@ "Videos": "Video's", "Playlists": "Afspeellijsten", "Current version: ": "Huidige versie: " -} +} \ No newline at end of file diff --git a/locales/pl.json b/locales/pl.json index fa4ec965a..52875f03f 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -136,6 +136,7 @@ "Shared `x`": "Udostępniono `x`", "`x` views": "`x` wyświetleń", "Premieres in `x`": "Publikacja za `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Cześć! Wygląda na to, że masz wyłączoną obsługę JavaScriptu. Kliknij tutaj, żeby zobaczyć komentarze. Pamiętaj, że wczytywanie może potrwać dłużej.", "View YouTube comments": "Wyświetl komentarze z YouTube", "View more comments on Reddit": "Wyświetl więcej komentarzy na Reddicie", diff --git a/locales/ru.json b/locales/ru.json index e603b98f2..b51589f45 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -136,6 +136,7 @@ "Shared `x`": "Опубликовано `x`", "`x` views": "`x` просмотров", "Premieres in `x`": "Премьера через `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Похоже, у вас отключён JavaScript. Чтобы увидить комментарии, нажмите сюда, но учтите: они могут загружаться немного медленнее.", "View YouTube comments": "Смотреть комментарии с YouTube", "View more comments on Reddit": "Посмотреть больше комментариев на Reddit", diff --git a/locales/uk.json b/locales/uk.json index 319f22d7e..1bc281e9c 100644 --- a/locales/uk.json +++ b/locales/uk.json @@ -136,6 +136,7 @@ "Shared `x`": "Розміщено `x`", "`x` views": "`x` переглядів", "Premieres in `x`": "Прем’єра через `x`", + "Premieres `x`": "", "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "Схоже, у вас відключений JavaScript. Щоб побачити коментарі, натисніть сюда, але майте на увазі, що вони можуть завантажуватися трохи довше.", "View YouTube comments": "Переглянути коментарі з YouTube", "View more comments on Reddit": "Переглянути більше коментарів на Reddit", diff --git a/src/invidious.cr b/src/invidious.cr index 63c441778..2ea2c5180 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -519,7 +519,7 @@ get "/watch" do |env| engagement = ((video.dislikes.to_f + video.likes.to_f)/video.views * 100) playability_status = video.player_response["playabilityStatus"]? - if playability_status && playability_status["status"] == "LIVE_STREAM_OFFLINE" + if playability_status && playability_status["status"] == "LIVE_STREAM_OFFLINE" && !video.premiere_timestamp reason = playability_status["reason"]?.try &.as_s end reason ||= "" diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 5765d0c8d..78a52db3a 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -851,8 +851,12 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}) if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) && !region video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) - # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours) - if (refresh && Time.utc - video.updated > 10.minutes) || force_refresh + # If record was last updated over 10 minutes ago, or video has since premiered, + # refresh (expire param in response lasts for 6 hours) + if (refresh && + (Time.utc - video.updated > 10.minutes) || + (video.premiere_timestamp && video.premiere_timestamp.as(Time) < Time.utc)) || + force_refresh begin video = fetch_video(id, proxies, region) video_array = video.to_a diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index b6307b9c6..7fa5f45bb 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -30,7 +30,8 @@ var video_data = { length_seconds: '<%= video.info["length_seconds"].to_f %>', video_series: <%= video_series.to_json %>, params: <%= params.to_json %>, - preferences: <%= preferences.to_json %> + preferences: <%= preferences.to_json %>, + premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %> } diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 2f03190ae..85ca8b8be 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -40,7 +40,8 @@ var video_data = { hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>', show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>', params: <%= params.to_json %>, - preferences: <%= preferences.to_json %> + preferences: <%= preferences.to_json %>, + premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %> } @@ -72,6 +73,10 @@ var video_data = {

<%= reason %>

+ <% elsif video.premiere_timestamp %> +

+ <%= translate(locale, "Premieres in `x`", recode_date((video.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %> +

<% end %>
@@ -172,7 +177,11 @@ var video_data = { <%= rendered "components/subscribe_widget" %>

- <%= translate(locale, "Shared `x`", video.published.to_s("%B %-d, %Y")) %> + <% if video.premiere_timestamp %> + <%= translate(locale, "Premieres `x`", video.premiere_timestamp.not_nil!.to_s("%B %-d, %R UTC")) %> + <% else %> + <%= translate(locale, "Shared `x`", video.published.to_s("%B %-d, %Y")) %> + <% end %>

From 9122f8acee758794f168989c9da5234bd4c6e688 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 8 Jun 2019 10:52:47 -0500 Subject: [PATCH 039/146] Add title overlay to embedded videos --- assets/css/default.css | 6 ++++++ assets/css/videojs-overlay.css | 1 + assets/js/player.js | 16 ++++++++++++++++ assets/js/videojs-overlay.min.js | 2 ++ src/invidious/views/embed.ecr | 2 ++ src/invidious/views/licenses.ecr | 14 ++++++++++++++ 6 files changed, 41 insertions(+) create mode 100644 assets/css/videojs-overlay.css create mode 100644 assets/js/videojs-overlay.min.js diff --git a/assets/css/default.css b/assets/css/default.css index bdb3e18e3..0727cfddd 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -351,6 +351,12 @@ span > select { background-color: rgba(0, 182, 240, 1); } +/* Overlay */ +.video-js .vjs-overlay { + background-color: rgba(35, 35, 35, 0.5); + color: rgba(255, 255, 255, 1); +} + /* ProgressBar marker */ .vjs-marker { background-color: rgba(255, 255, 255, 1); diff --git a/assets/css/videojs-overlay.css b/assets/css/videojs-overlay.css new file mode 100644 index 000000000..3ba5a574d --- /dev/null +++ b/assets/css/videojs-overlay.css @@ -0,0 +1 @@ +.video-js .vjs-overlay{color:#fff;position:absolute;text-align:center}.video-js .vjs-overlay-no-background{max-width:33%}.video-js .vjs-overlay-background{background-color:#646464;background-color:rgba(255,255,255,0.4);border-radius:3px;padding:10px;width:33%}.video-js .vjs-overlay-top-left{top:5px;left:5px}.video-js .vjs-overlay-top{left:50%;margin-left:-16.5%;top:5px}.video-js .vjs-overlay-top-right{right:5px;top:5px}.video-js .vjs-overlay-right{right:5px;top:50%;transform:translateY(-50%)}.video-js .vjs-overlay-bottom-right{bottom:3.5em;right:5px}.video-js .vjs-overlay-bottom{bottom:3.5em;left:50%;margin-left:-16.5%}.video-js .vjs-overlay-bottom-left{bottom:3.5em;left:5px}.video-js .vjs-overlay-left{left:5px;top:50%;transform:translateY(-50%)}.video-js .vjs-overlay-center{left:50%;margin-left:-16.5%;top:50%;transform:translateY(-50%)}.video-js .vjs-no-flex .vjs-overlay-left,.video-js .vjs-no-flex .vjs-overlay-center,.video-js .vjs-no-flex .vjs-overlay-right{margin-top:-15px} diff --git a/assets/js/player.js b/assets/js/player.js index 8854d859d..ae9a8b257 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -102,6 +102,22 @@ var player = videojs('player', options, function () { }); }); +if (location.pathname.startsWith('/embed/')) { + player.overlay({ + overlays: [{ + start: 'loadstart', + content: '

' + player_data.title + '

', + end: 'playing', + align: 'top' + }, { + start: 'pause', + content: '

' + player_data.title + '

', + end: 'playing', + align: 'top' + }] + }); +} + player.on('error', function (event) { if (player.error().code === 2 || player.error().code === 4) { setInterval(setTimeout(function (event) { diff --git a/assets/js/videojs-overlay.min.js b/assets/js/videojs-overlay.min.js new file mode 100644 index 000000000..8182c26cf --- /dev/null +++ b/assets/js/videojs-overlay.min.js @@ -0,0 +1,2 @@ +/*! @name videojs-overlay @version 2.1.4 @license Apache-2.0 */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e(require("video.js"),require("global/window")):"function"==typeof define&&define.amd?define(["video.js","global/window"],e):t.videojsOverlay=e(t.videojs,t.window)}(this,function(t,e){"use strict";function n(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}t=t&&t.hasOwnProperty("default")?t.default:t,e=e&&e.hasOwnProperty("default")?e.default:e;var r={align:"top-left",class:"",content:"This overlay will show up while the video is playing",debug:!1,showBackground:!0,attachToControlBar:!1,overlays:[{start:"playing",end:"paused"}]},i=t.getComponent("Component"),o=t.dom||t,s=t.registerPlugin||t.plugin,a=function(t){return"number"==typeof t&&t==t},h=function(t){return"string"==typeof t&&/^\S+$/.test(t)},d=function(r){var i,s;function d(t,e){var i;return i=r.call(this,t,e)||this,["start","end"].forEach(function(t){var e=i.options_[t];if(a(e))i[t+"Event_"]="timeupdate";else if(h(e))i[t+"Event_"]=e;else if("start"===t)throw new Error('invalid "start" option; expected number or string')}),["endListener_","rewindListener_","startListener_"].forEach(function(t){i[t]=function(e){return d.prototype[t].call(n(n(i)),e)}}),"timeupdate"===i.startEvent_&&i.on(t,"timeupdate",i.rewindListener_),i.debug('created, listening to "'+i.startEvent_+'" for "start" and "'+(i.endEvent_||"nothing")+'" for "end"'),i.hide(),i}s=r,(i=d).prototype=Object.create(s.prototype),i.prototype.constructor=i,i.__proto__=s;var l=d.prototype;return l.createEl=function(){var t=this.options_,n=t.content,r=t.showBackground?"vjs-overlay-background":"vjs-overlay-no-background",i=o.createEl("div",{className:"\n vjs-overlay\n vjs-overlay-"+t.align+"\n "+t.class+"\n "+r+"\n vjs-hidden\n "});return"string"==typeof n?i.innerHTML=n:n instanceof e.DocumentFragment?i.appendChild(n):o.appendContent(i,n),i},l.debug=function(){if(this.options_.debug){for(var e=t.log,n=e,r=arguments.length,i=new Array(r),o=0;o=n:n===e},l.show=function(){return r.prototype.show.call(this),this.off(this.player(),this.startEvent_,this.startListener_),this.debug("shown"),this.debug('unbound `startListener_` from "'+this.startEvent_+'"'),this.endEvent_&&(this.debug('bound `endListener_` to "'+this.endEvent_+'"'),this.on(this.player(),this.endEvent_,this.endListener_)),this},l.shouldShow_=function(t,e){var n=this.options_.start,r=this.options_.end;return a(n)?a(r)?t>=n&&t=n):n===e},l.startListener_=function(t){var e=this.player().currentTime();this.shouldShow_(e,t.type)&&this.show()},l.endListener_=function(t){var e=this.player().currentTime();this.shouldHide_(e,t.type)&&this.hide()},l.rewindListener_=function(t){var e=this.player().currentTime(),n=this.previousTime_,r=this.options_.start,i=this.options_.end;e <%= rendered "components/player_sources" %> + + <%= HTML.escape(video.title) %> - Invidious