Merge branch 'master' into api-only

locales/en-US.json

@@ -0,0 +1,155 @@
+{
+  "`x` subscribers": "`x` subscribers",
+  "`x` videos": "`x` videos",
+  "LIVE": "LIVE",
+  "Shared `x` ago": "Shared `x` ago",
+  "Unsubscribe": "Unsubscribe",
+  "Subscribe": "Subscribe",
+  "Login to subscribe to `x`": "Login to subscribe to `x`",
+  "View channel on YouTube": "View channel on YouTube",
+  "newest": "newest",
+  "oldest": "oldest",
+  "popular": "popular",
+  "Preview page": "Preview page",
+  "Next page": "Next page",
+  "Clear watch history?": "Clear watch history?",
+  "Yes": "Yes",
+  "No": "No",
+  "Import and Export Data": "Import and Export Data",
+  "Import": "Import",
+  "Import Invidious data": "Import Invidious data",
+  "Import YouTube subscriptions": "Import YouTube subscriptions",
+  "Import FreeTube subscriptions (.db)": "Import FreeTube subscriptions (.db)",
+  "Import NewPipe subscriptions (.json)": "Import NewPipe subscriptions (.json)",
+  "Import NewPipe data (.zip)": "Import NewPipe data (.zip)",
+  "Export": "Export",
+  "Export subscriptions as OPML": "Export subscriptions as OPML",
+  "Export subscriptions as OPML (for NewPipe & FreeTube)": "Export subscriptions as OPML (for NewPipe & FreeTube)",
+  "Export data as JSON": "Export data as JSON",
+  "Delete account?": "Delete account?",
+  "History": "History",
+  "Previous page": "Previous page",
+  "An alternative front-end to YouTube": "An alternative front-end to YouTube",
+  "JavaScript license information": "JavaScript license information",
+  "source": "source",
+  "Login": "Login",
+  "Login/Register": "Login/Register",
+  "Login to Google": "Login to Google",
+  "User ID:": "User ID:",
+  "Password:": "Password:",
+  "Time (h:mm:ss):": "Time (h:mm:ss):",
+  "Text CAPTCHA": "Text CAPTCHA",
+  "Image CAPTCHA": "Image CAPTCHA",
+  "Sign In": "Sign In",
+  "Register": "Register",
+  "Email:": "Email:",
+  "Google verification code:": "Google verification code:",
+  "Preferences": "Preferences",
+  "Player preferences": "Player preferences",
+  "Always loop: ": "Always loop: ",
+  "Autoplay: ": "Autoplay: ",
+  "Autoplay next video: ": "Autoplay next video: ",
+  "Listen by default: ": "Listen by default: ",
+  "Default speed: ": "Default speed: ",
+  "Preferred video quality: ": "Preferred video quality: ",
+  "Player volume: ": "Player volume: ",
+  "Default comments: ": "Default comments: ",
+  "Default captions: ": "Default captions: ",
+  "Fallback captions: ": "Fallback captions: ",
+  "Show related videos? ": "Show related videos? ",
+  "Visual preferences": "Visual preferences",
+  "Dark mode: ": "Dark mode: ",
+  "Thin mode: ": "Thin mode: ",
+  "Subscription preferences": "Subscription preferences",
+  "Redirect homepage to feed: ": "Redirect homepage to feed: ",
+  "Number of videos shown in feed: ": "Number of videos shown in feed: ",
+  "Sort videos by: ": "Sort videos by: ",
+  "published": "published",
+  "published - reverse": "published - reverse",
+  "alphabetically": "alphabetically",
+  "alphabetically - reverse": "alphabetically - reverse",
+  "channel name": "channel name",
+  "channel name - reverse": "channel name - reverse",
+  "Only show latest video from channel: ": "Only show latest video from channel: ",
+  "Only show latest unwatched video from channel: ": "Only show latest unwatched video from channel: ",
+  "Only show unwatched: ": "Only show unwatched: ",
+  "Only show notifications (if there are any): ": "Only show notifications (if there are any): ",
+  "Data preferences": "Data preferences",
+  "Clear watch history": "Clear watch history",
+  "Import/Export data": "Import/Export data",
+  "Manage subscriptions": "Manage subscriptions",
+  "Watch history": "Watch history",
+  "Delete account": "Delete account",
+  "Save preferences": "Save preferences",
+  "Subscription manager": "Subscription manager",
+  "`x` subscriptions": "`x` subscriptions",
+  "Import/Export": "Import/Export",
+  "unsubscribe": "unsubscribe",
+  "Subscriptions": "Subscriptions",
+  "`x` unseen notifications": "`x` unseen notifications",
+  "search": "search",
+  "Sign out": "Sign out",
+  "Released under the AGPLv3 by Omar Roth.": "Released under the AGPLv3 by Omar Roth.",
+  "Source available here.": "Source available here.",
+  "View JavaScript license information.": "View JavaScript license information.",
+  "Trending": "Trending",
+  "Watch video on Youtube": "Watch video on Youtube",
+  "Genre: ": "Genre: ",
+  "License: ": "License: ",
+  "Family friendly? ": "Family friendly? ",
+  "Wilson score: ": "Wilson score: ",
+  "Engagement: ": "Engagement: ",
+  "Whitelisted regions: ": "Whitelisted regions: ",
+  "Blacklisted regions: ": "Blacklisted regions: ",
+  "Shared `x`": "Shared `x`",
+  "Hi! Looks like you have JavaScript disabled. Click here to view comments, keep in mind it may take a bit longer to load.": "Hi! Looks like you have JavaScript disabled. Click here to view comments, keep in mind it may take a bit longer to load.",
+  "View YouTube comments": "View YouTube comments",
+  "View more comments on Reddit": "View more comments on Reddit",
+  "View `x` comments": "View `x` comments",
+  "View Reddit comments": "View Reddit comments",
+  "Hide replies": "Hide replies",
+  "Show replies": "Show replies",
+  "Incorrect password": "Incorrect password",
+  "Quota exceeded, try again in a few hours": "Quota exceeded, try again in a few hours",
+  "Unable to login, make sure two-factor authentication (Authenticator or SMS) is enabled.": "Unable to login, make sure two-factor authentication (Authenticator or SMS) is enabled.",
+  "Invalid TFA code": "Invalid TFA code",
+  "Login failed. This may be because two-factor authentication is not enabled on your account.": "Login failed. This may be because two-factor authentication is not enabled on your account.",
+  "Invalid answer": "Invalid answer",
+  "Invalid CAPTCHA": "Invalid CAPTCHA",
+  "CAPTCHA is a required field": "CAPTCHA is a required field",
+  "User ID is a required field": "User ID is a required field",
+  "Password is a required field": "Password is a required field",
+  "Invalid username or password": "Invalid username or password",
+  "Please sign in using 'Sign in with Google'": "Please sign in using 'Sign in with Google'",
+  "Password cannot be empty": "Password cannot be empty",
+  "Password cannot be longer than 55 characters": "Password cannot be longer than 55 characters",
+  "Please sign in": "Please sign in",
+  "Invidious Private Feed for `x`": "Invidious Private Feed for `x`",
+  "channel:`x`": "channel:`x`",
+  "Deleted or invalid channel": "Deleted or invalid channel",
+  "This channel does not exist.": "This channel does not exist.",
+  "Could not get channel info.": "Could not get channel info.",
+  "Could not fetch comments": "Could not fetch comments",
+  "View `x` replies": "View `x` replies",
+  "`x` ago": "`x` ago",
+  "Load more": "Load more",
+  "`x` points": "`x` points",
+  "Could not create mix.": "Could not create mix.",
+  "Playlist is empty": "Playlist is empty",
+  "Invalid playlist.": "Invalid playlist.",
+  "Playlist does not exist.": "Playlist does not exist.",
+  "Could not pull trending pages.": "Could not pull trending pages.",
+  "Hidden field \"challenge\" is a required field": "Hidden field \"challenge\" is a required field",
+  "Hidden field \"token\" is a required field": "Hidden field \"token\" is a required field",
+  "Invalid challenge": "Invalid challenge",
+  "Invalid token": "Invalid token",
+  "Invalid user": "Invalid user",
+  "Token is expired, please try again": "Token is expired, please try again",
+  "`x` years": "`x` years",
+  "`x` months": "`x` months",
+  "`x` weeks": "`x` weeks",
+  "`x` days": "`x` days",
+  "`x` hours": "`x` hours",
+  "`x` minutes": "`x` minutes",
+  "`x` seconds": "`x` seconds"
+}

src/invidious.cr

@@ -37,7 +37,7 @@ video_threads = CONFIG.video_threads
 
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
-  parser.on("-t THREADS", "--crawl-threads=THREADS", "Number of threads for crawling (default: #{crawl_threads})") do |number|
+  parser.on("-t THREADS", "--crawl-threads=THREADS", "Number of threads for crawling YouTube (default: #{crawl_threads})") do |number|
     begin
       crawl_threads = number.to_i
     rescue ex
@@ -606,6 +606,29 @@ get "/api/v1/channels/:ucid" do |env|
   is_family_friendly = channel_html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True"
   allowed_regions = channel_html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",")
 
+  related_channels = channel_html.xpath_nodes(%q(//div[contains(@class, "branded-page-related-channels")]/ul/li))
+  related_channels = related_channels.map do |node|
+    related_id = node["data-external-id"]?
+    related_id ||= ""
+
+    anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a))
+    related_title = anchor.try &.["title"]
+    related_title ||= ""
+
+    related_author_url = anchor.try &.["href"]
+    related_author_url ||= ""
+
+    related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"]
+    related_author_thumbnail ||= ""
+
+    {
+      id:               related_id,
+      author:           related_title,
+      author_url:       related_author_url,
+      author_thumbnail: related_author_thumbnail,
+    }
+  end
+
   total_views = 0_i64
   sub_count = 0_i64
   joined = Time.unix(0)
@@ -708,6 +731,32 @@ get "/api/v1/channels/:ucid" do |env|
           end
         end
       end
+
+      json.field "relatedChannels" do
+        json.array do
+          related_channels.each do |related_channel|
+            json.object do
+              json.field "author", related_channel[:author]
+              json.field "authorId", related_channel[:id]
+              json.field "authorUrl", related_channel[:author_url]
+
+              json.field "authorThumbnails" do
+                json.array do
+                  qualities = [32, 48, 76, 100, 176, 512]
+
+                  qualities.each do |quality|
+                    json.object do
+                      json.field "url", related_channel[:author_thumbnail].gsub("=s48-", "=s#{quality}-")
+                      json.field "width", quality
+                      json.field "height", quality
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
     end
   end
 

src/invidious/helpers/helpers.cr

@@ -156,7 +156,9 @@ def extract_items(nodeset, ucid = nil)
       )
     when .includes? "yt-lockup-channel"
       author = title.strip
-      ucid = id.split("/")[-1]
+
+      ucid = node.xpath_node(%q(.//button[contains(@class, "yt-uix-subscription-button")])).try &.["data-channel-external-id"]?
+      ucid ||= id.split("/")[-1]
 
       author_thumbnail = node.xpath_node(%q(.//div/span/img)).try &.["data-thumb"]?
       author_thumbnail ||= node.xpath_node(%q(.//div/span/img)).try &.["src"]

src/invidious/search.cr

@@ -51,12 +51,12 @@ alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
 def channel_search(query, page, channel)
   client = make_client(YT_URL)
 
-  response = client.get("/user/#{channel}")
+  response = client.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
   document = XML.parse_html(response.body)
   canonical = document.xpath_node(%q(//link[@rel="canonical"]))
 
   if !canonical
-    response = client.get("/channel/#{channel}")
+    response = client.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
     document = XML.parse_html(response.body)
     canonical = document.xpath_node(%q(//link[@rel="canonical"]))
   end

src/invidious/users.cr

@@ -262,6 +262,10 @@ def validate_response(challenge, token, user_id, operation, key, db)
 end
 
 def generate_captcha(key, db)
+  second = Random::Secure.rand(12)
+  second_angle = second * 30
+  second = second * 5
+
   minute = Random::Secure.rand(12)
   minute_angle = minute * 30
   minute = minute * 5
@@ -290,6 +294,7 @@ def generate_captcha(key, db)
   <text x="50"     y="15"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text>
 
   <circle cx="50" cy="50" r="3" fill="black"></circle>
+  <line id="second" transform="rotate(#{second_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="12" fill="black" stroke="black" stroke-width="1"></line>
   <line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line>
   <line id="hour"   transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line>
   </svg>
@@ -303,7 +308,7 @@ def generate_captcha(key, db)
     image = "data:image/png;base64,#{image}"
   end
 
-  answer = "#{hour}:#{minute.to_s.rjust(2, '0')}"
+  answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}"
   answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer)
 
   challenge, token = create_response(answer, "sign_in", key, db)

src/invidious/videos.cr

@@ -508,7 +508,7 @@ class VideoRedirect < Exception
 end
 
 def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true, region = nil)
-  if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool)
+  if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) && !region
     video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
 
     # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours)
@@ -534,7 +534,9 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32})
 
     args = arg_array(video_array)
 
-    db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array)
+    if !region
+      db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array)
+    end
   end
 
   return video

src/invidious/views/popular.ecr

@@ -0,0 +1,7 @@
+<div class="pure-g">
+<% popular_videos.each_slice(4) do |slice| %>
+        <% slice.each do |item| %>
+            <%= rendered "components/item" %>
+        <% end %>
+<% end %>
+</div>

src/invidious/views/top.ecr

@@ -0,0 +1,7 @@
+<div class="pure-g">
+<% top_videos.each_slice(4) do |slice| %>
+        <% slice.each do |item| %>
+            <%= rendered "components/item" %>
+        <% end %>
+<% end %>
+</div>