Merge f65627f55f into 53e8a5d62d

* Update to latest version WEB_CREATOR

* fix comment about using web embed as a fallback

.github/CODEOWNERS

@@ -6,7 +6,7 @@ docker/ @unixfox
 kubernetes/ @unixfox
 
 README.md @thefrenchghosty
-config/config.example.yml @thefrenchghosty @SamantazFox @unixfox
+config/config.example.yml @SamantazFox @unixfox
 
 scripts/ @syeopite
 shards.lock @syeopite

assets/js/comments.js

@@ -10,6 +10,14 @@ String.prototype.supplant = function (o) {
     });
 };
 
+function updateReplyLinks() {
+    document.querySelectorAll("a[href^='/comment_viewer']").forEach(function (replyLink) {
+        replyLink.setAttribute("href", "javascript:void(0)");
+        replyLink.removeAttribute("target");
+    });
+}
+updateReplyLinks();
+
 function toggle_comments(event) {
     var target = event.target;
     var body = target.parentNode.parentNode.parentNode.children[1];
@@ -104,6 +112,7 @@ function get_youtube_comments() {
                 })
             });
             comments.innerHTML = commentInnerHtml;
+            updateReplyLinks();
             comments.children[0].children[0].children[0].onclick = toggle_comments;
             if (video_data.support_reddit) {
                 comments.children[0].children[1].children[0].onclick = swap_comments;
@@ -143,6 +152,7 @@ function get_youtube_replies(target, load_more, load_replies) {
                 body = body.parentNode.parentNode;
                 body.removeChild(body.lastElementChild);
                 body.insertAdjacentHTML('beforeend', response.contentHtml);
+                updateReplyLinks();
             } else {
                 body.removeChild(body.lastElementChild);
 
@@ -161,6 +171,7 @@ function get_youtube_replies(target, load_more, load_replies) {
 
                 body.appendChild(p);
                 body.appendChild(div);
+                updateReplyLinks();
             }
         },
         onNon200: function (xhr) {

assets/js/community.js

@@ -1,37 +1,20 @@
 'use strict';
 var community_data = JSON.parse(document.getElementById('community_data').textContent);
 
-function hide_youtube_replies(event) {
-    var target = event.target;
+// first page of community posts are loaded without javascript so we need to update the Load more button
+var initialLoadMore = document.querySelector('a[data-onclick="get_youtube_replies"]');
+initialLoadMore.setAttribute('href', 'javascript:void(0);');
+initialLoadMore.removeAttribute('target');
 
-    var sub_text = target.getAttribute('data-inner-text');
-    var inner_text = target.getAttribute('data-sub-text');
-
-    var body = target.parentNode.parentNode.children[1];
-    body.style.display = 'none';
-
-    target.innerHTML = sub_text;
-    target.onclick = show_youtube_replies;
-    target.setAttribute('data-inner-text', inner_text);
-    target.setAttribute('data-sub-text', sub_text);
+function updateReplyLinks() {
+    document.querySelectorAll("a[href^='/comment_viewer']").forEach(function (replyLink) {
+        replyLink.setAttribute("href", "javascript:void(0)");
+        replyLink.removeAttribute("target");
+    });
 }
+updateReplyLinks();
 
-function show_youtube_replies(event) {
-    var target = event.target;
-
-    var sub_text = target.getAttribute('data-inner-text');
-    var inner_text = target.getAttribute('data-sub-text');
-
-    var body = target.parentNode.parentNode.children[1];
-    body.style.display = '';
-
-    target.innerHTML = sub_text;
-    target.onclick = hide_youtube_replies;
-    target.setAttribute('data-inner-text', inner_text);
-    target.setAttribute('data-sub-text', sub_text);
-}
-
-function get_youtube_replies(target, load_more) {
+function get_youtube_replies(target) {
     var continuation = target.getAttribute('data-continuation');
 
     var body = target.parentNode.parentNode;
@@ -47,29 +30,10 @@ function get_youtube_replies(target, load_more) {
 
     helpers.xhr('GET', url, {}, {
         on200: function (response) {
-            if (load_more) {
-                body = body.parentNode.parentNode;
-                body.removeChild(body.lastElementChild);
-                body.innerHTML += response.contentHtml;
-            } else {
-                body.removeChild(body.lastElementChild);
-
-                var p = document.createElement('p');
-                var a = document.createElement('a');
-                p.appendChild(a);
-
-                a.href = 'javascript:void(0)';
-                a.onclick = hide_youtube_replies;
-                a.setAttribute('data-sub-text', community_data.hide_replies_text);
-                a.setAttribute('data-inner-text', community_data.show_replies_text);
-                a.textContent = community_data.hide_replies_text;
-
-                var div = document.createElement('div');
-                div.innerHTML = response.contentHtml;
-
-                body.appendChild(p);
-                body.appendChild(div);
-            }
+            body = body.parentNode.parentNode;
+            body.removeChild(body.lastElementChild);
+            body.insertAdjacentHTML('beforeend', response.contentHtml);
+            updateReplyLinks();
         },
         onNon200: function (xhr) {
             body.innerHTML = fallback;

src/invidious/channels/community.cr

@@ -279,7 +279,7 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing
 
   if format == "html"
     response = JSON.parse(response)
-    content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode)
+    content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode, ucid, "community")
 
     response = JSON.build do |json|
       json.object do

src/invidious/comments/youtube.cr

@@ -57,7 +57,7 @@ module Invidious::Comments
     return initial_data
   end
 
-  def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false)
+  def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", type = "video", ucid = nil)
     contents = nil
 
     if on_response_received_endpoints = response["onResponseReceivedEndpoints"]?
@@ -115,7 +115,11 @@ module Invidious::Comments
           json.field "commentCount", comment_count
         end
 
-        if is_post
+        if !ucid.nil?
+          json.field "authorId", ucid
+        end
+
+        if type == "post"
           json.field "postId", id
         else
           json.field "videoId", id
@@ -302,7 +306,8 @@ module Invidious::Comments
 
     if format == "html"
       response = JSON.parse(response)
-      content_html = Frontend::Comments.template_youtube(response, locale, thin_mode)
+      content_html = Frontend::Comments.template_youtube(response, locale, thin_mode, id, type)
+
       response = JSON.build do |json|
         json.object do
           json.field "contentHtml", content_html

src/invidious/frontend/comments_youtube.cr

@@ -1,7 +1,7 @@
 module Invidious::Frontend::Comments
   extend self
 
-  def template_youtube(comments, locale, thin_mode, is_replies = false)
+  def template_youtube(comments, locale, thin_mode, id, type = "video", is_replies = false)
     String.build do |html|
       root = comments["comments"].as_a
       root.each do |child|
@@ -13,17 +13,17 @@ module Invidious::Frontend::Comments
           )
 
           replies_html = <<-END_HTML
-          <div id="replies" class="pure-g">
+          <div class="pure-g replies">
             <div class="pure-u-1-24"></div>
             <div class="pure-u-23-24">
               <p>
-                <a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}"
+                <a target="_blank" href="/comment_viewer?continuation=#{child["replies"]["continuation"]}&id=#{id}&type=#{type}" data-continuation="#{child["replies"]["continuation"]}"
                   data-onclick="get_youtube_replies" data-load-replies>#{replies_count_text}</a>
               </p>
             </div>
           </div>
           END_HTML
-        elsif comments["authorId"]? && !comments["singlePost"]?
+        elsif comments["authorId"]? && !comments["singlePost"]? && type != "post"
           # for posts we should display a link to the post
           replies_count_text = translate_count(locale,
             "comments_view_x_replies",
@@ -147,7 +147,12 @@ module Invidious::Frontend::Comments
           |
         END_HTML
 
-        if comments["videoId"]?
+        if type == "post" && !comments["singlePost"]?
+          html << <<-END_HTML
+            <a href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{id}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a>
+            |
+          END_HTML
+        elsif comments["videoId"]?
           html << <<-END_HTML
             <a rel="noreferrer noopener" href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a>
             |
@@ -196,7 +201,7 @@ module Invidious::Frontend::Comments
         <div class="pure-g">
           <div class="pure-u-1">
             <p>
-              <a href="javascript:void(0)" data-continuation="#{comments["continuation"]}"
+              <a target="_blank" href="/comment_viewer?continuation=#{comments["continuation"]}&id=#{id}&type=#{type}" data-continuation="#{comments["continuation"]}"
                 data-onclick="get_youtube_replies" data-load-more #{"data-load-replies" if is_replies}>#{translate(locale, "Load more")}</a>
             </p>
           </div>

src/invidious/routes/api/v1/channels.cr

@@ -394,7 +394,7 @@ module Invidious::Routes::API::V1::Channels
     locale = env.get("preferences").as(Preferences).locale
 
     env.response.content_type = "application/json"
-    id = env.params.url["id"].to_s
+    id = URI.encode_www_form(env.params.url["id"].to_s)
     ucid = env.params.query["ucid"]?
 
     thin_mode = env.params.query["thin_mode"]?
@@ -406,9 +406,9 @@ module Invidious::Routes::API::V1::Channels
     if ucid.nil?
       response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
       return error_json(400, "Invalid post ID") if response["error"]?
-      ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+      ucid = URI.encode_www_form(response.dig("endpoint", "browseEndpoint", "browseId").as_s)
     else
-      ucid = ucid.to_s
+      ucid = URI.encode_www_form(ucid.to_s)
     end
 
     begin
@@ -423,7 +423,7 @@ module Invidious::Routes::API::V1::Channels
 
     env.response.content_type = "application/json"
 
-    id = env.params.url["id"]
+    id = URI.encode_www_form(env.params.url["id"])
 
     thin_mode = env.params.query["thin_mode"]?
     thin_mode = thin_mode == "true"
@@ -432,15 +432,23 @@ module Invidious::Routes::API::V1::Channels
     format ||= "json"
 
     continuation = env.params.query["continuation"]?
+    ucid = env.params.query["ucid"]?
+
+    if ucid.nil?
+      response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+      return error_json(400, "Invalid post ID") if response["error"]?
+      ucid = URI.encode_www_form(response.dig("endpoint", "browseEndpoint", "browseId").as_s)
+    else
+      ucid = URI.encode_www_form(ucid.to_s)
+    end
 
     case continuation
     when nil, ""
-      ucid = env.params.query["ucid"]
       comments = Comments.fetch_community_post_comments(ucid, id)
     else
       comments = YoutubeAPI.browse(continuation: continuation)
     end
-    return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true)
+    return Comments.parse_youtube(id, comments, format, locale, thin_mode, type: "post", ucid: ucid)
   end
 
   def self.channels(env)

src/invidious/routes/api/v1/videos.cr

@@ -392,7 +392,7 @@ module Invidious::Routes::API::V1::Videos
 
     env.response.content_type = "application/json"
 
-    clip_id = env.params.url["id"]
+    clip_id = URI.encode_www_form(env.params.url["id"])
     region = env.params.query["region"]?
     proxy = {"1", "true"}.any? &.== env.params.query["local"]?
 

src/invidious/routes/channels.cr

@@ -237,7 +237,7 @@ module Invidious::Routes::Channels
 
   def self.post(env)
     # /post/{postId}
-    id = env.params.url["id"]
+    id = URI.encode_www_form(env.params.url["id"])
     ucid = env.params.query["ucid"]?
 
     prefs = env.get("preferences").as(Preferences)
@@ -253,14 +253,14 @@ module Invidious::Routes::Channels
     nojs = nojs == "1"
 
     if !ucid.nil?
-      ucid = ucid.to_s
+      ucid = URI.encode_www_form(ucid.to_s)
       post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
     else
       # resolve the url to get the author's UCID
       response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
       return error_template(400, "Invalid post ID") if response["error"]?
 
-      ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+      ucid = URI.encode_www_form(response.dig("endpoint", "browseEndpoint", "browseId").as_s)
       post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
     end
 
@@ -268,7 +268,7 @@ module Invidious::Routes::Channels
 
     if nojs
       comments = Comments.fetch_community_post_comments(ucid, id)
-      comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"]
+      comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, type: "post", ucid: ucid))["contentHtml"]
     end
     templated "post"
   end

src/invidious/routes/watch.cr

@@ -332,4 +332,53 @@ module Invidious::Routes::Watch
       return error_template(400, "Invalid label or itag")
     end
   end
+
+  # used for fetching replies/ fetching more comments when js is disabled.
+  def self.comments(env)
+    locale = env.get("preferences").as(Preferences).locale
+    region = env.params.query["region"]?
+
+    id = URI.encode_www_form(env.params.query["id"])
+    continuation = env.params.query["continuation"]?
+
+    source = env.params.query["source"]? || "youtube"
+
+    thin_mode = env.params.query["thin_mode"]? == "true"
+    comment_type = env.params.query["type"]? || "video"
+
+    parent_comment = nil
+    if comment_type == "community"
+      # community posts
+      comment_html = JSON.parse(fetch_channel_community(id, continuation, locale, "html", thin_mode))["contentHtml"]
+    elsif comment_type == "post"
+      # replies to a community post
+      ucid = env.params.query["ucid"]?
+      if ucid.nil?
+        response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+        return error_json(400, "Invalid post ID") if response["error"]?
+        ucid = URI.encode_www_form(response.dig("endpoint", "browseEndpoint", "browseId").as_s)
+      else
+        ucid = URI.encode_www_form(ucid.to_s)
+      end
+      case continuation
+      when nil, ""
+        comments = Comments.fetch_community_post_comments(ucid, id)
+      else
+        comments = YoutubeAPI.browse(continuation: continuation)
+      end
+      comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, type: "post", ucid: ucid))["contentHtml"]
+    else
+      # video comments
+      if source == "youtube"
+        comment_html = JSON.parse(Comments.fetch_youtube(id, continuation, "html", locale, thin_mode, region))["contentHtml"]
+      elsif source == "reddit"
+        comments, reddit_thread = Comments.fetch_reddit(id)
+        comment_html = Frontend::Comments.template_reddit(comments, locale)
+
+        comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
+        comment_html = Comments.replace_links(comment_html)
+      end
+    end
+    templated "comments_no_js"
+  end
 end

src/invidious/routing.cr

@@ -170,6 +170,8 @@ module Invidious::Routing
 
     get "/embed/", Routes::Embed, :redirect
     get "/embed/:id", Routes::Embed, :show
+    # currently only for fetching continuations when js is disabled.
+    get "/comment_viewer", Routes::Watch, :comments
   end
 
   def register_yt_playlist_routes

src/invidious/videos/parser.cr

@@ -53,6 +53,10 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  if CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
@@ -102,6 +106,13 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
+  # Second try in case WEB_CREATOR doesn't work with po_token.
+  # Only trigger if reason found and po_token configured.
+  if reason && CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
   if reason.nil? && CONFIG.po_token.nil?
@@ -114,10 +125,9 @@ def extract_video_info(video_id : String)
   end
 
   # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  # Only trigger if reason found or didn't work wth Android client.
+  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
+  if reason && CONFIG.po_token.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -185,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
-  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+  if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+    microformat = {} of String => JSON::Any
+  end
 
   raise BrokenTubeException.new("videoDetails") if !video_details
-  raise BrokenTubeException.new("microformat") if !microformat
 
   # Basic video infos
 
@@ -225,7 +236,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
-  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  family_friendly = microformat["isFamilySafe"]?.try &.as_bool
   is_listed = video_details["isCrawlable"]?.try &.as_bool
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 

src/invidious/views/comments_no_js.ecr

@@ -0,0 +1,8 @@
+<% content_for "header" do %>
+<title>Invidious</title>
+<% end %>
+
+<!-- basic comments page for people with js disabled. -->
+<div class="comments post-comments">
+    <%= comment_html %>
+</div>

src/invidious/views/community.ecr

@@ -27,7 +27,7 @@
     </div>
 <% else %>
     <div class="h-box pure-g comments" id="comments">
-        <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %>
+        <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode, ucid, "community") %>
     </div>
 <% end %>
 

src/invidious/views/post.ecr

@@ -4,7 +4,7 @@
 
 <div>
     <div id="post" class="comments post-comments">
-        <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %>
+        <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode, id, "post") %>
     </div>
 
     <% if nojs %>

src/invidious/yt_backend/youtube_api.cr

@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -80,6 +81,14 @@ module YoutubeAPI
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20240918.03.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -291,8 +300,9 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
+      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+        "embedUrl" => "https://www.google.com/",
       } of String => String | Int64
     end