Add support for community post page/comments (#4010)

src/invidious/channels/community.cr

@@ -24,7 +24,33 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
   return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
 end
 
-def extract_channel_community(items, *, ucid, locale, format, thin_mode)
+def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
+  object = {
+    "2:string"    => "community",
+    "25:embedded" => {
+      "22:string" => post_id.to_s,
+    },
+    "45:embedded" => {
+      "2:varint" => 1_i64,
+      "3:varint" => 1_i64,
+    },
+  }
+  params = object.try { |i| Protodec::Any.cast_json(i) }
+    .try { |i| Protodec::Any.from_json(i) }
+    .try { |i| Base64.urlsafe_encode(i) }
+    .try { |i| URI.encode_www_form(i) }
+
+  initial_data = YoutubeAPI.browse(ucid, params: params)
+
+  items = [] of JSON::Any
+  extract_items(initial_data) do |item|
+    items << item
+  end
+
+  return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true)
+end
+
+def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false)
   if message = items[0]["messageRenderer"]?
     error_message = (message["text"]["simpleText"]? ||
                      message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
@@ -39,6 +65,9 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode)
   response = JSON.build do |json|
     json.object do
       json.field "authorId", ucid
+      if is_single_post
+        json.field "singlePost", true
+      end
       json.field "comments" do
         json.array do
           items.each do |post|
@@ -240,8 +269,10 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode)
           end
         end
       end
-      if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
-        json.field "continuation", extract_channel_community_cursor(cont.as_s)
+      if !is_single_post
+        if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
+          json.field "continuation", extract_channel_community_cursor(cont.as_s)
+        end
       end
     end
   end

src/invidious/comments/youtube.cr

@@ -13,6 +13,51 @@ module Invidious::Comments
 
     client_config = YoutubeAPI::ClientConfig.new(region: region)
     response = YoutubeAPI.next(continuation: ctoken, client_config: client_config)
+    return parse_youtube(id, response, format, locale, thin_mode, sort_by)
+  end
+
+  def fetch_community_post_comments(ucid, post_id)
+    object = {
+      "2:string"    => "community",
+      "25:embedded" => {
+        "22:string" => post_id,
+      },
+      "45:embedded" => {
+        "2:varint" => 1_i64,
+        "3:varint" => 1_i64,
+      },
+      "53:embedded" => {
+        "4:embedded" => {
+          "6:varint"  => 0_i64,
+          "27:varint" => 1_i64,
+          "29:string" => post_id,
+          "30:string" => ucid,
+        },
+        "8:string" => "comments-section",
+      },
+    }
+
+    object_parsed = object.try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+
+    object2 = {
+      "80226972:embedded" => {
+        "2:string" => ucid,
+        "3:string" => object_parsed,
+      },
+    }
+
+    continuation = object2.try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    initial_data = YoutubeAPI.browse(continuation: continuation)
+    return initial_data
+  end
+
+  def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", isPost = false)
     contents = nil
 
     if on_response_received_endpoints = response["onResponseReceivedEndpoints"]?
@@ -68,7 +113,11 @@ module Invidious::Comments
           json.field "commentCount", comment_count
         end
 
-        json.field "videoId", id
+        if isPost
+          json.field "postId", id
+        else
+          json.field "videoId", id
+        end
 
         json.field "comments" do
           json.array do

src/invidious/frontend/comments_youtube.cr

@@ -23,6 +23,24 @@ module Invidious::Frontend::Comments
             </div>
           </div>
           END_HTML
+        elsif comments["authorId"]? && !comments["singlePost"]?
+          # for posts we should display a link to the post
+          replies_count_text = translate_count(locale,
+            "comments_view_x_replies",
+            child["replyCount"].as_i64 || 0,
+            NumberFormatting::Separator
+          )
+
+          replies_html = <<-END_HTML
+          <div class="pure-g">
+            <div class="pure-u-1-24"></div>
+            <div class="pure-u-23-24">
+              <p>
+                <a href="/post/#{child["commentId"]}?ucid=#{comments["authorId"]}">#{replies_count_text}</a>
+              </p>
+            </div>
+          </div>
+          END_HTML
         end
 
         if !thin_mode

src/invidious/routes/api/v1/channels.cr

@@ -343,6 +343,59 @@ module Invidious::Routes::API::V1::Channels
     end
   end
 
+  def self.post(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    env.response.content_type = "application/json"
+    id = env.params.url["id"].to_s
+    ucid = env.params.query["ucid"]?
+
+    thin_mode = env.params.query["thin_mode"]?
+    thin_mode = thin_mode == "true"
+
+    format = env.params.query["format"]?
+    format ||= "json"
+
+    if ucid.nil?
+      response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+      return error_json(400, "Invalid post ID") if response["error"]?
+      ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+    else
+      ucid = ucid.to_s
+    end
+
+    begin
+      fetch_channel_community_post(ucid, id, locale, format, thin_mode)
+    rescue ex
+      return error_json(500, ex)
+    end
+  end
+
+  def self.post_comments(env)
+    locale = env.get("preferences").as(Preferences).locale
+
+    env.response.content_type = "application/json"
+
+    id = env.params.url["id"]
+
+    thin_mode = env.params.query["thin_mode"]?
+    thin_mode = thin_mode == "true"
+
+    format = env.params.query["format"]?
+    format ||= "json"
+
+    continuation = env.params.query["continuation"]?
+
+    case continuation
+    when nil, ""
+      ucid = env.params.query["ucid"]
+      comments = Comments.fetch_community_post_comments(ucid, id)
+    else
+      comments = YoutubeAPI.browse(continuation: continuation)
+    end
+    return Comments.parse_youtube(id, comments, format, locale, thin_mode, isPost: true)
+  end
+
   def self.channels(env)
     locale = env.get("preferences").as(Preferences).locale
     ucid = env.params.url["ucid"]

src/invidious/routes/api/v1/misc.cr

@@ -162,17 +162,20 @@ module Invidious::Routes::API::V1::Misc
       resolved_url = YoutubeAPI.resolve_url(url.as(String))
       endpoint = resolved_url["endpoint"]
       pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
-      if resolved_ucid = endpoint.dig?("watchEndpoint", "videoId")
-      elsif resolved_ucid = endpoint.dig?("browseEndpoint", "browseId")
-      elsif pageType == "WEB_PAGE_TYPE_UNKNOWN"
+      if pageType == "WEB_PAGE_TYPE_UNKNOWN"
         return error_json(400, "Unknown url")
       end
+
+      sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
+      params = sub_endpoint.try &.dig?("params")
     rescue ex
       return error_json(500, ex)
     end
     JSON.build do |json|
       json.object do
-        json.field "ucid", resolved_ucid.try &.as_s || ""
+        json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
+        json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
+        json.field "params", params.try &.as_s
         json.field "pageType", pageType
       end
     end

src/invidious/routes/channels.cr

@@ -165,6 +165,11 @@ module Invidious::Routes::Channels
     end
     locale, user, subscriptions, continuation, ucid, channel = data
 
+    # redirect to post page
+    if lb = env.params.query["lb"]?
+      env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
+    end
+
     thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
     thin_mode = thin_mode == "true"
 
@@ -193,6 +198,44 @@ module Invidious::Routes::Channels
     templated "community"
   end
 
+  def self.post(env)
+    # /post/{postId}
+    id = env.params.url["id"]
+    ucid = env.params.query["ucid"]?
+
+    prefs = env.get("preferences").as(Preferences)
+
+    locale = prefs.locale
+
+    thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode
+    thin_mode = thin_mode == "true"
+
+    nojs = env.params.query["nojs"]?
+
+    nojs ||= "0"
+    nojs = nojs == "1"
+
+    if !ucid.nil?
+      ucid = ucid.to_s
+      post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
+    else
+      # resolve the url to get the author's UCID
+      response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+      return error_template(400, "Invalid post ID") if response["error"]?
+
+      ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+      post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
+    end
+
+    post_response = JSON.parse(post_response)
+
+    if nojs
+      comments = Comments.fetch_community_post_comments(ucid, id)
+      comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, isPost: true))["contentHtml"]
+    end
+    templated "post"
+  end
+
   def self.channels(env)
     data = self.fetch_basic_information(env)
     return data if !data.is_a?(Tuple)

src/invidious/routing.cr

@@ -128,6 +128,7 @@ module Invidious::Routing
     get "/channel/:ucid/live", Routes::Channels, :live
     get "/user/:user/live", Routes::Channels, :live
     get "/c/:user/live", Routes::Channels, :live
+    get "/post/:id", Routes::Channels, :post
 
     # Channel catch-all, to redirect future routes to the channel's home
     # NOTE: defined last in order to be processed after the other routes
@@ -253,6 +254,10 @@ module Invidious::Routing
         get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
       {% end %}
 
+      # Posts
+      get "/api/v1/post/:id", {{namespace}}::Channels, :post
+      get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments
+
       # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
       get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
       get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect
@@ -262,6 +267,7 @@ module Invidious::Routing
       get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
       get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag
 
+
       # Authenticated
 
       # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr

src/invidious/views/community.ecr

@@ -26,7 +26,7 @@
         <p><%= error_message %></p>
     </div>
 <% else %>
-    <div class="h-box pure-g" id="comments">
+    <div class="h-box pure-g comments" id="comments">
         <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %>
     </div>
 <% end %>

src/invidious/views/post.ecr

@@ -0,0 +1,48 @@
+<% content_for "header" do %>
+<title>Invidious</title>
+<% end %>
+
+<div>
+    <div id="post" class="comments post-comments">
+        <%= IV::Frontend::Comments.template_youtube(post_response.not_nil!, locale, thin_mode) %>
+    </div>
+
+    <% if nojs %>
+        <hr>
+    <% end %>
+    <br />
+
+    <div id="comments" class="comments post-comments">
+        <% if nojs %>
+            <%= comment_html %>
+        <% else %>
+            <noscript>
+                <a href="/post/<%= id %>?ucid=<%= ucid %>&nojs=1">
+                    <%= translate(locale, "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.") %>
+                </a>
+            </noscript>
+        <% end %>
+    </div>
+</div>
+
+<script id="video_data" type="application/json">
+<%=
+{
+    "id" => id,
+    "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")),
+    "reddit_comments_text" => "",
+    "reddit_permalink_text" => "",
+    "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")),
+    "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")),
+    "show_replies_text" => HTML.escape(translate(locale, "Show replies")),
+    "params" => {
+        "comments": ["youtube"]
+    },
+    "preferences" => prefs,
+    "base_url" => "/api/v1/post/#{URI.encode_www_form(id)}/comments",
+    "ucid" => ucid
+}.to_pretty_json
+%>
+</script>
+<script src="/js/comments.js?v=<%= ASSET_COMMIT %>"></script>
+<script src="/js/post.js?v=<%= ASSET_COMMIT %>"></script>

src/invidious/views/watch.ecr

@@ -64,7 +64,8 @@ we're going to need to do it here in order to allow for translations.
     "premiere_timestamp" => video.premiere_timestamp.try &.to_unix,
     "vr" => video.is_vr,
     "projection_type" => video.projection_type,
-    "local_disabled" => CONFIG.disabled?("local")
+    "local_disabled" => CONFIG.disabled?("local"),
+    "support_reddit" => true
 }.to_pretty_json
 %>
 </script>
@@ -270,7 +271,7 @@ we're going to need to do it here in order to allow for translations.
             <hr>
 
             <% end %>
-            <div id="comments">
+            <div id="comments" class="comments">
                 <% if nojs %>
                     <%= comment_html %>
                 <% else %>
@@ -352,4 +353,5 @@ we're going to need to do it here in order to allow for translations.
         </div>
     <% end %>
 </div>
+<script src="/js/comments.js?v=<%= ASSET_COMMIT %>"></script>
 <script src="/js/watch.js?v=<%= ASSET_COMMIT %>"></script>