From bcd239ac2b438ec721ffe35868371a9c81573f0a Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 9 Jul 2019 09:31:04 -0500 Subject: [PATCH] Add community page --- assets/js/community.js | 101 ++++++++++++++++++++++++++++++ locales/ar.json | 1 + locales/de.json | 1 + locales/el.json | 1 + locales/en-US.json | 1 + locales/eo.json | 1 + locales/es.json | 1 + locales/fr.json | 1 + locales/it.json | 1 + locales/nb_NO.json | 1 + locales/nl.json | 1 + locales/pl.json | 1 + locales/ru.json | 1 + locales/uk.json | 1 + locales/zh-CN.json | 1 + src/invidious.cr | 59 ++++++++++++++++- src/invidious/channels.cr | 88 ++++++++++++++++++++++---- src/invidious/comments.cr | 85 ++++++++++++++++++++----- src/invidious/views/channel.ecr | 5 ++ src/invidious/views/community.ecr | 80 +++++++++++++++++++++++ src/invidious/views/licenses.ecr | 14 +++++ src/invidious/views/playlists.ecr | 7 ++- 22 files changed, 422 insertions(+), 31 deletions(-) create mode 100644 assets/js/community.js create mode 100644 src/invidious/views/community.ecr diff --git a/assets/js/community.js b/assets/js/community.js new file mode 100644 index 00000000..754ec6d3 --- /dev/null +++ b/assets/js/community.js @@ -0,0 +1,101 @@ +String.prototype.supplant = function (o) { + return this.replace(/{([^{}]*)}/g, function (a, b) { + var r = o[b]; + return typeof r === 'string' || typeof r === 'number' ? r : a; + }); +} + +function hide_youtube_replies(event) { + var target = event.target; + + sub_text = target.getAttribute('data-inner-text'); + inner_text = target.getAttribute('data-sub-text'); + + body = target.parentNode.parentNode.children[1]; + body.style.display = 'none'; + + target.innerHTML = sub_text; + target.onclick = show_youtube_replies; + target.setAttribute('data-inner-text', inner_text); + target.setAttribute('data-sub-text', sub_text); +} + +function show_youtube_replies(event) { + var target = event.target; + + sub_text = target.getAttribute('data-inner-text'); + inner_text = target.getAttribute('data-sub-text'); + + body = target.parentNode.parentNode.children[1]; + body.style.display = ''; + + target.innerHTML = sub_text; + target.onclick = hide_youtube_replies; + target.setAttribute('data-inner-text', inner_text); + target.setAttribute('data-sub-text', sub_text); +} + +function number_with_separator(val) { + while (/(\d+)(\d{3})/.test(val.toString())) { + val = val.toString().replace(/(\d+)(\d{3})/, '$1' + ',' + '$2'); + } + return val; +} + +function get_youtube_replies(target, load_more) { + var continuation = target.getAttribute('data-continuation'); + + var body = target.parentNode.parentNode; + var fallback = body.innerHTML; + body.innerHTML = + '

'; + + var url = '/api/v1/channels/comments/' + community_data.ucid + + '?format=html' + + '&hl=' + community_data.preferences.locale + + '&thin_mode=' + community_data.preferences.thin_mode + + '&continuation=' + continuation; + var xhr = new XMLHttpRequest(); + xhr.responseType = 'json'; + xhr.timeout = 10000; + xhr.open('GET', url, true); + + xhr.onreadystatechange = function () { + if (xhr.readyState == 4) { + if (xhr.status == 200) { + if (load_more) { + body = body.parentNode.parentNode; + body.removeChild(body.lastElementChild); + body.innerHTML += xhr.response.contentHtml; + } else { + body.removeChild(body.lastElementChild); + + var p = document.createElement('p'); + var a = document.createElement('a'); + p.appendChild(a); + + a.href = 'javascript:void(0)'; + a.onclick = hide_youtube_replies; + a.setAttribute('data-sub-text', community_data.hide_replies_text); + a.setAttribute('data-inner-text', community_data.show_replies_text); + a.innerText = community_data.hide_replies_text; + + var div = document.createElement('div'); + div.innerHTML = xhr.response.contentHtml; + + body.appendChild(p); + body.appendChild(div); + } + } else { + body.innerHTML = fallback; + } + } + } + + xhr.ontimeout = function () { + console.log('Pulling comments failed.'); + body.innerHTML = fallback; + } + + xhr.send(); +} diff --git a/locales/ar.json b/locales/ar.json index 0e89bd42..baaecab6 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -316,5 +316,6 @@ "Video mode": "وضع الفيديو", "Videos": "الفيديوهات", "Playlists": "قوائم التشغيل", + "Community": "", "Current version: ": "الإصدار الحالى" } \ No newline at end of file diff --git a/locales/de.json b/locales/de.json index 8bd91473..f4c96ed3 100644 --- a/locales/de.json +++ b/locales/de.json @@ -316,5 +316,6 @@ "Video mode": "Videomodus", "Videos": "Videos", "Playlists": "Wiedergabelisten", + "Community": "", "Current version: ": "Aktuelle Version: " } \ No newline at end of file diff --git a/locales/el.json b/locales/el.json index 0012f314..006e49bf 100644 --- a/locales/el.json +++ b/locales/el.json @@ -361,5 +361,6 @@ "Video mode": "Λειτουργία βίντεο", "Videos": "Βίντεο", "Playlists": "Λίστες Αναπαραγωγής", + "Community": "", "Current version: ": "Τρέχουσα έκδοση: " } \ No newline at end of file diff --git a/locales/en-US.json b/locales/en-US.json index 05f01819..9af291b8 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -361,5 +361,6 @@ "Video mode": "Video mode", "Videos": "Videos", "Playlists": "Playlists", + "Community": "Community", "Current version: ": "Current version: " } \ No newline at end of file diff --git a/locales/eo.json b/locales/eo.json index 59d7229c..861ec875 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -316,5 +316,6 @@ "Video mode": "Videa reĝimo", "Videos": "Videoj", "Playlists": "Ludlistoj", + "Community": "", "Current version: ": "Nuna versio: " } \ No newline at end of file diff --git a/locales/es.json b/locales/es.json index 394a3c31..f2d51d68 100644 --- a/locales/es.json +++ b/locales/es.json @@ -316,5 +316,6 @@ "Video mode": "Modo de vídeo", "Videos": "Vídeos", "Playlists": "Listas de reproducción", + "Community": "", "Current version: ": "Versión actual: " } \ No newline at end of file diff --git a/locales/fr.json b/locales/fr.json index 7c4c408c..c79183c8 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -316,5 +316,6 @@ "Video mode": "Mode Vidéo", "Videos": "Vidéos", "Playlists": "Liste de lecture", + "Community": "", "Current version: ": "Version actuelle : " } \ No newline at end of file diff --git a/locales/it.json b/locales/it.json index 1c07413d..4557df72 100644 --- a/locales/it.json +++ b/locales/it.json @@ -315,5 +315,6 @@ "Video mode": "Modalità video", "Videos": "", "Playlists": "", + "Community": "", "Current version: ": "" } \ No newline at end of file diff --git a/locales/nb_NO.json b/locales/nb_NO.json index 316a38ab..fd71dd45 100644 --- a/locales/nb_NO.json +++ b/locales/nb_NO.json @@ -316,5 +316,6 @@ "Video mode": "Video-modus", "Videos": "Videoer", "Playlists": "Spillelister", + "Community": "", "Current version: ": "Nåværende versjon: " } \ No newline at end of file diff --git a/locales/nl.json b/locales/nl.json index 19413a4f..71f3c643 100644 --- a/locales/nl.json +++ b/locales/nl.json @@ -316,5 +316,6 @@ "Video mode": "Videomodus", "Videos": "Video's", "Playlists": "Afspeellijsten", + "Community": "", "Current version: ": "Huidige versie: " } \ No newline at end of file diff --git a/locales/pl.json b/locales/pl.json index 4f95bdbe..d075b782 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -316,5 +316,6 @@ "Video mode": "Tryb wideo", "Videos": "Filmy", "Playlists": "Playlisty", + "Community": "", "Current version: ": "Aktualna wersja: " } \ No newline at end of file diff --git a/locales/ru.json b/locales/ru.json index a4f77c19..aeba27b1 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -316,5 +316,6 @@ "Video mode": "Видео режим", "Videos": "Видео", "Playlists": "Плейлисты", + "Community": "", "Current version: ": "Текущая версия: " } \ No newline at end of file diff --git a/locales/uk.json b/locales/uk.json index a260b694..9204d277 100644 --- a/locales/uk.json +++ b/locales/uk.json @@ -316,5 +316,6 @@ "Video mode": "Відеорежим", "Videos": "Відео", "Playlists": "Плейлисти", + "Community": "", "Current version: ": "Поточна версія: " } \ No newline at end of file diff --git a/locales/zh-CN.json b/locales/zh-CN.json index caa565f9..ca95431c 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -316,5 +316,6 @@ "Video mode": "视频模式", "Videos": "视频", "Playlists": "播放列表", + "Community": "", "Current version: ": "当前版本：" } \ No newline at end of file diff --git a/src/invidious.cr b/src/invidious.cr index 79968448..17f30e21 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2861,6 +2861,16 @@ get "/user/:user/videos" do |env| env.redirect "/channel/#{user}/videos" end +get "/user/:user/about" do |env| + user = env.params.url["user"] + env.redirect "/channel/#{user}" +end + +get "/channel:ucid/about" do |env| + ucid = env.params.url["ucid"] + env.redirect "/channel/#{ucid}" +end + get "/channel/:ucid" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -2968,6 +2978,46 @@ get "/channel/:ucid/playlists" do |env| templated "playlists" end +get "/channel/:ucid/community" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + if user + user = user.as(User) + subscriptions = user.subscriptions + end + subscriptions ||= [] of String + + ucid = env.params.url["ucid"] + + thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode + thin_mode = thin_mode == "true" + + continuation = env.params.query["continuation"]? + # sort_by = env.params.query["sort_by"]?.try &.downcase + + begin + channel = get_about_info(ucid, locale) + rescue ex + error_message = ex.message + env.response.status_code = 500 + next templated "error" + end + + if !channel.tabs.includes? "community" + next env.redirect "/channel/#{channel.ucid}" + end + + begin + items = JSON.parse(fetch_channel_community(ucid, continuation, locale, config, Kemal.config, "json", thin_mode)) + rescue ex + env.response.status_code = 500 + error_message = ex.message + end + + templated "community" +end + # API Endpoints get "/api/v1/stats" do |env| @@ -3757,12 +3807,17 @@ end ucid = env.params.url["ucid"] - continuation = env.params.query["continuation"]? + thin_mode = env.params.query["thin_mode"]? + thin_mode = thin_mode == "true" + format = env.params.query["format"]? + format ||= "json" + + continuation = env.params.query["continuation"]? # sort_by = env.params.query["sort_by"]?.try &.downcase begin - fetch_channel_community(ucid, continuation, locale, config, Kemal.config) + fetch_channel_community(ucid, continuation, locale, config, Kemal.config, format, thin_mode) rescue ex env.response.status_code = 400 error_message = {"error" => ex.message}.to_json diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index d7f68b11..b5f2eb04 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -123,6 +123,7 @@ struct AboutChannel is_family_friendly: Bool, allowed_regions: Array(String), related_channels: Array(AboutRelatedChannel), + tabs: Array(String), }) end @@ -617,7 +618,7 @@ def extract_channel_playlists_cursor(url, auto_generated) end # TODO: Add "sort_by" -def fetch_channel_community(ucid, continuation, locale, config, kemal_config) +def fetch_channel_community(ucid, continuation, locale, config, kemal_config, format, thin_mode) client = make_client(YT_URL) headers = HTTP::Headers.new headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" @@ -632,11 +633,10 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) raise error_message end + ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] + if !continuation || continuation.empty? response = JSON.parse(response.body.match(/window\["ytInitialData"\] = (?.*?);\n/).try &.["info"] || "{}") - ucid = response["responseContext"]["serviceTrackingParams"] - .as_a.select { |service| service["service"] == "GFEEDBACK" }[0]?.try &.["params"] - .as_a.select { |param| param["key"] == "browse_id" }[0]?.try &.["value"].as_s body = response["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? if !body @@ -645,6 +645,8 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] else + continuation = produce_channel_community_continuation(ucid, continuation) + headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] headers["content-type"] = "application/x-www-form-urlencoded" @@ -663,10 +665,6 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) response = client.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) body = JSON.parse(response.body) - ucid = body["response"]["responseContext"]["serviceTrackingParams"] - .as_a.select { |service| service["service"] == "GFEEDBACK" }[0]?.try &.["params"] - .as_a.select { |param| param["key"] == "browse_id" }[0]?.try &.["value"].as_s - body = body["response"]["continuationContents"]["itemSectionContinuation"]? || body["response"]["continuationContents"]["backstageCommentsContinuation"]? @@ -685,7 +683,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) raise error_message end - JSON.build do |json| + response = JSON.build do |json| json.object do json.field "authorId", ucid json.field "comments" do @@ -755,6 +753,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) json.field "likeCount", like_count json.field "commentId", post["postId"]? || post["commentId"]? || "" + json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid if attachment = post["backstageAttachment"]? json.field "attachment" do @@ -837,7 +836,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) json.field "replies" do json.object do json.field "replyCount", reply_count - json.field "continuation", continuation + json.field "continuation", extract_channel_community_cursor(continuation) end end end @@ -847,11 +846,71 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config) end if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"] - json.field "continuation", continuation + continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s + json.field "continuation", extract_channel_community_cursor(continuation) end end end + + if format == "html" + response = JSON.parse(response) + content_html = template_youtube_comments(response, locale, thin_mode) + + response = JSON.build do |json| + json.object do + json.field "contentHtml", content_html + end + end + end + + return response +end + +def produce_channel_community_continuation(ucid, cursor) + cursor = URI.escape(cursor) + continuation = IO::Memory.new + + continuation.write(Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02]) + continuation.write(write_var_int(3 + ucid.size + write_var_int(cursor.size).size + cursor.size)) + + continuation.write(Bytes[0x12, ucid.size]) + continuation.print(ucid) + + continuation.write(Bytes[0x1a]) + continuation.write(write_var_int(cursor.size)) + continuation.print(cursor) + continuation.rewind + + continuation = Base64.urlsafe_encode(continuation.to_slice) + continuation = URI.escape(continuation) + + return continuation +end + +def extract_channel_community_cursor(continuation) + continuation = URI.unescape(continuation) + continuation = Base64.decode(continuation) + + # 0xe2 0xa9 0x85 0xb2 0x02 + continuation += 5 + + total_size = read_var_int(continuation[0, 4]) + continuation += write_var_int(total_size).size + + # 0x12 + continuation += 1 + ucid_size = continuation[0] + continuation += 1 + ucid = continuation[0, ucid_size] + continuation += ucid_size + + # 0x1a + continuation += 1 + until continuation[0] == 'E'.ord + continuation += 1 + end + + return String.new(continuation) end def get_about_info(ucid, locale) @@ -947,6 +1006,8 @@ def get_about_info(ucid, locale) auto_generated = true end + tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase } + return AboutChannel.new( ucid: ucid, author: author, @@ -961,7 +1022,8 @@ def get_about_info(ucid, locale) joined: joined, is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, - related_channels: related_channels + related_channels: related_channels, + tabs: tabs ) end diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index e2de8714..cef09ff5 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -112,7 +112,7 @@ def fetch_youtube_comments(id, db, continuation, format, locale, thin_mode, regi end end - comments = JSON.build do |json| + response = JSON.build do |json| json.object do if body["header"]? count_text = body["header"]["commentsHeaderRenderer"]["countText"] @@ -223,15 +223,15 @@ def fetch_youtube_comments(id, db, continuation, format, locale, thin_mode, regi end if format == "html" - comments = JSON.parse(comments) - content_html = template_youtube_comments(comments, locale, thin_mode) + response = JSON.parse(response) + content_html = template_youtube_comments(response, locale, thin_mode) - comments = JSON.build do |json| + response = JSON.build do |json| json.object do json.field "contentHtml", content_html - if comments["commentCount"]? - json.field "commentCount", comments["commentCount"] + if response["commentCount"]? + json.field "commentCount", response["commentCount"] else json.field "commentCount", 0 end @@ -239,7 +239,7 @@ def fetch_youtube_comments(id, db, continuation, format, locale, thin_mode, regi end end - return comments + return response end def fetch_reddit_comments(id, sort_by = "confidence") @@ -286,7 +286,7 @@ def template_youtube_comments(comments, locale, thin_mode)

#{translate(locale, "View `x` replies", child["replies"]["replyCount"].to_s)} + onclick="get_youtube_replies(this)">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}

@@ -300,9 +300,9 @@ def template_youtube_comments(comments, locale, thin_mode) end html << <<-END_HTML -
+
- +

@@ -310,11 +310,66 @@ def template_youtube_comments(comments, locale, thin_mode) #{child["author"]}

#{child["contentHtml"]}

- #{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""} - | - [YT] - | - #{number_with_separator(child["likeCount"])} + END_HTML + + if child["attachment"]? + attachment = child["attachment"] + + case attachment["type"] + when "image" + attachment = attachment["imageThumbnails"][1] + + html << <<-END_HTML +
+
+ +
+
+ END_HTML + when "video" + html << <<-END_HTML +
+
+
+ END_HTML + + if attachment["error"]? + html << <<-END_HTML +

#{attachment["error"]}

+ END_HTML + else + html << <<-END_HTML + + END_HTML + end + + html << <<-END_HTML +
+
+
+ END_HTML + end + end + + html << <<-END_HTML + #{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""} + | + END_HTML + + if comments["videoId"]? + html << <<-END_HTML + [YT] + | + END_HTML + elsif comments["authorId"]? + html << <<-END_HTML + [YT] + | + END_HTML + end + + html << <<-END_HTML + #{number_with_separator(child["likeCount"])} END_HTML if child["creatorHeart"]? diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 88be697a..624dbe9c 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -49,6 +49,11 @@ <%= translate(locale, "Playlists") %> <% end %>
+
+ <% if channel.tabs.includes? "community" %> + <%= translate(locale, "Community") %> + <% end %> +
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr new file mode 100644 index 00000000..705d06a8 --- /dev/null +++ b/src/invidious/views/community.ecr @@ -0,0 +1,80 @@ +<% content_for "header" do %> +<%= channel.author %> - Invidious +<% end %> + +<% if channel.banner %> +
+ "> +
+ +
+
+
+<% end %> + +
+
+
+ + <%= channel.author %> +
+
+
+

+ +

+
+
+ +
+ <% ucid = channel.ucid %> + <% author = channel.author %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> + <%= rendered "components/subscribe_widget" %> +
+ +
+
+ <%= translate(locale, "View channel on YouTube") %> + <% if !channel.auto_generated %> +
+ <%= translate(locale, "Videos") %> +
+ <% end %> +
+ <%= translate(locale, "Playlists") %> +
+
+ <% if channel.tabs.includes? "community" %> + <%= translate(locale, "Community") %> + <% end %> +
+
+
+
+ +
+
+
+ +<% if error_message %> +
+

<%= error_message %>

+
+<% else %> +
+ <%= template_youtube_comments(items.not_nil!, locale, thin_mode) %> +
+<% end %> + + + diff --git a/src/invidious/views/licenses.ecr b/src/invidious/views/licenses.ecr index ec89c745..7cffb7fc 100644 --- a/src/invidious/views/licenses.ecr +++ b/src/invidious/views/licenses.ecr @@ -9,6 +9,20 @@

<%= translate(locale, "JavaScript license information") %>

+ + + + + + + +
+ community.js + + AGPL-3.0 + + <%= translate(locale, "source") %> +
embed.js diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index 8d1236aa..0a5a0c13 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -36,7 +36,7 @@
- <%= translate(locale, "View channel on YouTube") %> + <%= translate(locale, "View channel on YouTube") %>
<%= translate(locale, "Videos") %> @@ -46,6 +46,11 @@ <%= translate(locale, "Playlists") %> <% end %>
+
+ <% if channel.tabs.includes? "community" %> + <%= translate(locale, "Community") %> + <% end %> +