-
-
- END_HTML
- end
-
- if !thin_mode
- author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).full_path}"
- else
- author_thumbnail = ""
- end
-
- html += <<-END_HTML
-
-
-

-
-
-
-
- #{child["author"]}
-
-
#{child["contentHtml"]}
-
#{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}
- |
-
[YT]
- |
-
#{number_with_separator(child["likeCount"])}
- END_HTML
-
- if child["creatorHeart"]?
- if !thin_mode
- creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).full_path}"
- else
- creator_thumbnail = ""
+ END_HTML
end
- html += <<-END_HTML
+ if !thin_mode
+ author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).full_path}"
+ else
+ author_thumbnail = ""
+ end
+
+ html << <<-END_HTML
+
+
+

+
+
+
+
+ #{child["author"]}
+
+
#{child["contentHtml"]}
+ END_HTML
+
+ if child["attachment"]?
+ attachment = child["attachment"]
+
+ case attachment["type"]
+ when "image"
+ attachment = attachment["imageThumbnails"][1]
+
+ html << <<-END_HTML
+
+
+

+
+
+ END_HTML
+ when "video"
+ html << <<-END_HTML
+
+
+
+ END_HTML
+
+ if attachment["error"]?
+ html << <<-END_HTML
+
#{attachment["error"]}
+ END_HTML
+ else
+ html << <<-END_HTML
+
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+
+
+ END_HTML
+ end
+ end
+
+ html << <<-END_HTML
+
#{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}
+ |
+ END_HTML
+
+ if comments["videoId"]?
+ html << <<-END_HTML
+
[YT]
+ |
+ END_HTML
+ elsif comments["authorId"]?
+ html << <<-END_HTML
+
[YT]
+ |
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
#{number_with_separator(child["likeCount"])}
+ END_HTML
+
+ if child["creatorHeart"]?
+ if !thin_mode
+ creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).full_path}"
+ else
+ creator_thumbnail = ""
+ end
+
+ html << <<-END_HTML

@@ -340,84 +388,77 @@ def template_youtube_comments(comments, locale, thin_mode)
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+ #{replies_html}
+
+
END_HTML
end
- html += <<-END_HTML
-
- #{replies_html}
+ if comments["continuation"]?
+ html << <<-END_HTML
+
-
- END_HTML
+ END_HTML
+ end
end
-
- if comments["continuation"]?
- html += <<-END_HTML
-
- END_HTML
- end
-
- return html
end
def template_reddit_comments(root, locale)
- html = ""
- root.each do |child|
- if child.data.is_a?(RedditComment)
- child = child.data.as(RedditComment)
- author = child.author
- score = child.score
- body_html = HTML.unescape(child.body_html)
+ String.build do |html|
+ root.each do |child|
+ if child.data.is_a?(RedditComment)
+ child = child.data.as(RedditComment)
+ body_html = HTML.unescape(child.body_html)
- replies_html = ""
- if child.replies.is_a?(RedditThing)
- replies = child.replies.as(RedditThing)
- replies_html = template_reddit_comments(replies.data.as(RedditListing).children, locale)
- end
+ replies_html = ""
+ if child.replies.is_a?(RedditThing)
+ replies = child.replies.as(RedditThing)
+ replies_html = template_reddit_comments(replies.data.as(RedditListing).children, locale)
+ end
- content = <<-END_HTML
-
- [ - ]
- #{author}
- #{translate(locale, "`x` points", number_with_separator(score))}
- #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}
-
-
- #{body_html}
- #{replies_html}
-
- END_HTML
-
- if child.depth > 0
- html += <<-END_HTML
+ if child.depth > 0
+ html << <<-END_HTML
- END_HTML
- else
- html += <<-END_HTML
+ END_HTML
+ else
+ html << <<-END_HTML
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+ [ - ]
+ #{child.author}
+ #{translate(locale, "`x` points", number_with_separator(child.score))}
+ #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}
+ #{translate(locale, "permalink")}
+
+
+ #{body_html}
+ #{replies_html}
+
+
+
END_HTML
end
end
end
-
- return html
end
def replace_links(html)
@@ -517,114 +558,111 @@ def content_to_comment_html(content)
end
text
- end.join.rchop('\ufeff')
+ end.join("").delete('\ufeff')
return comment_html
end
def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
- continuation = IO::Memory.new
+ data = IO::Memory.new
- continuation.write(Bytes[0x12, 0x26])
+ data.write Bytes[0x12, 0x26]
- continuation.write(Bytes[0x12, video_id.size])
- continuation.print(video_id)
+ data.write_byte 0x12
+ VarInt.to_io(data, video_id.bytesize)
+ data.print video_id
- continuation.write(Bytes[0xc0, 0x01, 0x01])
- continuation.write(Bytes[0xc8, 0x01, 0x01])
- continuation.write(Bytes[0xe0, 0x01, 0x01])
+ data.write Bytes[0xc0, 0x01, 0x01]
+ data.write Bytes[0xc8, 0x01, 0x01]
+ data.write Bytes[0xe0, 0x01, 0x01]
- continuation.write(Bytes[0xa2, 0x02, 0x0d])
- continuation.write(Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])
+ data.write Bytes[0xa2, 0x02, 0x0d]
+ data.write Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]
- continuation.write(Bytes[0x40, 0x00])
- continuation.write(Bytes[0x18, 0x06])
+ data.write Bytes[0x40, 0x00]
+ data.write Bytes[0x18, 0x06]
if cursor.empty?
- continuation.write(Bytes[0x32])
- continuation.write(write_var_int(video_id.size + 8))
+ data.write Bytes[0x32]
+ VarInt.to_io(data, cursor.bytesize + video_id.bytesize + 8)
- continuation.write(Bytes[0x22, video_id.size + 4])
- continuation.write(Bytes[0x22, video_id.size])
- continuation.print(video_id)
+ data.write Bytes[0x22, video_id.bytesize + 4]
+ data.write Bytes[0x22, video_id.bytesize]
+ data.print video_id
case sort_by
when "top"
- continuation.write(Bytes[0x30, 0x00])
+ data.write Bytes[0x30, 0x00]
when "new", "newest"
- continuation.write(Bytes[0x30, 0x01])
+ data.write Bytes[0x30, 0x01]
end
- continuation.write(Bytes[0x78, 0x02])
+ data.write(Bytes[0x78, 0x02])
else
- continuation.write(Bytes[0x32])
- continuation.write(write_var_int(cursor.size + video_id.size + 11))
+ data.write Bytes[0x32]
+ VarInt.to_io(data, cursor.bytesize + video_id.bytesize + 11)
- continuation.write(Bytes[0x0a])
- continuation.write(write_var_int(cursor.size))
- continuation.print(cursor)
+ data.write_byte 0x0a
+ VarInt.to_io(data, cursor.bytesize)
+ data.print cursor
- continuation.write(Bytes[0x22, video_id.size + 4])
- continuation.write(Bytes[0x22, video_id.size])
- continuation.print(video_id)
+ data.write Bytes[0x22, video_id.bytesize + 4]
+ data.write Bytes[0x22, video_id.bytesize]
+ data.print video_id
case sort_by
when "top"
- continuation.write(Bytes[0x30, 0x00])
+ data.write Bytes[0x30, 0x00]
when "new", "newest"
- continuation.write(Bytes[0x30, 0x01])
+ data.write Bytes[0x30, 0x01]
end
- continuation.write(Bytes[0x28, 0x14])
+ data.write Bytes[0x28, 0x14]
end
- continuation.rewind
- continuation = continuation.gets_to_end
-
- continuation = Base64.urlsafe_encode(continuation.to_slice)
+ continuation = Base64.urlsafe_encode(data)
continuation = URI.escape(continuation)
return continuation
end
def produce_comment_reply_continuation(video_id, ucid, comment_id)
- continuation = IO::Memory.new
+ data = IO::Memory.new
- continuation.write(Bytes[0x12, 0x26])
+ data.write Bytes[0x12, 0x26]
- continuation.write(Bytes[0x12, video_id.size])
- continuation.print(video_id)
+ data.write_byte 0x12
+ VarInt.to_io(data, video_id.size)
+ data.print video_id
- continuation.write(Bytes[0xc0, 0x01, 0x01])
- continuation.write(Bytes[0xc8, 0x01, 0x01])
- continuation.write(Bytes[0xe0, 0x01, 0x01])
+ data.write Bytes[0xc0, 0x01, 0x01]
+ data.write Bytes[0xc8, 0x01, 0x01]
+ data.write Bytes[0xe0, 0x01, 0x01]
- continuation.write(Bytes[0xa2, 0x02, 0x0d])
- continuation.write(Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])
+ data.write Bytes[0xa2, 0x02, 0x0d]
+ data.write Bytes[0x28, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]
- continuation.write(Bytes[0x40, 0x00])
- continuation.write(Bytes[0x18, 0x06])
+ data.write Bytes[0x40, 0x00]
+ data.write Bytes[0x18, 0x06]
- continuation.write(Bytes[0x32, ucid.size + video_id.size + comment_id.size + 16])
- continuation.write(Bytes[0x1a, ucid.size + video_id.size + comment_id.size + 14])
+ data.write(Bytes[0x32, ucid.size + video_id.size + comment_id.size + 16])
+ data.write(Bytes[0x1a, ucid.size + video_id.size + comment_id.size + 14])
- continuation.write(Bytes[0x12, comment_id.size])
- continuation.print(comment_id)
+ data.write_byte 0x12
+ VarInt.to_io(data, comment_id.size)
+ data.print comment_id
- continuation.write(Bytes[0x22, 0x02, 0x08, 0x00]) # ??
+ data.write(Bytes[0x22, 0x02, 0x08, 0x00]) # ??
- continuation.write(Bytes[ucid.size + video_id.size + 7])
- continuation.write(Bytes[ucid.size])
- continuation.print(ucid)
- continuation.write(Bytes[0x32, video_id.size])
- continuation.print(video_id)
- continuation.write(Bytes[0x40, 0x01])
- continuation.write(Bytes[0x48, 0x0a])
+ data.write(Bytes[ucid.size + video_id.size + 7])
+ data.write(Bytes[ucid.size])
+ data.print(ucid)
+ data.write(Bytes[0x32, video_id.size])
+ data.print(video_id)
+ data.write(Bytes[0x40, 0x01])
+ data.write(Bytes[0x48, 0x0a])
- continuation.rewind
- continuation = continuation.gets_to_end
-
- continuation = Base64.urlsafe_encode(continuation.to_slice)
+ continuation = Base64.urlsafe_encode(data.to_slice)
continuation = URI.escape(continuation)
return continuation
diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr
index 63fa690ee..119c7d3b0 100644
--- a/src/invidious/helpers/handlers.cr
+++ b/src/invidious/helpers/handlers.cr
@@ -176,3 +176,41 @@ class HTTP::Client
response
end
end
+
+# https://github.com/will/crystal-pg/pull/171
+class PG::Statement < ::DB::Statement
+ protected def perform_query(args : Enumerable) : ResultSet
+ params = args.map { |arg| PQ::Param.encode(arg) }
+ conn = self.conn
+ conn.send_parse_message(@sql)
+ conn.send_bind_message params
+ conn.send_describe_portal_message
+ conn.send_execute_message
+ conn.send_sync_message
+ conn.expect_frame PQ::Frame::ParseComplete
+ conn.expect_frame PQ::Frame::BindComplete
+ frame = conn.read
+ case frame
+ when PQ::Frame::RowDescription
+ fields = frame.fields
+ when PQ::Frame::NoData
+ fields = nil
+ else
+ raise "expected RowDescription or NoData, got #{frame}"
+ end
+ ResultSet.new(self, fields)
+ rescue IO::Error
+ raise DB::ConnectionLost.new(connection)
+ end
+
+ protected def perform_exec(args : Enumerable) : ::DB::ExecResult
+ result = perform_query(args)
+ result.each { }
+ ::DB::ExecResult.new(
+ rows_affected: result.rows_affected,
+ last_insert_id: 0_i64 # postgres doesn't support this
+ )
+ rescue IO::Error
+ raise DB::ConnectionLost.new(connection)
+ end
+end
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 3155cb676..9cefcf144 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -87,12 +87,53 @@ end
struct Config
module ConfigPreferencesConverter
+ def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
+ value.to_yaml(yaml)
+ end
+
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences
Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple)
end
+ end
- def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder)
- value.to_yaml(yaml)
+ module FamilyConverter
+ def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
+ case value
+ when Socket::Family::UNSPEC
+ yaml.scalar nil
+ when Socket::Family::INET
+ yaml.scalar "ipv4"
+ when Socket::Family::INET6
+ yaml.scalar "ipv6"
+ end
+ end
+
+ def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
+ if node.is_a?(YAML::Nodes::Scalar)
+ case node.value.downcase
+ when "ipv4"
+ Socket::Family::INET
+ when "ipv6"
+ Socket::Family::INET6
+ else
+ Socket::Family::UNSPEC
+ end
+ else
+ node.raise "Expected scalar, not #{node.class}"
+ end
+ end
+ end
+
+ def disabled?(option)
+ case disabled = CONFIG.disable_proxy
+ when Bool
+ return disabled
+ when Array
+ if disabled.includes? option
+ return true
+ else
+ return false
+ end
end
end
@@ -105,7 +146,6 @@ struct Config
hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
- use_feed_events: {type: Bool | Int32, default: false}, # Update feeds on receiving notifications
default_home: {type: String, default: "Top"},
feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending", "Subscriptions"]},
top_enabled: {type: Bool, default: true},
@@ -119,11 +159,13 @@ struct Config
default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple),
converter: ConfigPreferencesConverter,
},
- dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs
- check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc.
- cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
- banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc.
- hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
+ dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs
+ check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc.
+ cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
+ banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc.
+ hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
+ disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
+ force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
})
end
@@ -147,7 +189,7 @@ def rank_videos(db, n)
published = rs.read(Time)
# Exponential decay, older videos tend to rank lower
- temperature = wilson_score * Math.exp(-0.000005*((Time.now - published).total_minutes))
+ temperature = wilson_score * Math.exp(-0.000005*((Time.utc - published).total_minutes))
top << {temperature, id}
end
end
@@ -161,40 +203,42 @@ def rank_videos(db, n)
return top[0..n - 1]
end
-def login_req(login_form, f_req)
+def login_req(f_req)
data = {
+ # Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard
+ # Generally this is much longer (>1250 characters), see also
+ # https://github.com/ytdl-org/youtube-dl/commit/baf67a604d912722b0fe03a40e9dc5349a2208cb .
+ # For now this can be empty.
+ "bgRequest" => %|["identifier",""]|,
"pstMsg" => "1",
"checkConnection" => "youtube",
"checkedDomains" => "youtube",
"hl" => "en",
- "deviceinfo" => %q([null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]),
+ "deviceinfo" => %|[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]|,
"f.req" => f_req,
"flowName" => "GlifWebSignIn",
"flowEntry" => "ServiceLogin",
+ # "cookiesDisabled" => "false",
+ # "gmscoreversion" => "undefined",
+ # "continue" => "https://accounts.google.com/ManageAccount",
+ # "azt" => "",
+ # "bgHash" => "",
}
- data = login_form.merge(data)
-
return HTTP::Params.encode(data)
end
-def html_to_content(description_html)
- if !description_html
- description = ""
- description_html = ""
- else
- description_html = description_html.to_s
- description = description_html.gsub("