From 6cd884555c3ea3a108fe20d5b7ccbba9bd49bb4c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 23 Jun 2019 12:54:46 -0500 Subject: [PATCH] Patch StaticFileHandler to serve files from memory --- src/invidious.cr | 16 +- src/invidious/helpers/static_file_handler.cr | 195 +++++++++++++++++++ 2 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 src/invidious/helpers/static_file_handler.cr diff --git a/src/invidious.cr b/src/invidious.cr index e8647a6a..c22702e3 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -3466,9 +3466,9 @@ get "/api/v1/trending" do |env| json.array do trending.each do |video| video.to_json(locale, config, Kemal.config, json) - end - end end + end + end videos end @@ -3740,11 +3740,11 @@ end json.array do videos.each do |video| video.to_json(locale, config, Kemal.config, json) - end - end - end end end + end + end +end {"/api/v1/channels/:ucid/latest", "/api/v1/channels/latest/:ucid"}.each do |route| get route do |env| @@ -3766,11 +3766,11 @@ end json.array do videos.each do |video| video.to_json(locale, config, Kemal.config, json) - end - end end end end + end +end {"/api/v1/channels/:ucid/playlists", "/api/v1/channels/playlists/:ucid"}.each do |route| get route do |env| @@ -3951,11 +3951,11 @@ get "/api/v1/playlists/:plid" do |env| json.array do videos.each do |video| video.to_json(locale, config, Kemal.config, json) - end end end end end + end if format == "html" response = JSON.parse(response) diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr new file mode 100644 index 00000000..1b8f5395 --- /dev/null +++ b/src/invidious/helpers/static_file_handler.cr @@ -0,0 +1,195 @@ +# Since systems have a limit on number of open files (`ulimit -a`), +# we serve them from memory to avoid 'Too many open files' without needing +# to modify ulimit. +# +# Very heavily re-used: +# https://github.com/kemalcr/kemal/blob/master/src/kemal/helpers/helpers.cr +# https://github.com/kemalcr/kemal/blob/master/src/kemal/static_file_handler.cr +# +# Changes: +# - A `send_file` overload is added which supports sending a Slice, file_path, filestat +# - `StaticFileHandler` is patched to cache to and serve from @cached_files + +private def multipart(file, env : HTTP::Server::Context) + # See http://httpwg.org/specs/rfc7233.html + fileb = file.size + startb = endb = 0 + + if match = env.request.headers["Range"].match /bytes=(\d{1,})-(\d{0,})/ + startb = match[1].to_i { 0 } if match.size >= 2 + endb = match[2].to_i { 0 } if match.size >= 3 + end + + endb = fileb - 1 if endb == 0 + + if startb < endb < fileb + content_length = 1 + endb - startb + env.response.status_code = 206 + env.response.content_length = content_length + env.response.headers["Accept-Ranges"] = "bytes" + env.response.headers["Content-Range"] = "bytes #{startb}-#{endb}/#{fileb}" # MUST + + if startb > 1024 + skipped = 0 + # file.skip only accepts values less or equal to 1024 (buffer size, undocumented) + until (increase_skipped = skipped + 1024) > startb + file.skip(1024) + skipped = increase_skipped + end + if (skipped_minus_startb = skipped - startb) > 0 + file.skip skipped_minus_startb + end + else + file.skip(startb) + end + + IO.copy(file, env.response, content_length) + else + env.response.content_length = fileb + env.response.status_code = 200 # Range not satisfable, see 4.4 Note + IO.copy(file, env.response) + end +end + +# Set the Content-Disposition to "attachment" with the specified filename, +# instructing the user agents to prompt to save. +private def attachment(env : HTTP::Server::Context, filename : String? = nil, disposition : String? = nil) + disposition = "attachment" if disposition.nil? && filename + if disposition && filename + env.response.headers["Content-Disposition"] = "#{disposition}; filename=\"#{File.basename(filename)}\"" + end +end + +def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt8), filestat : File::Info, filename : String? = nil, disposition : String? = nil) + config = Kemal.config.serve_static + mime_type = MIME.from_filename(file_path, "application/octet-stream") + env.response.content_type = mime_type + env.response.headers["Accept-Ranges"] = "bytes" + env.response.headers["X-Content-Type-Options"] = "nosniff" + minsize = 860 # http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits ?? + request_headers = env.request.headers + filesize = data.bytesize + attachment(env, filename, disposition) + + Kemal.config.static_headers.try(&.call(env.response, file_path, filestat)) + + file = IO::Memory.new(data) + if env.request.method == "GET" && env.request.headers.has_key?("Range") + return multipart(file, env) + end + + condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path) + if condition && request_headers.includes_word?("Accept-Encoding", "gzip") + env.response.headers["Content-Encoding"] = "gzip" + Gzip::Writer.open(env.response) do |deflate| + IO.copy(file, deflate) + end + elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate") + env.response.headers["Content-Encoding"] = "deflate" + Flate::Writer.open(env.response) do |deflate| + IO.copy(file, deflate) + end + else + env.response.content_length = filesize + IO.copy(file, env.response) + end + + return +end + +module Kemal + class StaticFileHandler < HTTP::StaticFileHandler + CACHE_LIMIT = 5_000_000 # 5MB + @cached_files = {} of String => {data: Bytes, filestat: File::Info} + + def call(context : HTTP::Server::Context) + return call_next(context) if context.request.path.not_nil! == "/" + + case context.request.method + when "GET", "HEAD" + else + if @fallthrough + call_next(context) + else + context.response.status_code = 405 + context.response.headers.add("Allow", "GET, HEAD") + end + return + end + + config = Kemal.config.serve_static + original_path = context.request.path.not_nil! + request_path = URI.unescape(original_path) + + # File path cannot contains '\0' (NUL) because all filesystem I know + # don't accept '\0' character as file name. + if request_path.includes? '\0' + context.response.status_code = 400 + return + end + + expanded_path = File.expand_path(request_path, "/") + is_dir_path = if original_path.ends_with?('/') && !expanded_path.ends_with? '/' + expanded_path = expanded_path + '/' + true + else + expanded_path.ends_with? '/' + end + + file_path = File.join(@public_dir, expanded_path) + + if file = @cached_files[file_path]? + last_modified = file[:filestat].modification_time + add_cache_headers(context.response.headers, last_modified) + + if cache_request?(context, last_modified) + context.response.status_code = 304 + return + end + + puts "Sending cached file, #{@cached_files.sum { |element| element[1][:data].bytesize }}" + send_file(context, file_path, file[:data], file[:filestat]) + else + is_dir = Dir.exists? file_path + + if request_path != expanded_path + redirect_to context, expanded_path + elsif is_dir && !is_dir_path + redirect_to context, expanded_path + '/' + end + + if Dir.exists?(file_path) + if config.is_a?(Hash) && config["dir_listing"] == true + context.response.content_type = "text/html" + directory_listing(context.response, request_path, file_path) + else + call_next(context) + end + elsif File.exists?(file_path) + last_modified = modification_time(file_path) + add_cache_headers(context.response.headers, last_modified) + + if cache_request?(context, last_modified) + context.response.status_code = 304 + return + end + + if @cached_files.sum { |element| element[1][:data].bytesize } + (size = File.size(file_path)) < CACHE_LIMIT + data = Bytes.new(size) + File.open(file_path) do |file| + file.read(data) + end + filestat = File.info(file_path) + + @cached_files[file_path] = {data: data, filestat: filestat} + send_file(context, file_path, data, filestat) + else + send_file(context, file_path) + end + else + call_next(context) + end + end + end + end +end