diff --git a/shard.lock b/shard.lock index 50e64c64..f6cc22f4 100644 --- a/shard.lock +++ b/shard.lock @@ -35,6 +35,10 @@ shards: pg: git: https://github.com/will/crystal-pg.git version: 0.24.0 + + pool: + git: https://github.com/ysbaddaden/pool.git + version: 0.2.4 protodec: git: https://github.com/iv-org/protodec.git @@ -44,6 +48,10 @@ shards: git: https://github.com/luislavena/radix.git version: 0.4.1 + redis: + git: https://github.com/stefanwille/crystal-redis.git + version: 2.9.1 + spectator: git: https://github.com/icy-arctic-fox/spectator.git version: 0.10.6 diff --git a/shard.yml b/shard.yml index 14c2a84e..6b5cc9c1 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,8 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + redis: + github: stefanwille/crystal-redis http_proxy: github: mamantoha/http_proxy version: ~> 0.10.3 diff --git a/src/invidious.cr b/src/invidious.cr index 826d6768..f7155f44 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -32,6 +32,7 @@ require "xml" require "yaml" require "compress/zip" require "protodec/utils" +require "redis" require "./invidious/database/*" require "./invidious/database/migrations/*" @@ -61,7 +62,12 @@ alias IV = Invidious CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key -PG_DB = DB.open CONFIG.database_url +PG_DB = DB.open CONFIG.database_url +REDIS_DB = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url: CONFIG.redis_url || nil) + +if REDIS_DB.ping + puts "Connected to redis" +end ARCHIVE_URL = URI.parse("https://archive.org") PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") REDDIT_URL = URI.parse("https://www.reddit.com") diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c1766fbb..45efc3ff 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -87,6 +87,9 @@ class Config # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false + property redis_url : String? + property redis_socket : String? + # Jobs config structure. See jobs.cr and jobs/base_job.cr property jobs = Invidious::Jobs::JobsConfig.new diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr index 695f5b33..23958a5b 100644 --- a/src/invidious/database/videos.cr +++ b/src/invidious/database/videos.cr @@ -47,6 +47,15 @@ module Invidious::Database::Videos WHERE id = $1 SQL - return PG_DB.query_one?(request, id, as: Video) + if ((info = REDIS_DB.get(id)) && (time = REDIS_DB.get(id + ":time"))) + return Video.new({ + id: id, + info: JSON.parse(info).as_h, + updated: Time.parse(time, "%Y-%m-%d %H:%M:%S %z", Time::Location::UTC), + }) + else + return nil + end + end end diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7..6d198a42 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ae09e736..c6e69ee5 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -305,7 +305,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) video.schema_version != Video::SCHEMA_VERSION # cache control begin video = fetch_video(id, region) - Invidious::Database::Videos.update(video) + Invidious::Database::Videos.insert(video) rescue ex Invidious::Database::Videos.delete(id) raise ex