Merge pull request #27 from catspeed-cc/new-dev-token-updater

Merge: new-dev-token-updater to new-development

shard.lock

@@ -35,6 +35,10 @@ shards:
   pg:
     git: https://github.com/will/crystal-pg.git
     version: 0.24.0
+    
+  pool:
+    git: https://github.com/ysbaddaden/pool.git
+    version: 0.2.4
 
   protodec:
     git: https://github.com/iv-org/protodec.git
@@ -44,6 +48,10 @@ shards:
     git: https://github.com/luislavena/radix.git
     version: 0.4.1
 
+  redis:
+    git: https://github.com/stefanwille/crystal-redis.git
+    version: 2.9.1
+
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
     version: 0.10.6

shard.yml

@@ -28,6 +28,8 @@ dependencies:
   athena-negotiation:
     github: athena-framework/negotiation
     version: ~> 0.1.1
+  redis:
+    github: stefanwille/crystal-redis
   http_proxy:
     github: mamantoha/http_proxy
     version: ~> 0.10.3

src/invidious.cr

@@ -32,6 +32,7 @@ require "xml"
 require "yaml"
 require "compress/zip"
 require "protodec/utils"
+require "redis"
 
 require "./invidious/database/*"
 require "./invidious/database/migrations/*"
@@ -61,7 +62,12 @@ alias IV = Invidious
 CONFIG   = Config.load
 HMAC_KEY = CONFIG.hmac_key
 
-PG_DB       = DB.open CONFIG.database_url
+PG_DB    = DB.open CONFIG.database_url
+REDIS_DB = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url: CONFIG.redis_url || nil)
+
+if REDIS_DB.ping
+  puts "Connected to redis"
+end
 ARCHIVE_URL = URI.parse("https://archive.org")
 PUBSUB_URL  = URI.parse("https://pubsubhubbub.appspot.com")
 REDDIT_URL  = URI.parse("https://www.reddit.com")

src/invidious/config.cr

@@ -87,6 +87,9 @@ class Config
   # Used for crawling channels: threads should check all videos uploaded by a channel
   property full_refresh : Bool = false
 
+  property redis_url : String?
+  property redis_socket : String?
+
   # Jobs config structure. See jobs.cr and jobs/base_job.cr
   property jobs = Invidious::Jobs::JobsConfig.new
 

src/invidious/database/videos.cr

@@ -47,6 +47,15 @@ module Invidious::Database::Videos
       WHERE id = $1
     SQL
 
-    return PG_DB.query_one?(request, id, as: Video)
+    if ((info = REDIS_DB.get(id)) && (time = REDIS_DB.get(id + ":time")))
+      return Video.new({
+        id:      id,
+        info:    JSON.parse(info).as_h,
+        updated: Time.parse(time, "%Y-%m-%d %H:%M:%S %z", Time::Location::UTC),
+      })
+    else
+      return nil
+    end
+    
   end
 end

src/invidious/helpers/sig_helper.cr

@@ -175,8 +175,9 @@ module Invidious::SigHelper
     @queue = {} of TransactionID => Transaction
 
     @conn : Connection
+    @uri_or_path : String
 
-    def initialize(uri_or_path)
+    def initialize(@uri_or_path)
       @conn = Connection.new(uri_or_path)
       listen
     end
@@ -186,10 +187,26 @@ module Invidious::SigHelper
 
       LOGGER.debug("SigHelper: Multiplexor listening")
 
-      # TODO: reopen socket if unexpectedly closed
       spawn do
         loop do
-          receive_data
+          begin
+            receive_data
+          rescue ex
+            LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")
+            # We close the socket because for some reason is not closed.
+            @conn.close
+            loop do
+              begin
+                @conn = Connection.new(@uri_or_path)
+                LOGGER.info("SigHelper: Reconnected to SigHelper!")
+              rescue ex
+                LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
+                sleep 500.milliseconds
+                next
+              end
+              break if !@conn.closed?
+            end
+          end
           Fiber.yield
         end
       end

src/invidious/videos.cr

@@ -305,7 +305,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
        video.schema_version != Video::SCHEMA_VERSION # cache control
       begin
         video = fetch_video(id, region)
-        Invidious::Database::Videos.update(video)
+        Invidious::Database::Videos.insert(video)
       rescue ex
         Invidious::Database::Videos.delete(id)
         raise ex