jobs: Allow each job to have its own config

Each job can define its own config options, that will be automatically
added to the global YAML config structure.

By default, all jobs get an "enable" property that can prevent a specific
job from running, e.g if the instance admin wants to manage the database
cleaning by themselves.

config/config.example.yml

@@ -304,10 +304,8 @@ https_only: false
 ## Number of threads to use when crawling channel videos (during
 ## subscriptions update).
 ##
-## Notes:
-##  - Setting this to 0 will disable the channel videos crawl job.
-##  - This setting is overridden if "-c THREADS" or
-##    "--channel-threads=THREADS" are passed on the command line.
+## Notes: This setting is overridden if either "-c THREADS" or
+## "--channel-threads=THREADS" is passed on the command line.
 ##
 ## Accepted values: a positive integer
 ## Default: 1
@@ -335,10 +333,8 @@ full_refresh: false
 ##
 ## Number of threads to use when updating RSS feeds.
 ##
-## Notes:
-##  - Setting this to 0 will disable the channel videos crawl job.
-##  - This setting is overridden if "-f THREADS" or
-##    "--feed-threads=THREADS" are passed on the command line.
+## Notes: This setting is overridden if either "-f THREADS" or
+## "--feed-threads=THREADS" is passed on the command line.
 ##
 ## Accepted values: a positive integer
 ## Default: 1
@@ -361,6 +357,39 @@ feed_threads: 1
 #decrypt_polling: false
 
 
+jobs:
+
+  ## Options for the database cleaning job
+  clear_expired_items:
+
+    ## Enable/Disable job
+    ##
+    ## Accepted values: true, false
+    ## Default: true
+    ##
+    enable: true
+
+  ## Options for the channels updater job
+  refresh_channels:
+
+    ## Enable/Disable job
+    ##
+    ## Accepted values: true, false
+    ## Default: true
+    ##
+    enable: true
+
+  ## Options for the RSS feeds updater job
+  refresh_feeds:
+
+    ## Enable/Disable job
+    ##
+    ## Accepted values: true, false
+    ## Default: true
+    ##
+    enable: true
+
+
 # -----------------------------
 #  Captcha API
 # -----------------------------
@@ -859,7 +888,7 @@ default_user_preferences:
   ## Default: false
   ##
   #automatic_instance_redirect: false
-  
+
   ##
   ## Show the entire video description by default (when set to 'false',
   ## only the first few lines of the description are shown and a

src/invidious/config.cr

@@ -78,6 +78,10 @@ class Config
   property decrypt_polling : Bool = false
   # Used for crawling channels: threads should check all videos uploaded by a channel
   property full_refresh : Bool = false
+
+  # Jobs config structure. See jobs.cr and jobs/base_job.cr
+  property jobs = Invidious::Jobs::JobsConfig.new
+
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
   # HMAC signing key for CSRF tokens and verifying pubsub subscriptions

src/invidious/jobs.cr

@@ -1,12 +1,39 @@
 module Invidious::Jobs
   JOBS = [] of BaseJob
 
+  # Automatically generate a structure that wraps the various
+  # jobs' configs, so that the follwing YAML config can be used:
+  #
+  # jobs:
+  #   job_name:
+  #     enabled: true
+  #     some_property: "value"
+  #
+  macro finished
+    struct JobsConfig
+      include YAML::Serializable
+
+      {% for sc in BaseJob.subclasses %}
+        # Voodoo macro to transform `Some::Module::CustomJob` to `custom`
+        {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %}
+
+        getter {{ class_name }} = {{ sc.name }}::Config.new
+      {% end %}
+
+      def initialize
+      end
+    end
+  end
+
   def self.register(job : BaseJob)
     JOBS << job
   end
 
   def self.start_all
     JOBS.each do |job|
+      # Don't run the main rountine if the job is disabled by config
+      next if job.disabled?
+
       spawn { job.begin }
     end
   end

src/invidious/jobs/base_job.cr

@@ -1,3 +1,33 @@
 abstract class Invidious::Jobs::BaseJob
   abstract def begin
+
+  # When this base job class is inherited, make sure to define
+  # a basic "Config" structure, that contains the "enable" property,
+  # and to create the associated instance property.
+  #
+  macro inherited
+    macro finished
+      # This config structure can be expanded as required.
+      struct Config
+        include YAML::Serializable
+
+        property enable = true
+
+        def initialize
+        end
+      end
+
+      property cfg = Config.new
+
+      # Return true if job is enabled by config
+      protected def enabled? : Bool
+        return (@cfg.enable == true)
+      end
+
+      # Return true if job is disabled by config
+      protected def disabled? : Bool
+        return (@cfg.enable == false)
+      end
+    end
+  end
 end

src/invidious/jobs/clear_expired_items_job.cr

@@ -5,6 +5,8 @@ class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob
     loop do
       failed = false
 
+      LOGGER.info("jobs: running ClearExpiredItems job")
+
       begin
         Invidious::Database::Videos.delete_expired
         Invidious::Database::Nonces.delete_expired
@@ -14,8 +16,10 @@ class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob
 
       # Retry earlier than scheduled on DB error
       if failed
+        LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.")
         sleep 10.minutes
       else
+        LOGGER.info("jobs: ClearExpiredItems done.")
         sleep 1.hour
       end
     end