diff --git a/config/config.example.yml b/config/config.example.yml index 10734c3a..7db39844 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -304,10 +304,8 @@ https_only: false ## Number of threads to use when crawling channel videos (during ## subscriptions update). ## -## Notes: -## - Setting this to 0 will disable the channel videos crawl job. -## - This setting is overridden if "-c THREADS" or -## "--channel-threads=THREADS" are passed on the command line. +## Notes: This setting is overridden if either "-c THREADS" or +## "--channel-threads=THREADS" is passed on the command line. ## ## Accepted values: a positive integer ## Default: 1 @@ -335,10 +333,8 @@ full_refresh: false ## ## Number of threads to use when updating RSS feeds. ## -## Notes: -## - Setting this to 0 will disable the channel videos crawl job. -## - This setting is overridden if "-f THREADS" or -## "--feed-threads=THREADS" are passed on the command line. +## Notes: This setting is overridden if either "-f THREADS" or +## "--feed-threads=THREADS" is passed on the command line. ## ## Accepted values: a positive integer ## Default: 1 @@ -361,6 +357,39 @@ feed_threads: 1 #decrypt_polling: false +jobs: + + ## Options for the database cleaning job + clear_expired_items: + + ## Enable/Disable job + ## + ## Accepted values: true, false + ## Default: true + ## + enable: true + + ## Options for the channels updater job + refresh_channels: + + ## Enable/Disable job + ## + ## Accepted values: true, false + ## Default: true + ## + enable: true + + ## Options for the RSS feeds updater job + refresh_feeds: + + ## Enable/Disable job + ## + ## Accepted values: true, false + ## Default: true + ## + enable: true + + # ----------------------------- # Captcha API # ----------------------------- @@ -859,7 +888,7 @@ default_user_preferences: ## Default: false ## #automatic_instance_redirect: false - + ## ## Show the entire video description by default (when set to 'false', ## only the first few lines of the description are shown and a diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 786b65df..51b09fc7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -78,6 +78,10 @@ class Config property decrypt_polling : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false + + # Jobs config structure. See jobs.cr and jobs/base_job.cr + property jobs = Invidious::Jobs::JobsConfig.new + # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property https_only : Bool? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index ec0cad64..524a3624 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -1,12 +1,39 @@ module Invidious::Jobs JOBS = [] of BaseJob + # Automatically generate a structure that wraps the various + # jobs' configs, so that the follwing YAML config can be used: + # + # jobs: + # job_name: + # enabled: true + # some_property: "value" + # + macro finished + struct JobsConfig + include YAML::Serializable + + {% for sc in BaseJob.subclasses %} + # Voodoo macro to transform `Some::Module::CustomJob` to `custom` + {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %} + + getter {{ class_name }} = {{ sc.name }}::Config.new + {% end %} + + def initialize + end + end + end + def self.register(job : BaseJob) JOBS << job end def self.start_all JOBS.each do |job| + # Don't run the main rountine if the job is disabled by config + next if job.disabled? + spawn { job.begin } end end diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr index 47e75864..f90f0bfe 100644 --- a/src/invidious/jobs/base_job.cr +++ b/src/invidious/jobs/base_job.cr @@ -1,3 +1,33 @@ abstract class Invidious::Jobs::BaseJob abstract def begin + + # When this base job class is inherited, make sure to define + # a basic "Config" structure, that contains the "enable" property, + # and to create the associated instance property. + # + macro inherited + macro finished + # This config structure can be expanded as required. + struct Config + include YAML::Serializable + + property enable = true + + def initialize + end + end + + property cfg = Config.new + + # Return true if job is enabled by config + protected def enabled? : Bool + return (@cfg.enable == true) + end + + # Return true if job is disabled by config + protected def disabled? : Bool + return (@cfg.enable == false) + end + end + end end diff --git a/src/invidious/jobs/clear_expired_items_job.cr b/src/invidious/jobs/clear_expired_items_job.cr index 49f86106..17191aac 100644 --- a/src/invidious/jobs/clear_expired_items_job.cr +++ b/src/invidious/jobs/clear_expired_items_job.cr @@ -5,6 +5,8 @@ class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob loop do failed = false + LOGGER.info("jobs: running ClearExpiredItems job") + begin Invidious::Database::Videos.delete_expired Invidious::Database::Nonces.delete_expired @@ -14,8 +16,10 @@ class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob # Retry earlier than scheduled on DB error if failed + LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.") sleep 10.minutes else + LOGGER.info("jobs: ClearExpiredItems done.") sleep 1.hour end end