From b4674ffae425ebf2f7326cd55a9a250f32c0c182 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 3 Oct 2022 21:37:56 +0200 Subject: [PATCH] jobs: Allow each job to have its own config Each job can define its own config options, that will be automatically added to the global YAML config structure. By default, all jobs get an "enable" property that can prevent a specific job from running, e.g if the instance admin wants to manage the database cleaning by themselves. --- config/config.example.yml | 47 +++++++++++++++---- src/invidious/config.cr | 4 ++ src/invidious/jobs.cr | 27 +++++++++++ src/invidious/jobs/base_job.cr | 30 ++++++++++++ src/invidious/jobs/clear_expired_items_job.cr | 4 ++ 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 10734c3a..7db39844 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -304,10 +304,8 @@ https_only: false ## Number of threads to use when crawling channel videos (during ## subscriptions update). ## -## Notes: -## - Setting this to 0 will disable the channel videos crawl job. -## - This setting is overridden if "-c THREADS" or -## "--channel-threads=THREADS" are passed on the command line. +## Notes: This setting is overridden if either "-c THREADS" or +## "--channel-threads=THREADS" is passed on the command line. ## ## Accepted values: a positive integer ## Default: 1 @@ -335,10 +333,8 @@ full_refresh: false ## ## Number of threads to use when updating RSS feeds. ## -## Notes: -## - Setting this to 0 will disable the channel videos crawl job. -## - This setting is overridden if "-f THREADS" or -## "--feed-threads=THREADS" are passed on the command line. +## Notes: This setting is overridden if either "-f THREADS" or +## "--feed-threads=THREADS" is passed on the command line. ## ## Accepted values: a positive integer ## Default: 1 @@ -361,6 +357,39 @@ feed_threads: 1 #decrypt_polling: false +jobs: + + ## Options for the database cleaning job + clear_expired_items: + + ## Enable/Disable job + ## + ## Accepted values: true, false + ## Default: true + ## + enable: true + + ## Options for the channels updater job + refresh_channels: + + ## Enable/Disable job + ## + ## Accepted values: true, false + ## Default: true + ## + enable: true + + ## Options for the RSS feeds updater job + refresh_feeds: + + ## Enable/Disable job + ## + ## Accepted values: true, false + ## Default: true + ## + enable: true + + # ----------------------------- # Captcha API # ----------------------------- @@ -859,7 +888,7 @@ default_user_preferences: ## Default: false ## #automatic_instance_redirect: false - + ## ## Show the entire video description by default (when set to 'false', ## only the first few lines of the description are shown and a diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 786b65df..51b09fc7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -78,6 +78,10 @@ class Config property decrypt_polling : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false + + # Jobs config structure. See jobs.cr and jobs/base_job.cr + property jobs = Invidious::Jobs::JobsConfig.new + # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property https_only : Bool? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index ec0cad64..524a3624 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -1,12 +1,39 @@ module Invidious::Jobs JOBS = [] of BaseJob + # Automatically generate a structure that wraps the various + # jobs' configs, so that the follwing YAML config can be used: + # + # jobs: + # job_name: + # enabled: true + # some_property: "value" + # + macro finished + struct JobsConfig + include YAML::Serializable + + {% for sc in BaseJob.subclasses %} + # Voodoo macro to transform `Some::Module::CustomJob` to `custom` + {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %} + + getter {{ class_name }} = {{ sc.name }}::Config.new + {% end %} + + def initialize + end + end + end + def self.register(job : BaseJob) JOBS << job end def self.start_all JOBS.each do |job| + # Don't run the main rountine if the job is disabled by config + next if job.disabled? + spawn { job.begin } end end diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr index 47e75864..f90f0bfe 100644 --- a/src/invidious/jobs/base_job.cr +++ b/src/invidious/jobs/base_job.cr @@ -1,3 +1,33 @@ abstract class Invidious::Jobs::BaseJob abstract def begin + + # When this base job class is inherited, make sure to define + # a basic "Config" structure, that contains the "enable" property, + # and to create the associated instance property. + # + macro inherited + macro finished + # This config structure can be expanded as required. + struct Config + include YAML::Serializable + + property enable = true + + def initialize + end + end + + property cfg = Config.new + + # Return true if job is enabled by config + protected def enabled? : Bool + return (@cfg.enable == true) + end + + # Return true if job is disabled by config + protected def disabled? : Bool + return (@cfg.enable == false) + end + end + end end diff --git a/src/invidious/jobs/clear_expired_items_job.cr b/src/invidious/jobs/clear_expired_items_job.cr index 49f86106..17191aac 100644 --- a/src/invidious/jobs/clear_expired_items_job.cr +++ b/src/invidious/jobs/clear_expired_items_job.cr @@ -5,6 +5,8 @@ class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob loop do failed = false + LOGGER.info("jobs: running ClearExpiredItems job") + begin Invidious::Database::Videos.delete_expired Invidious::Database::Nonces.delete_expired @@ -14,8 +16,10 @@ class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob # Retry earlier than scheduled on DB error if failed + LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.") sleep 10.minutes else + LOGGER.info("jobs: ClearExpiredItems done.") sleep 1.hour end end