diff --git a/README.md b/README.md index 952b0e65..8b2f6ca5 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,9 @@ ## Features **Patches** + - revert d9df90b +- token updater patch (mooleshacat) **User features** - Lightweight diff --git a/config/config.example.yml b/config/config.example.yml index e9eebfde..759b81e0 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -173,6 +173,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/shard.lock b/shard.lock index 397bd8bc..50e64c64 100644 --- a/shard.lock +++ b/shard.lock @@ -10,7 +10,7 @@ shards: backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.3 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index 367f7c73..14c2a84e 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.3 development_dependencies: spectator: diff --git a/src/invidious.cr b/src/invidious.cr index 63f2a9cc..826d6768 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" @@ -189,6 +190,8 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +Invidious::Jobs.register Invidious::Jobs::MonitorCfgTokensJob.new() + Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new Invidious::Jobs.start_all diff --git a/src/invidious/config.cr b/src/invidious/config.cr index a097b7f1..c1766fbb 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -55,6 +55,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -129,6 +138,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index fec3f62c..3040d7a0 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.12.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/jobs/token_monitor.cr b/src/invidious/jobs/token_monitor.cr new file mode 100644 index 00000000..a26b8e61 --- /dev/null +++ b/src/invidious/jobs/token_monitor.cr @@ -0,0 +1,17 @@ + +class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob + include Invidious + def begin + loop do + + LOGGER.info("jobs: running MonitorCfgTokensJob job") + + ReloadTokens.get_tokens + + LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadTokens.pot.as(String)) + LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadTokens.vdata.as(String)) + + sleep 15.seconds + end + end +end diff --git a/src/invidious/reloadtoken.cr b/src/invidious/reloadtoken.cr new file mode 100644 index 00000000..f9428184 --- /dev/null +++ b/src/invidious/reloadtoken.cr @@ -0,0 +1,83 @@ +class ReloadTokens + + @@instance = new + + def self.pot + @@pot + end + + def self.vdata + @@vdata + end + + def initialize + + @@pot = "error" + @@vdata = "error" + + end + + def self.get_tokens + + # Load config from file or YAML string env var + env_config_file = "INVIDIOUS_CONFIG_FILE" + env_config_yaml = "INVIDIOUS_CONFIG" + + config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" + config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) + + config = Config.from_yaml(config_yaml) + + + + + # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + {% for ivar in Config.instance_vars %} + {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} + + if ENV.has_key?({{env_id}}) + env_value = ENV.fetch({{env_id}}) + success = false + + # Use YAML converter if specified + {% ann = ivar.annotation(::YAML::Field) %} + {% if ann && ann[:converter] %} + config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) + success = true + + # Use regular YAML parser otherwise + {% else %} + {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} + # Sort types to avoid parsing nulls and numbers as strings + {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} + {{ivar_types}}.each do |ivar_type| + if !success + begin + config.{{ivar.id}} = ivar_type.from_yaml(env_value) + success = true + rescue + # nop + end + end + end + {% end %} + + # Exit on fail + if !success + puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) + exit(1) + end + end + {% end %} + + + @@pot = config.po_token + @@vdata = config.visitor_data + + end + + def self.get_instance + return @@instance + end + +end \ No newline at end of file diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..70b15f26 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,16 @@ struct YoutubeConnectionPool def client(&) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -77,3 +81,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index d66bf7aa..1b640bc2 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -320,8 +320,8 @@ module YoutubeAPI client_context["client"]["platform"] = platform end - if CONFIG.visitor_data.is_a?(String) - client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String) + if ReloadTokens.vdata.is_a?(String) + client_context["client"]["visitorData"] = ReloadTokens.vdata.as(String) end return client_context @@ -482,7 +482,7 @@ module YoutubeAPI "contentPlaybackContext" => playback_ctx, }, "serviceIntegrityDimensions" => { - "poToken" => CONFIG.po_token, + "poToken" => ReloadTokens.pot.as(String), }, } @@ -616,8 +616,8 @@ module YoutubeAPI headers["User-Agent"] = user_agent end - if CONFIG.visitor_data.is_a?(String) - headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String) + if ReloadTokens.vdata.is_a?(String) + headers["X-Goog-Visitor-Id"] = ReloadTokens.vdata.as(String) end # Logging