From 61d9491ee60e356f7d826bd7e70e18cf6ba1472c Mon Sep 17 00:00:00 2001
From: mooleshacat <43627985+mooleshacat@users.noreply.github.com>
Date: Tue, 15 Oct 2024 17:52:25 -0400
Subject: [PATCH] few fixes
---
src/invidious.cr | 4 +-
src/invidious.cr~ | 248 +++++++
src/invidious/jobs/token_monitor.cr | 8 +-
src/invidious/jobs/token_monitor.cr~ | 17 +
.../{reloadpotoken.cr => reloadpotoken.cr~} | 0
src/invidious/reloadtoken.cr | 83 +++
src/invidious/yt_backend/youtube_api.cr | 10 +-
src/invidious/yt_backend/youtube_api.cr~ | 685 ++++++++++++++++++
8 files changed, 1043 insertions(+), 12 deletions(-)
create mode 100644 src/invidious.cr~
create mode 100644 src/invidious/jobs/token_monitor.cr~
rename src/invidious/{reloadpotoken.cr => reloadpotoken.cr~} (100%)
create mode 100644 src/invidious/reloadtoken.cr
create mode 100644 src/invidious/yt_backend/youtube_api.cr~
diff --git a/src/invidious.cr b/src/invidious.cr
index 0b420ddf..b2387158 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -189,9 +189,7 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL
Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
-ReloadPOToken.get_tokens #init
-
-Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new()
+Invidious::Jobs.register Invidious::Jobs::MonitorCfgTokensJob.new()
Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
diff --git a/src/invidious.cr~ b/src/invidious.cr~
new file mode 100644
index 00000000..0b420ddf
--- /dev/null
+++ b/src/invidious.cr~
@@ -0,0 +1,248 @@
+# "Invidious" (which is an alternative front-end to YouTube)
+# Copyright (C) 2019 Omar Roth
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+require "digest/md5"
+require "file_utils"
+
+# Require kemal, kilt, then our own overrides
+require "kemal"
+require "kilt"
+require "./ext/kemal_content_for.cr"
+require "./ext/kemal_static_file_handler.cr"
+
+require "athena-negotiation"
+require "openssl/hmac"
+require "option_parser"
+require "sqlite3"
+require "xml"
+require "yaml"
+require "compress/zip"
+require "protodec/utils"
+
+require "./invidious/database/*"
+require "./invidious/database/migrations/*"
+require "./invidious/http_server/*"
+require "./invidious/helpers/*"
+require "./invidious/yt_backend/*"
+require "./invidious/frontend/*"
+require "./invidious/videos/*"
+
+require "./invidious/jsonify/**"
+
+require "./invidious/*"
+require "./invidious/comments/*"
+require "./invidious/channels/*"
+require "./invidious/user/*"
+require "./invidious/search/*"
+require "./invidious/routes/**"
+require "./invidious/jobs/**"
+
+# Declare the base namespace for invidious
+module Invidious
+end
+
+# Simple alias to make code easier to read
+alias IV = Invidious
+
+CONFIG = Config.load
+HMAC_KEY = CONFIG.hmac_key
+
+PG_DB = DB.open CONFIG.database_url
+ARCHIVE_URL = URI.parse("https://archive.org")
+PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
+REDDIT_URL = URI.parse("https://www.reddit.com")
+YT_URL = URI.parse("https://www.youtube.com")
+HOST_URL = make_host_url(Kemal.config)
+
+CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
+MAX_ITEMS_PER_PAGE = 1500
+
+REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
+RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
+HTTP_CHUNK_SIZE = 10485760 # ~10MB
+
+CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
+CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
+CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
+
+# This is used to determine the `?v=` on the end of file URLs (for cache busting). We
+# only need to expire modified assets, so we can use this to find the last commit that changes
+# any assets
+ASSET_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}
+
+SOFTWARE = {
+ "name" => "invidious",
+ "version" => "#{CURRENT_VERSION}-#{CURRENT_COMMIT}",
+ "branch" => "#{CURRENT_BRANCH}",
+}
+
+YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
+
+# CLI
+Kemal.config.extra_options do |parser|
+ parser.banner = "Usage: invidious [arguments]"
+ parser.on("-c THREADS", "--channel-threads=THREADS", "Number of threads for refreshing channels (default: #{CONFIG.channel_threads})") do |number|
+ begin
+ CONFIG.channel_threads = number.to_i
+ rescue ex
+ puts "THREADS must be integer"
+ exit
+ end
+ end
+ parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
+ begin
+ CONFIG.feed_threads = number.to_i
+ rescue ex
+ puts "THREADS must be integer"
+ exit
+ end
+ end
+ parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
+ CONFIG.output = output
+ end
+ parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level|
+ CONFIG.log_level = LogLevel.parse(log_level)
+ end
+ parser.on("-v", "--version", "Print version") do
+ puts SOFTWARE.to_pretty_json
+ exit
+ end
+ parser.on("--migrate", "Run any migrations (beta, use at your own risk!!") do
+ Invidious::Database::Migrator.new(PG_DB).migrate
+ exit
+ end
+end
+
+Kemal::CLI.new ARGV
+
+if CONFIG.output.upcase != "STDOUT"
+ FileUtils.mkdir_p(File.dirname(CONFIG.output))
+end
+OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a")
+LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
+
+# Check table integrity
+Invidious::Database.check_integrity(CONFIG)
+
+{% if !flag?(:skip_videojs_download) %}
+ # Resolve player dependencies. This is done at compile time.
+ #
+ # Running the script by itself would show some colorful feedback while this doesn't.
+ # Perhaps we should just move the script to runtime in order to get that feedback?
+
+ {% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %}
+ {% if flag?(:minified_player_dependencies) %}
+ {% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %}
+ {% else %}
+ {% puts run("../scripts/fetch-player-dependencies.cr").stringify %}
+ {% end %}
+ {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}
+{% end %}
+
+# Misc
+
+DECRYPT_FUNCTION =
+ if sig_helper_address = CONFIG.signature_server.presence
+ IV::DecryptFunction.new(sig_helper_address)
+ else
+ nil
+ end
+
+# Start jobs
+
+if CONFIG.channel_threads > 0
+ Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
+end
+
+if CONFIG.feed_threads > 0
+ Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
+end
+
+if CONFIG.statistics_enabled
+ Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE)
+end
+
+if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0)
+ Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, HMAC_KEY)
+end
+
+if CONFIG.popular_enabled
+ Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
+end
+
+CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
+Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
+
+Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
+
+ReloadPOToken.get_tokens #init
+
+Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new()
+
+Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
+
+Invidious::Jobs.start_all
+
+def popular_videos
+ Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
+end
+
+# Routing
+
+before_all do |env|
+ Invidious::Routes::BeforeAll.handle(env)
+end
+
+Invidious::Routing.register_all
+
+error 404 do |env|
+ Invidious::Routes::ErrorRoutes.error_404(env)
+end
+
+error 500 do |env, ex|
+ error_template(500, ex)
+end
+
+static_headers do |response|
+ response.headers.add("Cache-Control", "max-age=2629800")
+end
+
+# Init Kemal
+
+public_folder "assets"
+
+Kemal.config.powered_by_header = false
+add_handler FilteredCompressHandler.new
+add_handler APIHandler.new
+add_handler AuthHandler.new
+add_handler DenyFrame.new
+add_context_storage_type(Array(String))
+add_context_storage_type(Preferences)
+add_context_storage_type(Invidious::User)
+
+Kemal.config.logger = LOGGER
+Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding
+Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port
+Kemal.config.app_name = "Invidious"
+
+# Use in kemal's production mode.
+# Users can also set the KEMAL_ENV environmental variable for this to be set automatically.
+{% if flag?(:release) || flag?(:production) %}
+ Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV")
+{% end %}
+
+Kemal.run
diff --git a/src/invidious/jobs/token_monitor.cr b/src/invidious/jobs/token_monitor.cr
index a94f7ed1..e863acfe 100644
--- a/src/invidious/jobs/token_monitor.cr
+++ b/src/invidious/jobs/token_monitor.cr
@@ -1,15 +1,15 @@
-class Invidious::Jobs::MonitorCfgPotokensJob < Invidious::Jobs::BaseJob
+class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob
include Invidious
def begin
loop do
- LOGGER.info("jobs: running MonitorCfgPotokens job")
+ LOGGER.info("jobs: running MonitorCfgTokensJob job")
ReloadPOToken.get_tokens
- LOGGER.info("jobs: MonitorCfgPotokens: pot: " + ReloadPOToken.pot.as(String))
- LOGGER.info("jobs: MonitorCfgPotokens: vdata: " + ReloadPOToken.vdata.as(String))
+ LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadTokens.pot.as(String))
+ LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadTokens.vdata.as(String))
sleep 15.seconds
end
diff --git a/src/invidious/jobs/token_monitor.cr~ b/src/invidious/jobs/token_monitor.cr~
new file mode 100644
index 00000000..d923dd7d
--- /dev/null
+++ b/src/invidious/jobs/token_monitor.cr~
@@ -0,0 +1,17 @@
+
+class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob
+ include Invidious
+ def begin
+ loop do
+
+ LOGGER.info("jobs: running MonitorCfgTokensJob job")
+
+ ReloadPOToken.get_tokens
+
+ LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadPOToken.pot.as(String))
+ LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadPOToken.vdata.as(String))
+
+ sleep 15.seconds
+ end
+ end
+end
diff --git a/src/invidious/reloadpotoken.cr b/src/invidious/reloadpotoken.cr~
similarity index 100%
rename from src/invidious/reloadpotoken.cr
rename to src/invidious/reloadpotoken.cr~
diff --git a/src/invidious/reloadtoken.cr b/src/invidious/reloadtoken.cr
new file mode 100644
index 00000000..8ff56f48
--- /dev/null
+++ b/src/invidious/reloadtoken.cr
@@ -0,0 +1,83 @@
+class ReloadToken
+
+ @@instance = new
+
+ def self.pot
+ @@pot
+ end
+
+ def self.vdata
+ @@vdata
+ end
+
+ def initialize
+
+ @@pot = "error"
+ @@vdata = "error"
+
+ end
+
+ def self.get_tokens
+
+ # Load config from file or YAML string env var
+ env_config_file = "INVIDIOUS_CONFIG_FILE"
+ env_config_yaml = "INVIDIOUS_CONFIG"
+
+ config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
+ config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
+
+ config = Config.from_yaml(config_yaml)
+
+
+
+
+ # Update config from env vars (upcased and prefixed with "INVIDIOUS_")
+ {% for ivar in Config.instance_vars %}
+ {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
+
+ if ENV.has_key?({{env_id}})
+ env_value = ENV.fetch({{env_id}})
+ success = false
+
+ # Use YAML converter if specified
+ {% ann = ivar.annotation(::YAML::Field) %}
+ {% if ann && ann[:converter] %}
+ config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
+ success = true
+
+ # Use regular YAML parser otherwise
+ {% else %}
+ {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %}
+ # Sort types to avoid parsing nulls and numbers as strings
+ {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %}
+ {{ivar_types}}.each do |ivar_type|
+ if !success
+ begin
+ config.{{ivar.id}} = ivar_type.from_yaml(env_value)
+ success = true
+ rescue
+ # nop
+ end
+ end
+ end
+ {% end %}
+
+ # Exit on fail
+ if !success
+ puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}})
+ exit(1)
+ end
+ end
+ {% end %}
+
+
+ @@pot = config.po_token
+ @@vdata = config.visitor_data
+
+ end
+
+ def self.get_instance
+ return @@instance
+ end
+
+end
\ No newline at end of file
diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index b717a67e..7862fd80 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -320,8 +320,8 @@ module YoutubeAPI
client_context["client"]["platform"] = platform
end
- if ReloadPOToken.vdata.is_a?(String)
- client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String)
+ if ReloadToken.vdata.is_a?(String)
+ client_context["client"]["visitorData"] = ReloadToken.vdata.as(String)
end
return client_context
@@ -482,7 +482,7 @@ module YoutubeAPI
"contentPlaybackContext" => playback_ctx,
},
"serviceIntegrityDimensions" => {
- "poToken" => ReloadPOToken.pot.as(String),
+ "poToken" => ReloadToken.pot.as(String),
},
}
@@ -616,8 +616,8 @@ module YoutubeAPI
headers["User-Agent"] = user_agent
end
- if ReloadPOToken.vdata.is_a?(String)
- headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String)
+ if ReloadToken.vdata.is_a?(String)
+ headers["X-Goog-Visitor-Id"] = ReloadToken.vdata.as(String)
end
# Logging
diff --git a/src/invidious/yt_backend/youtube_api.cr~ b/src/invidious/yt_backend/youtube_api.cr~
new file mode 100644
index 00000000..b717a67e
--- /dev/null
+++ b/src/invidious/yt_backend/youtube_api.cr~
@@ -0,0 +1,685 @@
+#
+# This file contains youtube API wrappers
+#
+
+module YoutubeAPI
+ extend self
+
+ # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
+ private ANDROID_APP_VERSION = "19.32.34"
+ private ANDROID_VERSION = "12"
+ private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip"
+ private ANDROID_SDK_VERSION = 31_i64
+
+ private ANDROID_TS_APP_VERSION = "1.9"
+ private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip"
+
+ # For Apple device names, see https://gist.github.com/adamawolf/3048717
+ # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
+ # then go to the dedicated article of the major version you want.
+ private IOS_APP_VERSION = "19.32.8"
+ private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)"
+ private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build
+
+ private WINDOWS_VERSION = "10.0"
+
+ # Enumerate used to select one of the clients supported by the API
+ enum ClientType
+ Web
+ WebEmbeddedPlayer
+ WebMobile
+ WebScreenEmbed
+
+ Android
+ AndroidEmbeddedPlayer
+ AndroidScreenEmbed
+ AndroidTestSuite
+
+ IOS
+ IOSEmbedded
+ IOSMusic
+
+ TvHtml5
+ TvHtml5ScreenEmbed
+ end
+
+ # List of hard-coded values used by the different clients
+ HARDCODED_CLIENTS = {
+ ClientType::Web => {
+ name: "WEB",
+ name_proto: "1",
+ version: "2.20240814.00.00",
+ screen: "WATCH_FULL_SCREEN",
+ os_name: "Windows",
+ os_version: WINDOWS_VERSION,
+ platform: "DESKTOP",
+ },
+ ClientType::WebEmbeddedPlayer => {
+ name: "WEB_EMBEDDED_PLAYER",
+ name_proto: "56",
+ version: "1.20240812.01.00",
+ screen: "EMBED",
+ os_name: "Windows",
+ os_version: WINDOWS_VERSION,
+ platform: "DESKTOP",
+ },
+ ClientType::WebMobile => {
+ name: "MWEB",
+ name_proto: "2",
+ version: "2.20240813.02.00",
+ os_name: "Android",
+ os_version: ANDROID_VERSION,
+ platform: "MOBILE",
+ },
+ ClientType::WebScreenEmbed => {
+ name: "WEB",
+ name_proto: "1",
+ version: "2.20240814.00.00",
+ screen: "EMBED",
+ os_name: "Windows",
+ os_version: WINDOWS_VERSION,
+ platform: "DESKTOP",
+ },
+
+ # Android
+
+ ClientType::Android => {
+ name: "ANDROID",
+ name_proto: "3",
+ version: ANDROID_APP_VERSION,
+ android_sdk_version: ANDROID_SDK_VERSION,
+ user_agent: ANDROID_USER_AGENT,
+ os_name: "Android",
+ os_version: ANDROID_VERSION,
+ platform: "MOBILE",
+ },
+ ClientType::AndroidEmbeddedPlayer => {
+ name: "ANDROID_EMBEDDED_PLAYER",
+ name_proto: "55",
+ version: ANDROID_APP_VERSION,
+ },
+ ClientType::AndroidScreenEmbed => {
+ name: "ANDROID",
+ name_proto: "3",
+ version: ANDROID_APP_VERSION,
+ screen: "EMBED",
+ android_sdk_version: ANDROID_SDK_VERSION,
+ user_agent: ANDROID_USER_AGENT,
+ os_name: "Android",
+ os_version: ANDROID_VERSION,
+ platform: "MOBILE",
+ },
+ ClientType::AndroidTestSuite => {
+ name: "ANDROID_TESTSUITE",
+ name_proto: "30",
+ version: ANDROID_TS_APP_VERSION,
+ android_sdk_version: ANDROID_SDK_VERSION,
+ user_agent: ANDROID_TS_USER_AGENT,
+ os_name: "Android",
+ os_version: ANDROID_VERSION,
+ platform: "MOBILE",
+ },
+
+ # IOS
+
+ ClientType::IOS => {
+ name: "IOS",
+ name_proto: "5",
+ version: IOS_APP_VERSION,
+ user_agent: IOS_USER_AGENT,
+ device_make: "Apple",
+ device_model: "iPhone14,5",
+ os_name: "iPhone",
+ os_version: IOS_VERSION,
+ platform: "MOBILE",
+ },
+ ClientType::IOSEmbedded => {
+ name: "IOS_MESSAGES_EXTENSION",
+ name_proto: "66",
+ version: IOS_APP_VERSION,
+ user_agent: IOS_USER_AGENT,
+ device_make: "Apple",
+ device_model: "iPhone14,5",
+ os_name: "iPhone",
+ os_version: IOS_VERSION,
+ platform: "MOBILE",
+ },
+ ClientType::IOSMusic => {
+ name: "IOS_MUSIC",
+ name_proto: "26",
+ version: "7.14",
+ user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)",
+ device_make: "Apple",
+ device_model: "iPhone14,5",
+ os_name: "iPhone",
+ os_version: IOS_VERSION,
+ platform: "MOBILE",
+ },
+
+ # TV app
+
+ ClientType::TvHtml5 => {
+ name: "TVHTML5",
+ name_proto: "7",
+ version: "7.20240813.07.00",
+ },
+ ClientType::TvHtml5ScreenEmbed => {
+ name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
+ name_proto: "85",
+ version: "2.0",
+ screen: "EMBED",
+ },
+ }
+
+ ####################################################################
+ # struct ClientConfig
+ #
+ # Data structure used to pass a client configuration to the different
+ # API endpoints handlers.
+ #
+ # Use case examples:
+ #
+ # ```
+ # # Get Norwegian search results
+ # conf_1 = ClientConfig.new(region: "NO")
+ # YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1)
+ #
+ # # Use the Android client to request video streams URLs
+ # conf_2 = ClientConfig.new(client_type: ClientType::Android)
+ # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2)
+ #
+ #
+ struct ClientConfig
+ # Type of client to emulate.
+ # See `enum ClientType` and `HARDCODED_CLIENTS`.
+ property client_type : ClientType
+
+ # Region to provide to youtube, e.g to alter search results
+ # (this is passed as the `gl` parameter).
+ property region : String | Nil
+
+ # Initialization function
+ def initialize(
+ *,
+ @client_type = ClientType::Web,
+ @region = "US"
+ )
+ end
+
+ # Getter functions that provides easy access to hardcoded clients
+ # parameters (name/version strings and related API key)
+ def name : String
+ HARDCODED_CLIENTS[@client_type][:name]
+ end
+
+ def name_proto : String
+ HARDCODED_CLIENTS[@client_type][:name_proto]
+ end
+
+ # :ditto:
+ def version : String
+ HARDCODED_CLIENTS[@client_type][:version]
+ end
+
+ # :ditto:
+ def screen : String
+ HARDCODED_CLIENTS[@client_type][:screen]? || ""
+ end
+
+ def android_sdk_version : Int64?
+ HARDCODED_CLIENTS[@client_type][:android_sdk_version]?
+ end
+
+ def user_agent : String?
+ HARDCODED_CLIENTS[@client_type][:user_agent]?
+ end
+
+ def os_name : String?
+ HARDCODED_CLIENTS[@client_type][:os_name]?
+ end
+
+ def device_make : String?
+ HARDCODED_CLIENTS[@client_type][:device_make]?
+ end
+
+ def device_model : String?
+ HARDCODED_CLIENTS[@client_type][:device_model]?
+ end
+
+ def os_version : String?
+ HARDCODED_CLIENTS[@client_type][:os_version]?
+ end
+
+ def platform : String?
+ HARDCODED_CLIENTS[@client_type][:platform]?
+ end
+
+ # Convert to string, for logging purposes
+ def to_s
+ return {
+ client_type: self.name,
+ region: @region,
+ }.to_s
+ end
+ end
+
+ # Default client config, used if nothing is passed
+ DEFAULT_CLIENT_CONFIG = ClientConfig.new
+
+ ####################################################################
+ # make_context(client_config)
+ #
+ # Return, as a Hash, the "context" data required to request the
+ # youtube API endpoints.
+ #
+ private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash
+ # Use the default client config if nil is passed
+ client_config ||= DEFAULT_CLIENT_CONFIG
+
+ client_context = {
+ "client" => {
+ "hl" => "en",
+ "gl" => client_config.region || "US", # Can't be empty!
+ "clientName" => client_config.name,
+ "clientVersion" => client_config.version,
+ } of String => String | Int64,
+ }
+
+ # Add some more context if it exists in the client definitions
+ if !client_config.screen.empty?
+ client_context["client"]["clientScreen"] = client_config.screen
+ end
+
+ if client_config.screen == "EMBED"
+ client_context["thirdParty"] = {
+ "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+ } of String => String | Int64
+ end
+
+ if android_sdk_version = client_config.android_sdk_version
+ client_context["client"]["androidSdkVersion"] = android_sdk_version
+ end
+
+ if device_make = client_config.device_make
+ client_context["client"]["deviceMake"] = device_make
+ end
+
+ if device_model = client_config.device_model
+ client_context["client"]["deviceModel"] = device_model
+ end
+
+ if os_name = client_config.os_name
+ client_context["client"]["osName"] = os_name
+ end
+
+ if os_version = client_config.os_version
+ client_context["client"]["osVersion"] = os_version
+ end
+
+ if platform = client_config.platform
+ client_context["client"]["platform"] = platform
+ end
+
+ if ReloadPOToken.vdata.is_a?(String)
+ client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String)
+ end
+
+ return client_context
+ end
+
+ ####################################################################
+ # browse(continuation, client_config?)
+ # browse(browse_id, params, client_config?)
+ #
+ # Requests the youtubei/v1/browse endpoint with the required headers
+ # and POST data in order to get a JSON reply in english that can
+ # be easily parsed.
+ #
+ # Both forms can take an optional ClientConfig parameter (see
+ # `struct ClientConfig` above for more details).
+ #
+ # The requested data can either be:
+ #
+ # - A continuation token (ctoken). Depending on this token's
+ # contents, the returned data can be playlist videos, channel
+ # community tab content, channel info, ...
+ #
+ # - A playlist ID (parameters MUST be an empty string)
+ #
+ def browse(continuation : String, client_config : ClientConfig | Nil = nil)
+ # JSON Request data, required by the API
+ data = {
+ "context" => self.make_context(client_config),
+ "continuation" => continuation,
+ }
+
+ return self._post_json("/youtubei/v1/browse", data, client_config)
+ end
+
+ # :ditto:
+ def browse(
+ browse_id : String,
+ *, # Force the following parameters to be passed by name
+ params : String,
+ client_config : ClientConfig | Nil = nil
+ )
+ # JSON Request data, required by the API
+ data = {
+ "browseId" => browse_id,
+ "context" => self.make_context(client_config),
+ }
+
+ # Append the additional parameters if those were provided
+ # (this is required for channel info, playlist and community, e.g)
+ if params != ""
+ data["params"] = params
+ end
+
+ return self._post_json("/youtubei/v1/browse", data, client_config)
+ end
+
+ ####################################################################
+ # next(continuation, client_config?)
+ # next(data, client_config?)
+ #
+ # Requests the youtubei/v1/next endpoint with the required headers
+ # and POST data in order to get a JSON reply in english that can
+ # be easily parsed.
+ #
+ # Both forms can take an optional ClientConfig parameter (see
+ # `struct ClientConfig` above for more details).
+ #
+ # The requested data can be:
+ #
+ # - A continuation token (ctoken). Depending on this token's
+ # contents, the returned data can be videos comments,
+ # their replies, ... In this case, the string must be passed
+ # directly to the function. E.g:
+ #
+ # ```
+ # YoutubeAPI::next("ABCDEFGH_abcdefgh==")
+ # ```
+ #
+ # - Arbitrary parameters, in Hash form. See examples below for
+ # known examples of arbitrary data that can be passed to YouTube:
+ #
+ # ```
+ # # Get the videos related to a specific video ID
+ # YoutubeAPI::next({"videoId" => "dQw4w9WgXcQ"})
+ #
+ # # Get a playlist video's details
+ # YoutubeAPI::next({
+ # "videoId" => "9bZkp7q19f0",
+ # "playlistId" => "PL_oFlvgqkrjUVQwiiE3F3k3voF4tjXeP0",
+ # })
+ # ```
+ #
+ def next(continuation : String, *, client_config : ClientConfig | Nil = nil)
+ # JSON Request data, required by the API
+ data = {
+ "context" => self.make_context(client_config),
+ "continuation" => continuation,
+ }
+
+ return self._post_json("/youtubei/v1/next", data, client_config)
+ end
+
+ # :ditto:
+ def next(data : Hash, *, client_config : ClientConfig | Nil = nil)
+ # JSON Request data, required by the API
+ data2 = data.merge({
+ "context" => self.make_context(client_config),
+ })
+
+ return self._post_json("/youtubei/v1/next", data2, client_config)
+ end
+
+ # Allow a NamedTuple to be passed, too.
+ def next(data : NamedTuple, *, client_config : ClientConfig | Nil = nil)
+ return self.next(data.to_h, client_config: client_config)
+ end
+
+ ####################################################################
+ # player(video_id, params, client_config?)
+ #
+ # Requests the youtubei/v1/player endpoint with the required headers
+ # and POST data in order to get a JSON reply.
+ #
+ # The requested data is a video ID (`v=` parameter), with some
+ # additional parameters, formatted as a base64 string.
+ #
+ # An optional ClientConfig parameter can be passed, too (see
+ # `struct ClientConfig` above for more details).
+ #
+ def player(
+ video_id : String,
+ *, # Force the following parameters to be passed by name
+ params : String,
+ client_config : ClientConfig | Nil = nil
+ )
+ # Playback context, separate because it can be different between clients
+ playback_ctx = {
+ "html5Preference" => "HTML5_PREF_WANTS",
+ "referer" => "https://www.youtube.com/watch?v=#{video_id}",
+ } of String => String | Int64
+
+ if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s }
+ if sts = DECRYPT_FUNCTION.try &.get_sts
+ playback_ctx["signatureTimestamp"] = sts.to_i64
+ end
+ end
+
+ # JSON Request data, required by the API
+ data = {
+ "contentCheckOk" => true,
+ "videoId" => video_id,
+ "context" => self.make_context(client_config, video_id),
+ "racyCheckOk" => true,
+ "user" => {
+ "lockedSafetyMode" => false,
+ },
+ "playbackContext" => {
+ "contentPlaybackContext" => playback_ctx,
+ },
+ "serviceIntegrityDimensions" => {
+ "poToken" => ReloadPOToken.pot.as(String),
+ },
+ }
+
+ # Append the additional parameters if those were provided
+ if params != ""
+ data["params"] = params
+ end
+
+ return self._post_json("/youtubei/v1/player", data, client_config)
+ end
+
+ ####################################################################
+ # resolve_url(url, client_config?)
+ #
+ # Requests the youtubei/v1/navigation/resolve_url endpoint with the
+ # required headers and POST data in order to get a JSON reply.
+ #
+ # An optional ClientConfig parameter can be passed, too (see
+ # `struct ClientConfig` above for more details).
+ #
+ # Output:
+ #
+ # ```
+ # # Valid channel "brand URL" gives the related UCID and browse ID
+ # channel_a = YoutubeAPI.resolve_url("https://youtube.com/c/google")
+ # channel_a # => {
+ # "endpoint": {
+ # "browseEndpoint": {
+ # "params": "EgC4AQA%3D",
+ # "browseId":"UCK8sQmJBp8GCxrOtXWBpyEA"
+ # },
+ # ...
+ # }
+ # }
+ #
+ # # Invalid URL returns throws an InfoException
+ # channel_b = YoutubeAPI.resolve_url("https://youtube.com/c/invalid")
+ # ```
+ #
+ def resolve_url(url : String, client_config : ClientConfig | Nil = nil)
+ data = {
+ "context" => self.make_context(nil),
+ "url" => url,
+ }
+
+ return self._post_json("/youtubei/v1/navigation/resolve_url", data, client_config)
+ end
+
+ ####################################################################
+ # search(search_query, params, client_config?)
+ #
+ # Requests the youtubei/v1/search endpoint with the required headers
+ # and POST data in order to get a JSON reply. As the search results
+ # vary depending on the region, a region code can be specified in
+ # order to get non-US results.
+ #
+ # The requested data is a search string, with some additional
+ # parameters, formatted as a base64 string.
+ #
+ # An optional ClientConfig parameter can be passed, too (see
+ # `struct ClientConfig` above for more details).
+ #
+ def search(
+ search_query : String,
+ params : String,
+ client_config : ClientConfig | Nil = nil
+ )
+ # JSON Request data, required by the API
+ data = {
+ "query" => search_query,
+ "context" => self.make_context(client_config),
+ "params" => params,
+ }
+
+ return self._post_json("/youtubei/v1/search", data, client_config)
+ end
+
+ ####################################################################
+ # get_transcript(params, client_config?)
+ #
+ # Requests the youtubei/v1/get_transcript endpoint with the required headers
+ # and POST data in order to get a JSON reply.
+ #
+ # The requested data is a specially encoded protobuf string that denotes the specific language requested.
+ #
+ # An optional ClientConfig parameter can be passed, too (see
+ # `struct ClientConfig` above for more details).
+ #
+
+ def get_transcript(
+ params : String,
+ client_config : ClientConfig | Nil = nil
+ ) : Hash(String, JSON::Any)
+ data = {
+ "context" => self.make_context(client_config),
+ "params" => params,
+ }
+
+ return self._post_json("/youtubei/v1/get_transcript", data, client_config)
+ end
+
+ ####################################################################
+ # _post_json(endpoint, data, client_config?)
+ #
+ # Internal function that does the actual request to youtube servers
+ # and handles errors.
+ #
+ # The requested data is an endpoint (URL without the domain part)
+ # and the data as a Hash object.
+ #
+ def _post_json(
+ endpoint : String,
+ data : Hash,
+ client_config : ClientConfig | Nil
+ ) : Hash(String, JSON::Any)
+ # Use the default client config if nil is passed
+ client_config ||= DEFAULT_CLIENT_CONFIG
+
+ # Query parameters
+ url = "#{endpoint}?prettyPrint=false"
+
+ headers = HTTP::Headers{
+ "Content-Type" => "application/json; charset=UTF-8",
+ "Accept-Encoding" => "gzip, deflate",
+ "x-goog-api-format-version" => "2",
+ "x-youtube-client-name" => client_config.name_proto,
+ "x-youtube-client-version" => client_config.version,
+ }
+
+ if user_agent = client_config.user_agent
+ headers["User-Agent"] = user_agent
+ end
+
+ if ReloadPOToken.vdata.is_a?(String)
+ headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String)
+ end
+
+ # Logging
+ LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"")
+ LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")
+ LOGGER.trace("YoutubeAPI: POST data: #{data}")
+
+ # Send the POST request
+ body = YT_POOL.client() do |client|
+ client.post(url, headers: headers, body: data.to_json) do |response|
+ self._decompress(response.body_io, response.headers["Content-Encoding"]?)
+ end
+ end
+
+ # Convert result to Hash
+ initial_data = JSON.parse(body).as_h
+
+ # Error handling
+ if initial_data.has_key?("error")
+ code = initial_data["error"]["code"]
+ message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "")
+
+ # Logging
+ LOGGER.error("YoutubeAPI: Got error #{code} when requesting #{endpoint}")
+ LOGGER.error("YoutubeAPI: #{message}")
+ LOGGER.info("YoutubeAPI: POST data was: #{data}")
+
+ raise InfoException.new("Could not extract JSON. Youtube API returned \
+ error #{code} with message:
\"#{message}\"")
+ end
+
+ return initial_data
+ end
+
+ ####################################################################
+ # _decompress(body_io, headers)
+ #
+ # Internal function that reads the Content-Encoding headers and
+ # decompresses the content accordingly.
+ #
+ # We decompress the body ourselves (when using HTTP::Client) because
+ # the auto-decompress feature is broken in the Crystal stdlib.
+ #
+ # Read more:
+ # - https://github.com/iv-org/invidious/issues/2612
+ # - https://github.com/crystal-lang/crystal/issues/11354
+ #
+ def _decompress(body_io : IO, encodings : String?) : String
+ if encodings
+ # Multiple encodings can be combined, and are listed in the order
+ # in which they were applied. E.g: "deflate, gzip" means that the
+ # content must be first "gunzipped", then "defated".
+ encodings.split(',').reverse.each do |enc|
+ case enc.strip(' ')
+ when "gzip"
+ body_io = Compress::Gzip::Reader.new(body_io, sync_close: true)
+ when "deflate"
+ body_io = Compress::Deflate::Reader.new(body_io, sync_close: true)
+ end
+ end
+ end
+
+ return body_io.gets_to_end
+ end
+end # End of module