From eb8fcc9e88bf71694e7d9e40600ef3fd334bc46e Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Fri, 25 Aug 2023 16:08:02 -0700
Subject: [PATCH 01/61] Add support for using HTTP proxies

---
 config/config.example.yml                   | 11 +++++++++++
 shard.lock                                  | 10 +++++++---
 shard.yml                                   |  3 +++
 src/invidious.cr                            |  1 +
 src/invidious/config.cr                     | 11 +++++++++++
 src/invidious/yt_backend/connection_pool.cr | 18 ++++++++++++++++++
 6 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index 38085a20..9fdc4eda 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -160,6 +160,17 @@ https_only: false
 ##
 #force_resolve:
 
+##
+## Configuration for using a HTTP proxy
+##
+## If unset, then no HTTP proxy will be used.
+## 
+http_proxy:
+  user:
+  password:
+  host:
+  port:
+
 
 ##
 ## Use Innertube's transcripts API instead of timedtext for closed captions
diff --git a/shard.lock b/shard.lock
index efb60a59..85928c47 100644
--- a/shard.lock
+++ b/shard.lock
@@ -6,11 +6,11 @@ shards:
 
   athena-negotiation:
     git: https://github.com/athena-framework/negotiation.git
-    version: 0.1.1
+    version: 0.1.3
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
-    version: 1.2.1
+    version: 1.2.2
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
@@ -20,6 +20,10 @@ shards:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.2.2
 
+  http_proxy:
+    git: https://github.com/mamantoha/http_proxy.git
+    version: 0.10.1
+
   kemal:
     git: https://github.com/kemalcr/kemal.git
     version: 1.1.2
@@ -42,7 +46,7 @@ shards:
 
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
-    version: 0.10.4
+    version: 0.10.6
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
diff --git a/shard.yml b/shard.yml
index be06a7df..ddd510d4 100644
--- a/shard.yml
+++ b/shard.yml
@@ -28,6 +28,9 @@ dependencies:
   athena-negotiation:
     github: athena-framework/negotiation
     version: ~> 0.1.1
+  http_proxy:
+    github: mamantoha/http_proxy
+    version: ~> 0.10.1
 
 development_dependencies:
   spectator:
diff --git a/src/invidious.cr b/src/invidious.cr
index e0bd0101..d4114386 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -23,6 +23,7 @@ require "kilt"
 require "./ext/kemal_content_for.cr"
 require "./ext/kemal_static_file_handler.cr"
 
+require "http_proxy"
 require "athena-negotiation"
 require "openssl/hmac"
 require "option_parser"
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 09c2168b..e12054d0 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -54,6 +54,15 @@ struct ConfigPreferences
   end
 end
 
+struct HTTPProxyConfig
+  include YAML::Serializable
+
+  property user : String
+  property password : String
+  property host : String
+  property port : Int32
+end
+
 class Config
   include YAML::Serializable
 
@@ -126,6 +135,8 @@ class Config
   property host_binding : String = "0.0.0.0"
   # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
   property pool_size : Int32 = 100
+  # HTTP Proxy configuration
+  property http_proxy : HTTPProxyConfig? = nil
 
   # Use Innertube's transcripts API instead of timedtext for closed captions
   property use_innertube_for_captions : Bool = false
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index d3dbcc0e..0e4d8aff 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -26,12 +26,14 @@ struct YoutubeConnectionPool
 
   def client(&block)
     conn = pool.checkout
+    conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
     begin
       response = yield conn
     rescue ex
       conn.close
       conn = HTTP::Client.new(url)
 
+      conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@@ -46,6 +48,7 @@ struct YoutubeConnectionPool
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
       conn = HTTP::Client.new(url)
+      conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false)
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 
+  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+
   return client
 end
 
@@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block)
     client.close
   end
 end
+
+def make_configured_http_proxy_client
+  # This method is only called when configuration for an HTTP proxy are set
+  config_proxy = CONFIG.http_proxy.not_nil!
+
+  return HTTP::Proxy::Client.new(
+    config_proxy.host,
+    config_proxy.port,
+
+    username: config_proxy.user,
+    password: config_proxy.password,
+  )
+end

From 3b471ae964ad0122c81205965c221a352e3a658f Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Wed, 4 Oct 2023 14:36:04 -0400
Subject: [PATCH 02/61] Automatically initialize proxy via stdlib override

---
 .../helpers/crystal_class_overrides.cr        | 34 +++++++++++++++++++
 src/invidious/yt_backend/connection_pool.cr   |  7 ++--
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr
index bf56d826..71038703 100644
--- a/src/invidious/helpers/crystal_class_overrides.cr
+++ b/src/invidious/helpers/crystal_class_overrides.cr
@@ -18,6 +18,40 @@ end
 class HTTP::Client
   property family : Socket::Family = Socket::Family::UNSPEC
 
+  # Override stdlib to automatically initialize proxy if configured
+  #
+  # Accurate as of crystal 1.10.1
+
+  def initialize(@host : String, port = nil, tls : TLSContext = nil)
+    check_host_only(@host)
+
+    {% if flag?(:without_openssl) %}
+      if tls
+        raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time"
+      end
+      @tls = nil
+    {% else %}
+      @tls = case tls
+             when true
+               OpenSSL::SSL::Context::Client.new
+             when OpenSSL::SSL::Context::Client
+               tls
+             when false, nil
+               nil
+             end
+    {% end %}
+
+    @port = (port || (@tls ? 443 : 80)).to_i
+
+    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  end
+
+  def initialize(@io : IO, @host = "", @port = 80)
+    @reconnect = false
+
+    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  end
+
   private def io
     io = @io
     return io if io
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 0e4d8aff..f34f48c5 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -26,13 +26,15 @@ struct YoutubeConnectionPool
 
   def client(&block)
     conn = pool.checkout
+    # Proxy needs to be reinstated every time we get a client from the pool
     conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+
     begin
       response = yield conn
     rescue ex
       conn.close
-      conn = HTTP::Client.new(url)
 
+      conn = HTTP::Client.new(url)
       conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
@@ -48,7 +50,6 @@ struct YoutubeConnectionPool
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
       conn = HTTP::Client.new(url)
-      conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@@ -69,8 +70,6 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false)
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 
-  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
-
   return client
 end
 

From ccb2a6c58ef9b5b7e1c47938cfa4ec574a8560c7 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 28 Apr 2024 21:34:05 -0700
Subject: [PATCH 03/61] Bump http_proxy to v0.10.3

---
 shard.lock | 4 ++--
 shard.yml  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/shard.lock b/shard.lock
index 85928c47..99b4e99c 100644
--- a/shard.lock
+++ b/shard.lock
@@ -6,7 +6,7 @@ shards:
 
   athena-negotiation:
     git: https://github.com/athena-framework/negotiation.git
-    version: 0.1.3
+    version: 0.1.1
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
@@ -22,7 +22,7 @@ shards:
 
   http_proxy:
     git: https://github.com/mamantoha/http_proxy.git
-    version: 0.10.1
+    version: 0.10.3
 
   kemal:
     git: https://github.com/kemalcr/kemal.git
diff --git a/shard.yml b/shard.yml
index ddd510d4..70ebad0a 100644
--- a/shard.yml
+++ b/shard.yml
@@ -30,7 +30,7 @@ dependencies:
     version: ~> 0.1.1
   http_proxy:
     github: mamantoha/http_proxy
-    version: ~> 0.10.1
+    version: ~> 0.10.3
 
 development_dependencies:
   spectator:

From 6b7e7301009e1a9fc2b536bd8d8de04fb8e22ec0 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Wed, 22 May 2024 13:10:46 -0700
Subject: [PATCH 04/61] Validate override for crystal 1.12.1

---
 src/invidious/helpers/crystal_class_overrides.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr
index 71038703..a7d2a5e6 100644
--- a/src/invidious/helpers/crystal_class_overrides.cr
+++ b/src/invidious/helpers/crystal_class_overrides.cr
@@ -20,7 +20,7 @@ class HTTP::Client
 
   # Override stdlib to automatically initialize proxy if configured
   #
-  # Accurate as of crystal 1.10.1
+  # Accurate as of crystal 1.12.1
 
   def initialize(@host : String, port = nil, tls : TLSContext = nil)
     check_host_only(@host)

From 288e1dccda2256a9014364d693b3eb3d7933b242 Mon Sep 17 00:00:00 2001
From: giacomocerquone <cerquone@gmail.com>
Date: Thu, 13 Jun 2024 01:10:35 +0200
Subject: [PATCH 05/61] Fix player menus hiding onHover

---
 assets/css/player.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/assets/css/player.css b/assets/css/player.css
index 50c7a748..9cb400ad 100644
--- a/assets/css/player.css
+++ b/assets/css/player.css
@@ -68,6 +68,7 @@
 
 .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu {
   margin-bottom: 2em;
+  padding-top: 2em
 }
 
 .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px;

From c24ed85110cfa006992ce16bd4432eb39c8db71b Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 16 Jun 2024 14:49:48 -0700
Subject: [PATCH 06/61] Fix named arg syntax when passing force_resolve

---
 src/invidious/routes/video_playback.cr      | 8 ++++----
 src/invidious/yt_backend/connection_pool.cr | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index ec18f3b8..254c0b46 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback
       headers["Range"] = "bytes=#{range_for_head}"
     end
 
-    client = make_client(URI.parse(host), region, force_resolve = true)
+    client = make_client(URI.parse(host), region, force_resolve: true)
     response = HTTP::Client::Response.new(500)
     error = ""
     5.times do
@@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback
           if new_host != host
             host = new_host
             client.close
-            client = make_client(URI.parse(new_host), region, force_resolve = true)
+            client = make_client(URI.parse(new_host), region, force_resolve: true)
           end
 
           url = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback
         fvip = "3"
 
         host = "https://r#{fvip}---#{mn}.googlevideo.com"
-        client = make_client(URI.parse(host), region, force_resolve = true)
+        client = make_client(URI.parse(host), region, force_resolve: true)
       rescue ex
         error = ex.message
       end
@@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback
             break
           else
             client.close
-            client = make_client(URI.parse(host), region, force_resolve = true)
+            client = make_client(URI.parse(host), region, force_resolve: true)
           end
         end
 
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index d3dbcc0e..bcf6a003 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -70,7 +70,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false)
 end
 
 def make_client(url : URI, region = nil, force_resolve : Bool = false, &block)
-  client = make_client(url, region, force_resolve)
+  client = make_client(url, region, force_resolve: force_resolve)
   begin
     yield client
   ensure

From 1124dd645d0db872b01a0c476c205da057a8fd04 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Wed, 22 May 2024 11:29:28 -0700
Subject: [PATCH 07/61] Use `make_client` instead of calling `HTTP::Client`

Using `make_client` to create `HTTP::Client`, allows for a simple way to
easily add logic to all `HTTP::Client` initialized within Invidious.
---
 src/invidious/routes/api/v1/search.cr       |  4 +---
 src/invidious/yt_backend/connection_pool.cr | 13 ++++---------
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 2922b060..6785ef73 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search
     query = env.params.query["q"]? || ""
 
     begin
-      client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
-      client.before_request { |r| add_yt_headers(r) }
-
+      client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers = true)
       url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
 
       response = client.get(url).body
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index ca612083..84d857ec 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -30,11 +30,8 @@ struct YoutubeConnectionPool
       response = yield conn
     rescue ex
       conn.close
-      conn = HTTP::Client.new(url)
-
-      conn.family = CONFIG.force_resolve
+      conn = make_client(url)
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
-      conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
       response = yield conn
     ensure
       pool.release(conn)
@@ -45,16 +42,14 @@ struct YoutubeConnectionPool
 
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
-      conn = HTTP::Client.new(url)
-      conn.family = CONFIG.force_resolve
+      conn = make_client(url, force_solve = true)
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
-      conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
       conn
     end
   end
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_header : Bool = false)
   client = HTTP::Client.new(url)
 
   # Force the usage of a specific configured IP Family
@@ -62,7 +57,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false)
     client.family = CONFIG.force_resolve
   end
 
-  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
+  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_header
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 

From 3af668186997d21295247ed6e31c6fd4634fa511 Mon Sep 17 00:00:00 2001
From: syeopite <70992037+syeopite@users.noreply.github.com>
Date: Fri, 24 May 2024 13:11:14 -0700
Subject: [PATCH 08/61] Fix typo in argument to `make_client`

Co-authored-by: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com>
---
 src/invidious/yt_backend/connection_pool.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 84d857ec..8563cc3e 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -42,7 +42,7 @@ struct YoutubeConnectionPool
 
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
-      conn = make_client(url, force_solve = true)
+      conn = make_client(url, force_resolve = true)
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn
     end

From ee89db49ba6242771921c9204d57f47f3edb8975 Mon Sep 17 00:00:00 2001
From: syeopite <70992037+syeopite@users.noreply.github.com>
Date: Sun, 16 Jun 2024 15:18:21 +0000
Subject: [PATCH 09/61] Typo

Co-authored-by: Samantaz Fox <coding@samantaz.fr>
---
 src/invidious/yt_backend/connection_pool.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 8563cc3e..f7227d67 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -42,7 +42,7 @@ struct YoutubeConnectionPool
 
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
-      conn = make_client(url, force_resolve = true)
+      conn = make_client(url, force_resolve: true)
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn
     end

From bd48af825c27f08987ee039381a702ca91e52cb8 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 16 Jun 2024 14:15:05 -0700
Subject: [PATCH 10/61] Search API: Fix named arg syntax to make_client

---
 src/invidious/routes/api/v1/search.cr       | 2 +-
 src/invidious/yt_backend/connection_pool.cr | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 6785ef73..59a30745 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Search
     query = env.params.query["q"]? || ""
 
     begin
-      client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers = true)
+      client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
       url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
 
       response = client.get(url).body
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index f7227d67..0dc42261 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -49,7 +49,7 @@ struct YoutubeConnectionPool
   end
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_header : Bool = false)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false)
   client = HTTP::Client.new(url)
 
   # Force the usage of a specific configured IP Family
@@ -57,7 +57,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
     client.family = CONFIG.force_resolve
   end
 
-  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_header
+  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 

From 7521902e88a4654378b5a0428f9c538d52dcb9db Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 24 Aug 2024 19:37:04 -0700
Subject: [PATCH 11/61] Ensure IP family is always used when force_resolve

---
 src/invidious/yt_backend/connection_pool.cr | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 0dc42261..eaa94158 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -31,7 +31,6 @@ struct YoutubeConnectionPool
     rescue ex
       conn.close
       conn = make_client(url)
-      conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       response = yield conn
     ensure
       pool.release(conn)
@@ -42,9 +41,7 @@ struct YoutubeConnectionPool
 
   private def build_pool
     DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
-      conn = make_client(url, force_resolve: true)
-      conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
-      conn
+      next make_client(url, force_resolve: true)
     end
   end
 end
@@ -55,6 +52,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
   # Force the usage of a specific configured IP Family
   if force_resolve
     client.family = CONFIG.force_resolve
+    client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
   end
 
   client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers

From 46c58bd84cf2a867b897338bb2105648aed0118c Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 24 Aug 2024 19:38:02 -0700
Subject: [PATCH 12/61] Pool: Use force_resolve in fallback new client

---
 src/invidious/yt_backend/connection_pool.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index eaa94158..d474d760 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -30,7 +30,7 @@ struct YoutubeConnectionPool
       response = yield conn
     rescue ex
       conn.close
-      conn = make_client(url)
+      conn = make_client(url, force_resolve: true)
       response = yield conn
     ensure
       pool.release(conn)

From 6e39b9b303930f931b5a5a60c75528c4d9db3587 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 24 Aug 2024 19:41:39 -0700
Subject: [PATCH 13/61] make_client: add YouTube headers on *.youtube.com

---
 src/invidious/yt_backend/connection_pool.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index d474d760..bff4df72 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -55,7 +55,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
     client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
   end
 
-  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers
+  client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 

From 480e073fa9be184b6839619c38795af582247c19 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Fri, 8 Dec 2023 18:20:17 -0800
Subject: [PATCH 14/61] Use HTTP pools for image requests to YouTube

---
 src/invidious.cr                            |  8 ++++++++
 src/invidious/routes/images.cr              | 12 +++++-------
 src/invidious/yt_backend/connection_pool.cr | 15 +++++++++++++++
 3 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/src/invidious.cr b/src/invidious.cr
index 3804197e..81db2c6c 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -92,6 +92,14 @@ SOFTWARE = {
 
 YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
 
+# Image request pool
+
+GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
+
+# Mapping of subdomain => YoutubeConnectionPool
+# This is needed as we may need to access arbitrary subdomains of ytimg
+YTIMG_POOLS = {} of String => YoutubeConnectionPool
+
 # CLI
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index b6a2e110..1964d597 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -32,7 +32,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+      GGPHT_POOL.client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
@@ -80,7 +80,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+      get_ytimg_pool(authority).client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
@@ -119,7 +119,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+      get_ytimg_pool("i9").client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
@@ -165,8 +165,7 @@ module Invidious::Routes::Images
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
         thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-        # This can likely be optimized into a (small) pool sometime in the future.
-        if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+        if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200
           name = thumb[:url] + ".jpg"
           break
         end
@@ -199,8 +198,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      # This can likely be optimized into a (small) pool sometime in the future.
-      HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+      get_ytimg_pool("i").client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index ca612083..26bf2773 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -77,3 +77,18 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
     client.close
   end
 end
+
+# Fetches a HTTP pool for the specified subdomain of ytimg.com
+#
+# Creates a new one when the specified pool for the subdomain does not exist
+def get_ytimg_pool(subdomain)
+  if pool = YTIMG_POOLS[subdomain]?
+    return pool
+  else
+    LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
+    pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
+    YTIMG_POOLS[subdomain] = pool
+
+    return pool
+  end
+end

From 52bc9aa328e44ff32bb1d7f2e05625e4080459c7 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Fri, 8 Dec 2023 18:42:40 -0800
Subject: [PATCH 15/61] Refactor duplicate logic in image routes

---
 src/invidious/routes/images.cr | 97 ++++++++--------------------------
 1 file changed, 21 insertions(+), 76 deletions(-)

diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 1964d597..7fdd33b0 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -11,29 +11,9 @@ module Invidious::Routes::Images
       end
     end
 
-    # We're encapsulating this into a proc in order to easily reuse this
-    # portion of the code for each request block below.
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300
-        env.response.headers.delete("Transfer-Encoding")
-        return
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
       GGPHT_POOL.client &.get(url) do |resp|
-        return request_proc.call(resp)
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -61,27 +41,9 @@ module Invidious::Routes::Images
       end
     end
 
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Connection"] = "close"
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300
-        return env.response.headers.delete("Transfer-Encoding")
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
       get_ytimg_pool(authority).client &.get(url) do |resp|
-        return request_proc.call(resp)
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -101,26 +63,9 @@ module Invidious::Routes::Images
       end
     end
 
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300 && response.status_code != 404
-        return env.response.headers.delete("Transfer-Encoding")
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
       get_ytimg_pool("i9").client &.get(url) do |resp|
-        return request_proc.call(resp)
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -180,28 +125,28 @@ module Invidious::Routes::Images
       end
     end
 
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300 && response.status_code != 404
-        return env.response.headers.delete("Transfer-Encoding")
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
       get_ytimg_pool("i").client &.get(url) do |resp|
-        return request_proc.call(resp)
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
   end
+
+  private def self.proxy_image(env, response)
+    env.response.status_code = response.status_code
+    response.headers.each do |key, value|
+      if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+        env.response.headers[key] = value
+      end
+    end
+
+    env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+    if response.status_code >= 300
+      return env.response.headers.delete("Transfer-Encoding")
+    end
+
+    return proxy_file(response, env)
+  end
 end

From 06e1a508e8dc5417a61a02ad1eb08e94fb24ae99 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Fri, 8 Dec 2023 18:52:11 -0800
Subject: [PATCH 16/61] Fix headers not being added in image requests

Regression from #2364
---
 src/invidious/routes/images.cr | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 7fdd33b0..c4197746 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -12,7 +12,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      GGPHT_POOL.client &.get(url) do |resp|
+      GGPHT_POOL.client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex
@@ -42,7 +42,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      get_ytimg_pool(authority).client &.get(url) do |resp|
+      get_ytimg_pool(authority).client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex
@@ -64,7 +64,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      get_ytimg_pool("i9").client &.get(url) do |resp|
+      get_ytimg_pool("i9").client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex
@@ -110,7 +110,7 @@ module Invidious::Routes::Images
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
         thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-        if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200
+        if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200
           name = thumb[:url] + ".jpg"
           break
         end
@@ -126,7 +126,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      get_ytimg_pool("i").client &.get(url) do |resp|
+      get_ytimg_pool("i").client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex

From 4bc77b81bf994336e324d84ab82a362b330c827d Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 23 Dec 2023 13:47:47 -0800
Subject: [PATCH 17/61] Move YTIMG_POOLS to connection_pool.cr

---
 src/invidious.cr                            |  4 ---
 src/invidious/yt_backend/connection_pool.cr | 32 ++++++++++++---------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/invidious.cr b/src/invidious.cr
index 81db2c6c..e0e72415 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -96,10 +96,6 @@ YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
 
 GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
 
-# Mapping of subdomain => YoutubeConnectionPool
-# This is needed as we may need to access arbitrary subdomains of ytimg
-YTIMG_POOLS = {} of String => YoutubeConnectionPool
-
 # CLI
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 26bf2773..646d0d1a 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -1,17 +1,6 @@
-def add_yt_headers(request)
-  request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
-  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
-
-  request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
-  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-  request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
-
-  # Preserve original cookies and add new YT consent cookie for EU servers
-  request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
-  if !CONFIG.cookies.empty?
-    request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
-  end
-end
+# Mapping of subdomain => YoutubeConnectionPool
+# This is needed as we may need to access arbitrary subdomains of ytimg
+private YTIMG_POOLS = {} of String => YoutubeConnectionPool
 
 struct YoutubeConnectionPool
   property! url : URI
@@ -54,6 +43,21 @@ struct YoutubeConnectionPool
   end
 end
 
+def add_yt_headers(request)
+  request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
+  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+
+  request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
+  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+  request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
+
+  # Preserve original cookies and add new YT consent cookie for EU servers
+  request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
+  if !CONFIG.cookies.empty?
+    request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+  end
+end
+
 def make_client(url : URI, region = nil, force_resolve : Bool = false)
   client = HTTP::Client.new(url)
 

From 003c6f81dcf6399d1fa808866d0806b915a713ee Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Mon, 8 Jan 2024 14:13:38 -0800
Subject: [PATCH 18/61] Preserve connection close header of get_storyboard

---
 src/invidious/routes/images.cr | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index c4197746..251258ec 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -41,9 +41,14 @@ module Invidious::Routes::Images
       end
     end
 
+    # A callable proc to be used inside #proxy_image
+    callable_proc = ->(env : HTTP::Server::Context) {
+      env.response.headers["Connection"] = "close"
+    }
+
     begin
       get_ytimg_pool(authority).client &.get(url, headers) do |resp|
-        return self.proxy_image(env, resp)
+        return self.proxy_image(env, resp, callable_proc: callable_proc)
       end
     rescue ex
     end
@@ -133,7 +138,7 @@ module Invidious::Routes::Images
     end
   end
 
-  private def self.proxy_image(env, response)
+  private def self.proxy_image(env, response, callable_proc = nil)
     env.response.status_code = response.status_code
     response.headers.each do |key, value|
       if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -143,6 +148,10 @@ module Invidious::Routes::Images
 
     env.response.headers["Access-Control-Allow-Origin"] = "*"
 
+    if callable_proc
+      callable_proc.call(env)
+    end
+
     if response.status_code >= 300
       return env.response.headers.delete("Transfer-Encoding")
     end

From 75b68618ab14a9f884ee7215a467bc510e8bd2c2 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Thu, 25 Apr 2024 13:28:58 -0700
Subject: [PATCH 19/61] Remove useless proc usage in images.cr

---
 src/invidious/routes/images.cr | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 251258ec..639697db 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -41,14 +41,10 @@ module Invidious::Routes::Images
       end
     end
 
-    # A callable proc to be used inside #proxy_image
-    callable_proc = ->(env : HTTP::Server::Context) {
-      env.response.headers["Connection"] = "close"
-    }
-
     begin
       get_ytimg_pool(authority).client &.get(url, headers) do |resp|
-        return self.proxy_image(env, resp, callable_proc: callable_proc)
+        env.response.headers["Connection"] = "close"
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -138,7 +134,7 @@ module Invidious::Routes::Images
     end
   end
 
-  private def self.proxy_image(env, response, callable_proc = nil)
+  private def self.proxy_image(env, response)
     env.response.status_code = response.status_code
     response.headers.each do |key, value|
       if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -148,10 +144,6 @@ module Invidious::Routes::Images
 
     env.response.headers["Access-Control-Allow-Origin"] = "*"
 
-    if callable_proc
-      callable_proc.call(env)
-    end
-
     if response.status_code >= 300
       return env.response.headers.delete("Transfer-Encoding")
     end

From f8ec3123286e63148123ccba781dcd699f705e1d Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Thu, 19 Sep 2024 21:24:20 -0300
Subject: [PATCH 20/61] Logger: Add color support for different log levels

---
 config/config.example.yml       |  9 +++++++++
 src/invidious.cr                |  5 ++++-
 src/invidious/config.cr         |  2 ++
 src/invidious/helpers/logger.cr | 19 +++++++++++++++++--
 4 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index 219aa03f..37b932ea 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -222,6 +222,15 @@ https_only: false
 ##
 #log_level: Info
 
+##
+## Enables colors in logs. Useful for debugging purposes
+## This is overridden if "-k" or "--colorize" 
+## are passed on the command line.
+##
+## Accepted values: true, false
+## Default: false
+##
+#colorize_logs: false
 
 # -----------------------------
 #  Features
diff --git a/src/invidious.cr b/src/invidious.cr
index 3804197e..d9a479d1 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -117,6 +117,9 @@ Kemal.config.extra_options do |parser|
   parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level|
     CONFIG.log_level = LogLevel.parse(log_level)
   end
+  parser.on("-k", "--colorize", "Colorize logs") do
+    CONFIG.colorize_logs = true
+  end
   parser.on("-v", "--version", "Print version") do
     puts SOFTWARE.to_pretty_json
     exit
@@ -133,7 +136,7 @@ if CONFIG.output.upcase != "STDOUT"
   FileUtils.mkdir_p(File.dirname(CONFIG.output))
 end
 OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a")
-LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
+LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs)
 
 # Check table integrity
 Invidious::Database.check_integrity(CONFIG)
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index c4ddcdb3..d8543d35 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -68,6 +68,8 @@ class Config
   property output : String = "STDOUT"
   # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
   property log_level : LogLevel = LogLevel::Info
+  # Enables colors in logs. Useful for debugging purposes
+  property colorize_logs : Bool = false
   # Database configuration with separate parameters (username, hostname, etc)
   property db : DBConfig? = nil
 
diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr
index b443073e..36a3a7f9 100644
--- a/src/invidious/helpers/logger.cr
+++ b/src/invidious/helpers/logger.cr
@@ -1,3 +1,5 @@
+require "colorize"
+
 enum LogLevel
   All   = 0
   Trace = 1
@@ -10,7 +12,7 @@ enum LogLevel
 end
 
 class Invidious::LogHandler < Kemal::BaseLogHandler
-  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug)
+  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @color : Bool = true)
   end
 
   def call(context : HTTP::Server::Context)
@@ -39,10 +41,23 @@ class Invidious::LogHandler < Kemal::BaseLogHandler
     @io.flush
   end
 
+  def color(level)
+    case level
+    when LogLevel::Trace then :cyan
+    when LogLevel::Debug then :green
+    when LogLevel::Info  then :white
+    when LogLevel::Warn  then :yellow
+    when LogLevel::Error then :red
+    when LogLevel::Fatal then :magenta
+    else                      :default
+    end
+  end
+
   {% for level in %w(trace debug info warn error fatal) %}
     def {{level.id}}(message : String)
       if LogLevel::{{level.id.capitalize}} >= @level
-        puts("#{Time.utc} [{{level.id}}] #{message}")
+        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@color))
+
       end
     end
   {% end %}

From d77afdcf00f55a4455fb84dd90c4e5773167b759 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Fri, 20 Sep 2024 00:32:27 -0300
Subject: [PATCH 21/61] Logger: Make colorize_logs true by default

---
 config/config.example.yml       | 6 ++++--
 src/invidious/config.cr         | 2 +-
 src/invidious/helpers/logger.cr | 5 ++---
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index 37b932ea..fefc28be 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -226,11 +226,13 @@ https_only: false
 ## Enables colors in logs. Useful for debugging purposes
 ## This is overridden if "-k" or "--colorize" 
 ## are passed on the command line.
+## Colors are also disabled if the environment variable
+## NO_COLOR is present and has any value
 ##
 ## Accepted values: true, false
-## Default: false
+## Default: true
 ##
-#colorize_logs: false
+#colorize_logs: true
 
 # -----------------------------
 #  Features
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index d8543d35..054f8db7 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -69,7 +69,7 @@ class Config
   # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
   property log_level : LogLevel = LogLevel::Info
   # Enables colors in logs. Useful for debugging purposes
-  property colorize_logs : Bool = false
+  property colorize_logs : Bool = true
   # Database configuration with separate parameters (username, hostname, etc)
   property db : DBConfig? = nil
 
diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr
index 36a3a7f9..3c425ff4 100644
--- a/src/invidious/helpers/logger.cr
+++ b/src/invidious/helpers/logger.cr
@@ -12,7 +12,7 @@ enum LogLevel
 end
 
 class Invidious::LogHandler < Kemal::BaseLogHandler
-  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @color : Bool = true)
+  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @use_color : Bool = true)
   end
 
   def call(context : HTTP::Server::Context)
@@ -56,8 +56,7 @@ class Invidious::LogHandler < Kemal::BaseLogHandler
   {% for level in %w(trace debug info warn error fatal) %}
     def {{level.id}}(message : String)
       if LogLevel::{{level.id.capitalize}} >= @level
-        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@color))
-
+        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@use_color))
       end
     end
   {% end %}

From b2a83991d16cc9fa65f71309cd4a745f005cdf61 Mon Sep 17 00:00:00 2001
From: absidue <48293849+absidue@users.noreply.github.com>
Date: Fri, 20 Sep 2024 18:46:00 +0200
Subject: [PATCH 22/61] Fix parsing live_now and premiere_timestamp

---
 src/invidious/videos/parser.cr | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index 811a0a03..c28ba634 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -224,8 +224,17 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
     .try { |t| Time.parse_rfc3339(t.as_s) }
 
+  premiere_timestamp ||= player_response.dig?(
+    "playabilityStatus", "liveStreamability",
+    "liveStreamabilityRenderer", "offlineSlate",
+    "liveStreamOfflineSlateRenderer", "scheduledStartTime"
+  )
+    .try &.as_s.to_i64
+      .try { |t| Time.unix(t) }
+
   live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
-    .try &.as_bool || false
+    .try &.as_bool
+  live_now ||= video_details.dig?("isLive").try &.as_bool || false
 
   post_live_dvr = video_details.dig?("isPostLiveDvr")
     .try &.as_bool || false

From 17b525f2a66f6e832ccdc74522feebe68f73d9de Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Fri, 27 Sep 2024 18:08:21 -0300
Subject: [PATCH 23/61] Logger: colorize_logs false by default

---
 config/config.example.yml | 2 +-
 src/invidious/config.cr   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index fefc28be..d79622ad 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -232,7 +232,7 @@ https_only: false
 ## Accepted values: true, false
 ## Default: true
 ##
-#colorize_logs: true
+#colorize_logs: false
 
 # -----------------------------
 #  Features
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 054f8db7..d8543d35 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -69,7 +69,7 @@ class Config
   # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
   property log_level : LogLevel = LogLevel::Info
   # Enables colors in logs. Useful for debugging purposes
-  property colorize_logs : Bool = true
+  property colorize_logs : Bool = false
   # Database configuration with separate parameters (username, hostname, etc)
   property db : DBConfig? = nil
 

From d2edd4b63fe690c248ff8709b39098fcdad0e109 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Tue, 8 Oct 2024 18:36:50 -0300
Subject: [PATCH 24/61] fixup! Logger: Add color support for different log
 levels

---
 config/config.example.yml       | 2 +-
 src/invidious/helpers/logger.cr | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index d79622ad..f746d1f7 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -224,7 +224,7 @@ https_only: false
 
 ##
 ## Enables colors in logs. Useful for debugging purposes
-## This is overridden if "-k" or "--colorize" 
+## This is overridden if "-k" or "--colorize"
 ## are passed on the command line.
 ## Colors are also disabled if the environment variable
 ## NO_COLOR is present and has any value
diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr
index 3c425ff4..03349595 100644
--- a/src/invidious/helpers/logger.cr
+++ b/src/invidious/helpers/logger.cr
@@ -12,7 +12,9 @@ enum LogLevel
 end
 
 class Invidious::LogHandler < Kemal::BaseLogHandler
-  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @use_color : Bool = true)
+  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true)
+    Colorize.enabled = use_color
+    Colorize.on_tty_only!
   end
 
   def call(context : HTTP::Server::Context)
@@ -56,7 +58,7 @@ class Invidious::LogHandler < Kemal::BaseLogHandler
   {% for level in %w(trace debug info warn error fatal) %}
     def {{level.id}}(message : String)
       if LogLevel::{{level.id.capitalize}} >= @level
-        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@use_color))
+        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})))
       end
     end
   {% end %}

From 84e4746265d6077b1537b626c9742498f9cb253c Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Wed, 18 Sep 2024 18:14:28 -0300
Subject: [PATCH 25/61] SigHelper: Reconnect to signature helper

Signed-off-by: Fijxu <fijxu@nadeko.net>
---
 src/invidious/helpers/sig_helper.cr | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr
index 9e72c1c7..6d198a42 100644
--- a/src/invidious/helpers/sig_helper.cr
+++ b/src/invidious/helpers/sig_helper.cr
@@ -175,8 +175,9 @@ module Invidious::SigHelper
     @queue = {} of TransactionID => Transaction
 
     @conn : Connection
+    @uri_or_path : String
 
-    def initialize(uri_or_path)
+    def initialize(@uri_or_path)
       @conn = Connection.new(uri_or_path)
       listen
     end
@@ -186,10 +187,26 @@ module Invidious::SigHelper
 
       LOGGER.debug("SigHelper: Multiplexor listening")
 
-      # TODO: reopen socket if unexpectedly closed
       spawn do
         loop do
-          receive_data
+          begin
+            receive_data
+          rescue ex
+            LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")
+            # We close the socket because for some reason is not closed.
+            @conn.close
+            loop do
+              begin
+                @conn = Connection.new(@uri_or_path)
+                LOGGER.info("SigHelper: Reconnected to SigHelper!")
+              rescue ex
+                LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
+                sleep 500.milliseconds
+                next
+              end
+              break if !@conn.closed?
+            end
+          end
           Fiber.yield
         end
       end

From f51a3b8d2b52f83057d0b3be5686149984e66ada Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Wed, 9 Oct 2024 18:07:04 +0200
Subject: [PATCH 26/61] Makefile: Add MT option to enable the 'preview_mt' flag

---
 Makefile | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Makefile b/Makefile
index 9eb195df..ec22a0de 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,11 @@ STATIC   := 0
 
 NO_DBG_SYMBOLS := 0
 
+# Enable multi-threading.
+# Warning: Experimental feature!!
+# invidious is not stable when MT is enabled.
+MT := 0
+
 
 FLAGS ?=
 
@@ -19,6 +24,10 @@ ifeq ($(STATIC), 1)
   FLAGS += --static
 endif
 
+ifeq ($(MT), 1)
+  FLAGS += -Dpreview_mt
+endif
+
 
 ifeq ($(NO_DBG_SYMBOLS), 1)
   FLAGS += --no-debug

From 952b3625a0a8fb21ab04bc267f94a21c331109f6 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 10 Oct 2024 20:31:22 +0200
Subject: [PATCH 27/61] Add "Filipino (auto-generated)" to the list of caption
 languages

---
 locales/en-US.json              | 1 +
 src/invidious/videos/caption.cr | 1 +
 2 files changed, 2 insertions(+)

diff --git a/locales/en-US.json b/locales/en-US.json
index 7827d9c6..c23f6bc3 100644
--- a/locales/en-US.json
+++ b/locales/en-US.json
@@ -286,6 +286,7 @@
     "Esperanto": "Esperanto",
     "Estonian": "Estonian",
     "Filipino": "Filipino",
+    "Filipino (auto-generated)": "Filipino (auto-generated)",
     "Finnish": "Finnish",
     "French": "French",
     "French (auto-generated)": "French (auto-generated)",
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
index 484e61d2..c811cfe1 100644
--- a/src/invidious/videos/caption.cr
+++ b/src/invidious/videos/caption.cr
@@ -123,6 +123,7 @@ module Invidious::Videos
       "Esperanto",
       "Estonian",
       "Filipino",
+      "Filipino (auto-generated)",
       "Finnish",
       "French",
       "French (auto-generated)",

From ee728092823d8e82f71f35c31da8a27efec0f1b5 Mon Sep 17 00:00:00 2001
From: Brahim Hadriche <brahim.hadriche@gmail.com>
Date: Sat, 26 Oct 2024 12:40:31 -0400
Subject: [PATCH 28/61] [Alternative] Fix for channel live videos

---
 src/invidious/channels/videos.cr | 66 +++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 19 deletions(-)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index 6cc30142..e29d80ed 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -23,29 +23,57 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene
     else                    15 # Fallback to "videos"
     end
 
-  sort_by_numerical =
-    case sort_by
-    when "newest"  then 1_i64
-    when "popular" then 2_i64
-    when "oldest"  then 4_i64
-    else                1_i64 # Fallback to "newest"
-    end
+  if content_type == "livestreams"
+    sort_by_numerical =
+      case sort_by
+      when "newest"  then 12_i64
+      when "popular" then 14_i64
+      when "oldest"  then 13_i64
+      else                12_i64 # Fallback to "newest"
+      end
+  else
+    sort_by_numerical =
+      case sort_by
+      when "newest"  then 1_i64
+      when "popular" then 2_i64
+      when "oldest"  then 4_i64
+      else                1_i64 # Fallback to "newest"
+      end
+  end
 
-  object_inner_1 = {
-    "110:embedded" => {
-      "3:embedded" => {
-        "#{content_type_numerical}:embedded" => {
-          "1:embedded" => {
-            "1:string" => object_inner_2_encoded,
+  if content_type == "livestreams"
+    object_inner_1 = {
+      "110:embedded" => {
+        "3:embedded" => {
+          "#{content_type_numerical}:embedded" => {
+            "1:embedded" => {
+              "1:string" => object_inner_2_encoded,
+            },
+            "2:embedded" => {
+              "1:string" => "00000000-0000-0000-0000-000000000000",
+            },
+            "5:varint" => sort_by_numerical,
           },
-          "2:embedded" => {
-            "1:string" => "00000000-0000-0000-0000-000000000000",
-          },
-          "3:varint" => sort_by_numerical,
         },
       },
-    },
-  }
+    }
+  else
+    object_inner_1 = {
+      "110:embedded" => {
+        "3:embedded" => {
+          "#{content_type_numerical}:embedded" => {
+            "1:embedded" => {
+              "1:string" => object_inner_2_encoded,
+            },
+            "2:embedded" => {
+              "1:string" => "00000000-0000-0000-0000-000000000000",
+            },
+            "3:varint" => sort_by_numerical,
+          },
+        },
+      },
+    }
+  end
 
   object_inner_1_encoded = object_inner_1
     .try { |i| Protodec::Any.cast_json(i) }

From 711d52d47fcff4cc376551a81fba47dcaeb23e0c Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Tue, 29 Oct 2024 17:26:24 +0100
Subject: [PATCH 29/61] Shards: Update database dependencies

---
 shard.lock                                  | 6 +++---
 shard.yml                                   | 4 ++--
 src/invidious/yt_backend/connection_pool.cr | 9 ++++++++-
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/shard.lock b/shard.lock
index 397bd8bc..1f609631 100644
--- a/shard.lock
+++ b/shard.lock
@@ -14,7 +14,7 @@ shards:
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
-    version: 0.10.1
+    version: 0.13.1
 
   exception_page:
     git: https://github.com/crystal-loot/exception_page.git
@@ -30,7 +30,7 @@ shards:
 
   pg:
     git: https://github.com/will/crystal-pg.git
-    version: 0.24.0
+    version: 0.28.0
 
   protodec:
     git: https://github.com/iv-org/protodec.git
@@ -46,5 +46,5 @@ shards:
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
-    version: 0.18.0
+    version: 0.21.0
 
diff --git a/shard.yml b/shard.yml
index 367f7c73..e0e34c0c 100644
--- a/shard.yml
+++ b/shard.yml
@@ -12,10 +12,10 @@ targets:
 dependencies:
   pg:
     github: will/crystal-pg
-    version: ~> 0.24.0
+    version: ~> 0.28.0
   sqlite3:
     github: crystal-lang/crystal-sqlite3
-    version: ~> 0.18.0
+    version: ~> 0.21.0
   kemal:
     github: kemalcr/kemal
     version: ~> 1.1.2
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index ca612083..14cc2d47 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -44,7 +44,14 @@ struct YoutubeConnectionPool
   end
 
   private def build_pool
-    DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
+    options = DB::Pool::Options.new(
+      initial_pool_size: 0,
+      max_pool_size: capacity,
+      max_idle_pool_size: capacity,
+      checkout_timeout: timeout
+    )
+
+    DB::Pool(HTTP::Client).new(options) do
       conn = HTTP::Client.new(url)
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC

From 2e3a7ad044b3e37d15d0c87bb33cb85d2d04424f Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Wed, 30 Oct 2024 17:13:00 +0100
Subject: [PATCH 30/61] Update CHANGELOG.md

---
 CHANGELOG.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 15991668..f9892e17 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@
 
 ### Full list of pull requests merged since the last release (newest first)
 
+* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox)
+* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox)
+* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu)
+* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone)
+* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite)
+* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite)
 * Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox)
 * Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite)
 * Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov)
@@ -22,7 +28,10 @@
 
 [#4122]: https://github.com/iv-org/invidious/pull/4122
 [#4193]: https://github.com/iv-org/invidious/pull/4193
+[#4270]: https://github.com/iv-org/invidious/pull/4270
+[#4326]: https://github.com/iv-org/invidious/pull/4326
 [#4652]: https://github.com/iv-org/invidious/pull/4652
+[#4750]: https://github.com/iv-org/invidious/pull/4750
 [#4850]: https://github.com/iv-org/invidious/pull/4850
 [#4862]: https://github.com/iv-org/invidious/pull/4862
 [#4863]: https://github.com/iv-org/invidious/pull/4863
@@ -33,6 +42,9 @@
 [#4928]: https://github.com/iv-org/invidious/pull/4928
 [#4930]: https://github.com/iv-org/invidious/pull/4930
 [#4942]: https://github.com/iv-org/invidious/pull/4942
+[#4991]: https://github.com/iv-org/invidious/pull/4991
+[#4993]: https://github.com/iv-org/invidious/pull/4993
+[#4995]: https://github.com/iv-org/invidious/pull/4995
 
 
 ## v2.20240825.2 (2024-08-26)

From c243d08afb8509f7a98cd7aa1b77d4f409a7a823 Mon Sep 17 00:00:00 2001
From: Brahim Hadriche <brahim.hadriche@gmail.com>
Date: Wed, 30 Oct 2024 13:38:13 -0400
Subject: [PATCH 31/61] refactor

---
 src/invidious/channels/videos.cr | 49 +++++++++++++-------------------
 1 file changed, 19 insertions(+), 30 deletions(-)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index e29d80ed..bcdc8d8f 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -23,6 +23,13 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene
     else                    15 # Fallback to "videos"
     end
 
+  sort_type_numerical =
+    case content_type
+    when "videos"      then 3
+    when "livestreams" then 5
+    else                    3 # Fallback to "videos"
+    end
+
   if content_type == "livestreams"
     sort_by_numerical =
       case sort_by
@@ -41,39 +48,21 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene
       end
   end
 
-  if content_type == "livestreams"
-    object_inner_1 = {
-      "110:embedded" => {
-        "3:embedded" => {
-          "#{content_type_numerical}:embedded" => {
-            "1:embedded" => {
-              "1:string" => object_inner_2_encoded,
-            },
-            "2:embedded" => {
-              "1:string" => "00000000-0000-0000-0000-000000000000",
-            },
-            "5:varint" => sort_by_numerical,
+  object_inner_1 = {
+    "110:embedded" => {
+      "3:embedded" => {
+        "#{content_type_numerical}:embedded" => {
+          "1:embedded" => {
+            "1:string" => object_inner_2_encoded,
           },
+          "2:embedded" => {
+            "1:string" => "00000000-0000-0000-0000-000000000000",
+          },
+          "#{sort_type_numerical}:varint" => sort_by_numerical,
         },
       },
-    }
-  else
-    object_inner_1 = {
-      "110:embedded" => {
-        "3:embedded" => {
-          "#{content_type_numerical}:embedded" => {
-            "1:embedded" => {
-              "1:string" => object_inner_2_encoded,
-            },
-            "2:embedded" => {
-              "1:string" => "00000000-0000-0000-0000-000000000000",
-            },
-            "3:varint" => sort_by_numerical,
-          },
-        },
-      },
-    }
-  end
+    },
+  }
 
   object_inner_1_encoded = object_inner_1
     .try { |i| Protodec::Any.cast_json(i) }

From cdf93b29e6376ea0c023da825aeb9d83ec588873 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 31 Oct 2024 11:51:33 +0100
Subject: [PATCH 32/61] Routing: Remove deprecated /api/v1/channels/.../:ucid
 routes

---
 src/invidious/routing.cr | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index ba05da19..9f76f15f 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -243,17 +243,16 @@ module Invidious::Routing
 
       # Channels
       get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+      get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
+      get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
       get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
       get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
       get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
       get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
-
+      get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
+      get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
       get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
-
-      {% for route in {"videos", "latest", "playlists", "community", "search"} %}
-        get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
-        get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
-      {% end %}
+      get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
 
       # Posts
       get "/api/v1/post/:id", {{namespace}}::Channels, :post

From 6da18ddc41a20cad06c736a28aef6064433e3bd5 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 31 Oct 2024 11:52:09 +0100
Subject: [PATCH 33/61] Routing: Also remove outdated comment about
 notification routes

---
 src/invidious/routing.cr | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 9f76f15f..9009062f 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -270,11 +270,6 @@ module Invidious::Routing
 
       # Authenticated
 
-      # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
-      #
-      # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-      # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-
       get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
       post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
 

From 75c5881c553b8225f389db11733639ae62885c2f Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 31 Oct 2024 13:31:59 +0100
Subject: [PATCH 34/61] Locales: Add Bulgarian, Welsh and Lombard to the list

---
 src/invidious/helpers/i18n.cr | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr
index 23a1aafc..1ba3ea61 100644
--- a/src/invidious/helpers/i18n.cr
+++ b/src/invidious/helpers/i18n.cr
@@ -1,8 +1,22 @@
+# Languages requiring a better level of translation (at least 20%)
+# to be added to the list below:
+#
+#  "af"      => "", # Afrikaans
+#  "az"      => "", # Azerbaijani
+#  "be"      => "", # Belarusian
+#  "bn_BD"   => "", # Bengali (Bangladesh)
+#  "ia"      => "", # Interlingua
+#  "or"      => "", # Odia
+#  "tk"      => "", # Turkmen
+#  "tok      => "", # Toki Pona
+#
 LOCALES_LIST = {
   "ar"      => "العربية",               # Arabic
+  "bg"      => "български",             # Bulgarian
   "bn"      => "বাংলা",                 # Bengali
   "ca"      => "Català",                # Catalan
   "cs"      => "Čeština",               # Czech
+  "cy"      => "Cymraeg",               # Welsh
   "da"      => "Dansk",                 # Danish
   "de"      => "Deutsch",               # German
   "el"      => "Ελληνικά",              # Greek
@@ -23,6 +37,7 @@ LOCALES_LIST = {
   "it"      => "Italiano",              # Italian
   "ja"      => "日本語",                   # Japanese
   "ko"      => "한국어",                   # Korean
+  "lmo"     => "Lombard",               # Lombard
   "lt"      => "Lietuvių",              # Lithuanian
   "nb-NO"   => "Norsk bokmål",          # Norwegian Bokmål
   "nl"      => "Nederlands",            # Dutch

From ac6e796c732bb4be5a0fe6be9ba53ad49c49bd51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89milien=20=28perso=29?=
 <4016501+unixfox@users.noreply.github.com>
Date: Thu, 7 Nov 2024 14:04:43 +0100
Subject: [PATCH 35/61] checking the status code returned by youtube (#5052)

* checking the status code returned by youtube

* add documentation link

* Update src/invidious/yt_backend/youtube_api.cr

Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com>

---------

Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com>
---
 src/invidious/yt_backend/youtube_api.cr | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index baa3cd92..e0a3181f 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -638,6 +638,11 @@ module YoutubeAPI
     # Send the POST request
     body = YT_POOL.client() do |client|
       client.post(url, headers: headers, body: data.to_json) do |response|
+        if response.status_code != 200
+          raise InfoException.new("Error: non 200 status code. Youtube API returned \
+            status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
+            https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
+        end
         self._decompress(response.body_io, response.headers["Content-Encoding"]?)
       end
     end

From 792d0d5f6df912039a58768e6ff503ae00abe7c0 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 7 Nov 2024 13:14:36 +0000
Subject: [PATCH 36/61] CI: Check Crystal lint only on latest version (#5042)

* CI: Check Crystal lint only on latest version

* Apply suggestion from code review

Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com>

---------

Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com>
---
 .github/workflows/ci.yml | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 411ec769..dd472d1a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,7 +54,7 @@ jobs:
 
       - name: Install required APT packages
         run: |
-          sudo apt install -y libsqlite3-dev 
+          sudo apt install -y libsqlite3-dev
         shell: bash
 
       - name: Install Crystal
@@ -65,7 +65,9 @@ jobs:
       - name: Cache Shards
         uses: actions/cache@v3
         with:
-          path: ./lib
+          path: |
+            ./lib
+            ./bin
           key: shards-${{ hashFiles('shard.lock') }}
 
       - name: Install Shards
@@ -77,14 +79,6 @@ jobs:
       - name: Run tests
         run: crystal spec
 
-      - name: Run lint
-        run: |
-          if ! crystal tool format --check; then
-            crystal tool format
-            git diff
-            exit 1
-          fi
-
       - name: Build
         run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr
 
@@ -130,8 +124,12 @@ jobs:
       - name: Test Docker
         run: while curl -Isf http://localhost:3000; do sleep 1; done
 
-  ameba_lint:
+  lint:
+
     runs-on: ubuntu-latest
+
+    continue-on-error: true
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -151,7 +149,18 @@ jobs:
           key: shards-${{ hashFiles('shard.lock') }}
 
       - name: Install Shards
-        run: shards install
+        run: |
+          if ! shards check; then
+            shards install
+          fi
+
+      - name: Check Crystal formatter compliance
+        run: |
+          if ! crystal tool format --check; then
+            crystal tool format
+            git diff
+            exit 1
+          fi
 
       - name: Run Ameba linter
         run: bin/ameba

From cbc546f0320e4833927a654c26d384bb2e8a9f93 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 7 Nov 2024 22:54:21 +0100
Subject: [PATCH 37/61] Channels: Add function to generate the new ctoken
 objects

---
 src/invidious/channels/videos.cr | 207 +++++++++++++++----------------
 1 file changed, 103 insertions(+), 104 deletions(-)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index bcdc8d8f..7b3e3cfa 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -1,95 +1,3 @@
-def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
-  object_inner_2 = {
-    "2:0:embedded" => {
-      "1:0:varint" => 0_i64,
-    },
-    "5:varint"  => 50_i64,
-    "6:varint"  => 1_i64,
-    "7:varint"  => (page * 30).to_i64,
-    "9:varint"  => 1_i64,
-    "10:varint" => 0_i64,
-  }
-
-  object_inner_2_encoded = object_inner_2
-    .try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  content_type_numerical =
-    case content_type
-    when "videos"      then 15
-    when "livestreams" then 14
-    else                    15 # Fallback to "videos"
-    end
-
-  sort_type_numerical =
-    case content_type
-    when "videos"      then 3
-    when "livestreams" then 5
-    else                    3 # Fallback to "videos"
-    end
-
-  if content_type == "livestreams"
-    sort_by_numerical =
-      case sort_by
-      when "newest"  then 12_i64
-      when "popular" then 14_i64
-      when "oldest"  then 13_i64
-      else                12_i64 # Fallback to "newest"
-      end
-  else
-    sort_by_numerical =
-      case sort_by
-      when "newest"  then 1_i64
-      when "popular" then 2_i64
-      when "oldest"  then 4_i64
-      else                1_i64 # Fallback to "newest"
-      end
-  end
-
-  object_inner_1 = {
-    "110:embedded" => {
-      "3:embedded" => {
-        "#{content_type_numerical}:embedded" => {
-          "1:embedded" => {
-            "1:string" => object_inner_2_encoded,
-          },
-          "2:embedded" => {
-            "1:string" => "00000000-0000-0000-0000-000000000000",
-          },
-          "#{sort_type_numerical}:varint" => sort_by_numerical,
-        },
-      },
-    },
-  }
-
-  object_inner_1_encoded = object_inner_1
-    .try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  object = {
-    "80226972:embedded" => {
-      "2:string"  => ucid,
-      "3:string"  => object_inner_1_encoded,
-      "35:string" => "browse-feed#{ucid}videos102",
-    },
-  }
-
-  continuation = object.try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  return continuation
-end
-
-def make_initial_content_ctoken(ucid, content_type, sort_by) : String
-  return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
-end
-
 module Invidious::Channel::Tabs
   extend self
 
@@ -118,7 +26,7 @@ module Invidious::Channel::Tabs
   end
 
   def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
+    continuation ||= make_videos_ctoken(ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, author, ucid)
@@ -147,14 +55,10 @@ module Invidious::Channel::Tabs
   #  Shorts
   # -------------------
 
-  def get_shorts(channel : AboutChannel, continuation : String? = nil)
-    if continuation.nil?
-      # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
-      # TODO: try to extract the continuation tokens that allows other sorting options
-      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
-    else
-      initial_data = YoutubeAPI.browse(continuation: continuation)
-    end
+  def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+    continuation ||= make_shorts_ctoken(channel.ucid, sort_by)
+    initial_data = YoutubeAPI.browse(continuation: continuation)
+
     return extract_items(initial_data, channel.author, channel.ucid)
   end
 
@@ -162,9 +66,8 @@ module Invidious::Channel::Tabs
   #  Livestreams
   # -------------------
 
-  def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
-
+  def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+    continuation ||= make_livestreams_ctoken(channel.ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, channel.author, channel.ucid)
@@ -188,4 +91,100 @@ module Invidious::Channel::Tabs
 
     return items, next_continuation
   end
+
+  # -------------------
+  #  C-tokens
+  # -------------------
+
+  private def sort_options_videos_short(sort_by : String)
+    case sort_by
+    when "newest"  then return 4_i64
+    when "popular" then return 2_i64
+    when "oldest"  then return 5_i64
+    else                return 4_i64 # Fallback to "newest"
+    end
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "videos" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_videos_ctoken(ucid : String, sort_by = "newest")
+    object = {
+      "15:embedded" => {
+        "2:string" => "\n$00000000-0000-0000-0000-000000000000",
+        "4:varint" => sort_options_videos_short(sort_by),
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "shorts" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_shorts_ctoken(ucid : String, sort_by = "newest")
+    object = {
+      "10:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "4:varint" => sort_options_videos_short(sort_by),
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "livestreams" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_livestreams_ctoken(ucid : String, sort_by = "newest")
+    sort_by_numerical =
+      case sort_by
+      when "newest"  then 12_i64
+      when "popular" then 14_i64
+      when "oldest"  then 13_i64
+      else                12_i64 # Fallback to "newest"
+      end
+
+    object = {
+      "14:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "5:varint" => sort_by_numerical,
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # The protobuf structure common between videos/shorts/livestreams
+  private def channel_ctoken_wrap(ucid : String, object)
+    object_inner = {
+      "110:embedded" => {
+        "3:embedded" => object,
+      },
+    }
+
+    object_inner_encoded = object_inner
+      .try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    object = {
+      "80226972:embedded" => {
+        "2:string" => ucid,
+        "3:string" => object_inner_encoded,
+      },
+    }
+
+    continuation = object.try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    return continuation
+  end
 end

From 82248fad024de5289011e2ae26d5c390d5084827 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 7 Nov 2024 23:00:18 +0100
Subject: [PATCH 38/61] Channels: Add sort options to shorts

---
 src/invidious/routes/channels.cr | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 952098e0..d4e9fa68 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -82,13 +82,12 @@ module Invidious::Routes::Channels
       end
       next_continuation = nil
     else
-      # TODO: support sort option for shorts
-      sort_by = ""
-      sort_options = [] of String
+      sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+      sort_options = {"newest", "oldest", "popular"}
 
       # Fetch items and continuation token
       items, next_continuation = Channel::Tabs.get_shorts(
-        channel, continuation: continuation
+        channel, continuation: continuation, sort_by: sort_by
       )
     end
 

From 3196182d4d303520d8d49b915a70bdc0c8fb12cd Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Thu, 7 Nov 2024 20:41:04 -0800
Subject: [PATCH 39/61] Prevent PRs from being considered stale

---
 .github/workflows/stale.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 16d3269b..35740d60 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -14,13 +14,10 @@ jobs:
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 365
-        days-before-pr-stale: 90 
+        days-before-pr-stale: -1 
         days-before-close: 30
-        exempt-pr-labels: blocked,exempt-stale
         stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.'
-        stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.'
         stale-issue-label: "stale"
-        stale-pr-label: "stale"
         ascending: true
         # Never mark feature requests/enhancements as stale
         exempt-issue-labels: "feature-request,enhancement,exempt-stale"

From 78f18b257c7099d1d22b7566b62d5c2d596fa27f Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Thu, 7 Nov 2024 20:42:19 -0800
Subject: [PATCH 40/61] Double stale timer for issues

Days before staling is increased to 730 days
Days before closing is increased to 60 days
---
 .github/workflows/stale.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 35740d60..1294804a 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -13,9 +13,9 @@ jobs:
     - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 365
+        days-before-stale: 730
         days-before-pr-stale: -1 
-        days-before-close: 30
+        days-before-close: 60
         stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.'
         stale-issue-label: "stale"
         ascending: true

From ce910b5269c57658ef44d46e4024dc623f5ed46d Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Thu, 7 Nov 2024 20:45:23 -0800
Subject: [PATCH 41/61] Prevent discussion issues from being staled

---
 .github/workflows/stale.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 1294804a..498a2c1b 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -19,5 +19,5 @@ jobs:
         stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.'
         stale-issue-label: "stale"
         ascending: true
-        # Never mark feature requests/enhancements as stale
-        exempt-issue-labels: "feature-request,enhancement,exempt-stale"
+        # Exempt the following types of issues from being staled
+        exempt-issue-labels: "feature-request,enhancement,discussion,exempt-stale"

From 1a5047aad94454fd8a8d9623e17ee3782c68c3d0 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 12:33:14 +0100
Subject: [PATCH 42/61] Extractors: Add support for lockupViewModel

The 'lockupViewModel' structure is used in the channel "podcasts" tab
---
 src/invidious/yt_backend/extractors.cr | 76 +++++++++++++++++++++++++-
 1 file changed, 73 insertions(+), 3 deletions(-)

diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index 4074de86..cb8331a5 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -467,9 +467,9 @@ private module Parsers
   # Parses an InnerTube richItemRenderer into a SearchVideo.
   # Returns nil when the given object isn't a RichItemRenderer
   #
-  # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
-  # by the result page for hashtags and for the podcast tab on channels.
-  # It is located inside a continuationItems container for hashtags.
+  # A richItemRenderer seems to be a simple wrapper for a various other types,
+  # used on the hashtags result page and the channel podcast tab. It is located
+  # itself inside a richGridRenderer container.
   #
   module RichItemRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
@@ -482,6 +482,7 @@ private module Parsers
       child = VideoRendererParser.process(item_contents, author_fallback)
       child ||= ReelItemRendererParser.process(item_contents, author_fallback)
       child ||= PlaylistRendererParser.process(item_contents, author_fallback)
+      child ||= LockupViewModelParser.process(item_contents, author_fallback)
       return child
     end
 
@@ -582,6 +583,75 @@ private module Parsers
     end
   end
 
+  # Parses an InnerTube lockupViewModel into a SearchPlaylist.
+  # Returns nil when the given object is not a lockupViewModel.
+  #
+  # This structure is present since November 2024 on the "podcasts" tab of the
+  # channel page. It is usually (always?) encapsulated in a richItemRenderer.
+  #
+  module LockupViewModelParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["lockupViewModel"]?
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      playlist_id = item_contents["contentId"].as_s
+
+      thumbnail_view_model = item_contents.dig(
+        "contentImage", "collectionThumbnailViewModel",
+        "primaryThumbnail", "thumbnailViewModel"
+      )
+
+      thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s
+
+      # This complicated sequences tries to extract the following data structure:
+      # "overlays": [{
+      #   "thumbnailOverlayBadgeViewModel": {
+      #     "thumbnailBadges": [{
+      #       "thumbnailBadgeViewModel": {
+      #         "text": "430 episodes",
+      #         "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT"
+      #       }
+      #     }]
+      #   }
+      # }]
+      video_count = thumbnail_view_model.dig("overlays").as_a
+        .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
+        .flatten
+        .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes"))
+        .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
+
+      metadata = item_contents.dig("metadata", "lockupMetadataViewModel")
+      title = metadata.dig("title", "content").as_s
+
+      # TODO: Retrieve "updated" info from metadata parts
+      # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a
+      # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s)
+      # One of these parts should contain a string like: "Updated 2 days ago"
+
+      # TODO: Maybe add a button to access the first video of the playlist?
+      # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
+      # Available fields: "videoId", "playlistId", "params"
+
+      return SearchPlaylist.new({
+        title:           title,
+        id:              playlist_id,
+        author:          author_fallback.name,
+        ucid:            author_fallback.id,
+        video_count:     video_count || -1,
+        videos:          [] of SearchPlaylistVideo,
+        thumbnail:       thumbnail,
+        author_verified: false,
+      })
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
+
   # Parses an InnerTube continuationItemRenderer into a Continuation.
   # Returns nil when the given object isn't a continuationItemRenderer.
   #

From afc5b27d83d8b2b287842ed1ec43185135441d37 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 13:32:44 +0100
Subject: [PATCH 43/61] Extractors: Add support for shortsLockupViewModel

The 'shortsLockupViewModel' structure is used in the channel "shorts" tab
---
 src/invidious/yt_backend/extractors.cr | 58 ++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index cb8331a5..4416ef30 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -483,6 +483,7 @@ private module Parsers
       child ||= ReelItemRendererParser.process(item_contents, author_fallback)
       child ||= PlaylistRendererParser.process(item_contents, author_fallback)
       child ||= LockupViewModelParser.process(item_contents, author_fallback)
+      child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback)
       return child
     end
 
@@ -497,6 +498,9 @@ private module Parsers
   # reelItemRenderer items are used in the new (2022) channel layout,
   # in the "shorts" tab.
   #
+  # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel
+  # TODO: Confirm that hypothesis
+  #
   module ReelItemRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["reelItemRenderer"]?
@@ -652,6 +656,60 @@ private module Parsers
     end
   end
 
+  # Parses an InnerTube shortsLockupViewModel into a SearchVideo.
+  # Returns nil when the given object is not a shortsLockupViewModel.
+  #
+  # This structure is present since around October 2024 on the "shorts" tab of
+  # the channel page and likely replaces the reelItemRenderer structure. It is
+  # usually (always?) encapsulated in a richItemRenderer.
+  #
+  module ShortsLockupViewModelParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["shortsLockupViewModel"]?
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      # TODO: Maybe add support for "oardefault.jpg" thumbnails?
+      # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
+      # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...
+
+      video_id = item_contents.dig(
+        "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"
+      ).as_s
+
+      title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s
+
+      view_count = short_text_to_number(
+        item_contents.dig("overlayMetadata", "secondaryText", "content").as_s
+      )
+
+      # Approximate to one minute, as "shorts" generally don't exceed that.
+      # NOTE: The actual duration is not provided by Youtube anymore.
+      # TODO: Maybe use -1 as an error value and handle that on the frontend?
+      duration = 60_i32
+
+      SearchVideo.new({
+        title:              title,
+        id:                 video_id,
+        author:             author_fallback.name,
+        ucid:               author_fallback.id,
+        published:          Time.unix(0),
+        views:              view_count,
+        description_html:   "",
+        length_seconds:     duration,
+        premiere_timestamp: Time.unix(0),
+        author_verified:    false,
+        badges:             VideoBadges::None,
+      })
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
+
   # Parses an InnerTube continuationItemRenderer into a Continuation.
   # Returns nil when the given object isn't a continuationItemRenderer.
   #

From d27a5e7fae4a826b66950422ff8dfec4123dabf1 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 13:33:46 +0100
Subject: [PATCH 44/61] Channels: Rename ctoken generator functions as
 requested

---
 src/invidious/channels/videos.cr | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index 7b3e3cfa..9572adf3 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -26,7 +26,7 @@ module Invidious::Channel::Tabs
   end
 
   def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_videos_ctoken(ucid, sort_by)
+    continuation ||= make_initial_videos_ctoken(ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, author, ucid)
@@ -56,7 +56,7 @@ module Invidious::Channel::Tabs
   # -------------------
 
   def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_shorts_ctoken(channel.ucid, sort_by)
+    continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, channel.author, channel.ucid)
@@ -67,7 +67,7 @@ module Invidious::Channel::Tabs
   # -------------------
 
   def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_livestreams_ctoken(channel.ucid, sort_by)
+    continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, channel.author, channel.ucid)
@@ -108,7 +108,7 @@ module Invidious::Channel::Tabs
   # Generate the initial "continuation token" to get the first page of the
   # "videos" tab. The following page requires the ctoken provided in that
   # first page, and so on.
-  private def make_videos_ctoken(ucid : String, sort_by = "newest")
+  private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
     object = {
       "15:embedded" => {
         "2:string" => "\n$00000000-0000-0000-0000-000000000000",
@@ -122,7 +122,7 @@ module Invidious::Channel::Tabs
   # Generate the initial "continuation token" to get the first page of the
   # "shorts" tab. The following page requires the ctoken provided in that
   # first page, and so on.
-  private def make_shorts_ctoken(ucid : String, sort_by = "newest")
+  private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
     object = {
       "10:embedded" => {
         "2:embedded" => {
@@ -138,7 +138,7 @@ module Invidious::Channel::Tabs
   # Generate the initial "continuation token" to get the first page of the
   # "livestreams" tab. The following page requires the ctoken provided in that
   # first page, and so on.
-  private def make_livestreams_ctoken(ucid : String, sort_by = "newest")
+  private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
     sort_by_numerical =
       case sort_by
       when "newest"  then 12_i64

From 301aeffa780fca321793f8c2ef46844d613ce5c3 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 13:54:05 +0100
Subject: [PATCH 45/61] Channels: Multiple small fixes

Fix the "newest" link not being bold when 'sort_by' uses the default value
Show 60 videos per page, rather than 30
---
 src/invidious/routes/channels.cr | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index d4e9fa68..7d634cbb 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -20,10 +20,11 @@ module Invidious::Routes::Channels
     sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if channel.auto_generated
+      sort_by ||= "last"
       sort_options = {"last", "oldest", "newest"}
 
       items, next_continuation = fetch_channel_playlists(
-        channel.ucid, channel.author, continuation, (sort_by || "last")
+        channel.ucid, channel.author, continuation, sort_by
       )
 
       items.uniq! do |item|
@@ -49,9 +50,11 @@ module Invidious::Routes::Channels
         end
         next_continuation = nil
       else
+        sort_by ||= "newest"
         sort_options = {"newest", "oldest", "popular"}
-        items, next_continuation = Channel::Tabs.get_videos(
-          channel, continuation: continuation, sort_by: (sort_by || "newest")
+
+        items, next_continuation = Channel::Tabs.get_60_videos(
+          channel, continuation: continuation, sort_by: sort_by
         )
       end
     end

From 6dd662a5b84b3deb9e19e365f8b480357f63a2e9 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 17:25:23 +0100
Subject: [PATCH 46/61] Channels: lockupViewModel is also used in the
 "playlists" tab

---
 src/invidious/yt_backend/extractors.cr | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index 4416ef30..2631b62a 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -21,6 +21,7 @@ private ITEM_PARSERS = {
   Parsers::ItemSectionRendererParser,
   Parsers::ContinuationItemRendererParser,
   Parsers::HashtagRendererParser,
+  Parsers::LockupViewModelParser,
 }
 
 private alias InitialData = Hash(String, JSON::Any)
@@ -590,8 +591,9 @@ private module Parsers
   # Parses an InnerTube lockupViewModel into a SearchPlaylist.
   # Returns nil when the given object is not a lockupViewModel.
   #
-  # This structure is present since November 2024 on the "podcasts" tab of the
-  # channel page. It is usually (always?) encapsulated in a richItemRenderer.
+  # This structure is present since November 2024 on the "podcasts" and
+  # "playlists" tabs of the channel page. It is usually encapsulated in either
+  # a richItemRenderer or a richGridRenderer.
   #
   module LockupViewModelParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
@@ -608,7 +610,7 @@ private module Parsers
         "primaryThumbnail", "thumbnailViewModel"
       )
 
-      thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s
+      thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s
 
       # This complicated sequences tries to extract the following data structure:
       # "overlays": [{
@@ -621,10 +623,15 @@ private module Parsers
       #     }]
       #   }
       # }]
+      #
+      # NOTE: this simplistic `.to_i` conversion might not work on larger
+      # playlists and hasn't been tested.
       video_count = thumbnail_view_model.dig("overlays").as_a
         .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
         .flatten
-        .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes"))
+        .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node|
+          {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) }
+        })
         .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
 
       metadata = item_contents.dig("metadata", "lockupMetadataViewModel")

From 2a19dbb1fee20e5438751c3bb387f8757f4c2238 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 18:28:55 +0100
Subject: [PATCH 47/61] Channels: Use the same structure as in the other ctoken
 functions

Change explanation, courtesy of iBicha:
The \n is basically a decimal 10, which is 1010 binary. That is a field number
1, and a wire type 2 (length-delimited). Then the $ is a decimal 36, which is
exactly the length of 00000000-0000-0000-0000-000000000000.
So both objects end up being encoded into the same data.
---
 src/invidious/channels/videos.cr | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr
index 9572adf3..96400f47 100644
--- a/src/invidious/channels/videos.cr
+++ b/src/invidious/channels/videos.cr
@@ -111,7 +111,9 @@ module Invidious::Channel::Tabs
   private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
     object = {
       "15:embedded" => {
-        "2:string" => "\n$00000000-0000-0000-0000-000000000000",
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
         "4:varint" => sort_options_videos_short(sort_by),
       },
     }

From 09ccea1d31366e5f6ce1c0b3e13f3a8d84428184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89milien=20=28perso=29?=
 <4016501+unixfox@users.noreply.github.com>
Date: Fri, 8 Nov 2024 22:01:23 +0100
Subject: [PATCH 48/61] remove usage of TVHTML5_SIMPLY_EMBEDDED_PLAYER

---
 src/invidious/videos/parser.cr | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index fb8935d9..8dab5881 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -124,14 +124,6 @@ def extract_video_info(video_id : String)
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
 
-  # Last hope
-  # Only trigger if reason found or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
-  if reason && CONFIG.po_token.nil?
-    client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
-  end
-
   # Replace player response and reset reason
   if !new_player_response.nil?
     # Preserve captions & storyboard data before replacement

From b173d4acf21563d47d26718eca7932878fb424e6 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Fri, 8 Nov 2024 23:45:15 +0100
Subject: [PATCH 49/61] Update CHANGELOG.md

---
 CHANGELOG.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9892e17..061f977c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,14 @@
 
 ### Full list of pull requests merged since the last release (newest first)
 
+* Stale bot updates ([#5060], thanks @syeopite)
+* Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox)
+* Channels: Fix for live videos ([#5027], thanks @iBicha)
+* Locales: Add Bulgarian, Welsh and Lombard to the list ([#5046], by @SamantazFox)
+* Shards: Update database dependencies ([#5034], by @SamantazFox)
+* Logger: Add color support for different log levels ([#4931], thanks @Fijxu)
+* Fix named arg syntax when passing force_resolve ([#4754], thanks @syeopite)
+* Use make_client instead of calling HTTP::Client ([#4709], thanks @syeopite)
 * Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox)
 * Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox)
 * SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu)
@@ -31,7 +39,9 @@
 [#4270]: https://github.com/iv-org/invidious/pull/4270
 [#4326]: https://github.com/iv-org/invidious/pull/4326
 [#4652]: https://github.com/iv-org/invidious/pull/4652
+[#4709]: https://github.com/iv-org/invidious/pull/4709
 [#4750]: https://github.com/iv-org/invidious/pull/4750
+[#4754]: https://github.com/iv-org/invidious/pull/4754
 [#4850]: https://github.com/iv-org/invidious/pull/4850
 [#4862]: https://github.com/iv-org/invidious/pull/4862
 [#4863]: https://github.com/iv-org/invidious/pull/4863
@@ -41,10 +51,16 @@
 [#4923]: https://github.com/iv-org/invidious/pull/4923
 [#4928]: https://github.com/iv-org/invidious/pull/4928
 [#4930]: https://github.com/iv-org/invidious/pull/4930
+[#4931]: https://github.com/iv-org/invidious/pull/4931
 [#4942]: https://github.com/iv-org/invidious/pull/4942
 [#4991]: https://github.com/iv-org/invidious/pull/4991
 [#4993]: https://github.com/iv-org/invidious/pull/4993
 [#4995]: https://github.com/iv-org/invidious/pull/4995
+[#5027]: https://github.com/iv-org/invidious/pull/5027
+[#5034]: https://github.com/iv-org/invidious/pull/5034
+[#5046]: https://github.com/iv-org/invidious/pull/5046
+[#5059]: https://github.com/iv-org/invidious/pull/5059
+[#5060]: https://github.com/iv-org/invidious/pull/5060
 
 
 ## v2.20240825.2 (2024-08-26)

From 9d54cf903e4674577b096a47498ef42a06667bad Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Fri, 8 Nov 2024 15:51:54 -0800
Subject: [PATCH 50/61] Update shard.yml metadata

---
 shard.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/shard.yml b/shard.yml
index 513e5db3..e6a10c38 100644
--- a/shard.yml
+++ b/shard.yml
@@ -1,13 +1,12 @@
 name: invidious
-version: 0.20.1
+version: 2.20240825.2-dev
 
 authors:
   - Omar Roth <omarroth@protonmail.com>
-  - Invidious team
+  - Invidious team <contact@invidious.io>
 
-targets:
-  invidious:
-    main: src/invidious.cr
+description: |
+   Invidious is an alternative front-end to YouTube 
 
 dependencies:
   pg:
@@ -40,6 +39,10 @@ development_dependencies:
     github: crystal-ameba/ameba
     version: ~> 1.6.1
 
-crystal: ">= 1.0.0, < 2.0.0"
+crystal: ">= 1.10.0, < 2.0.0"
 
 license: AGPLv3
+
+repository: https://github.com/iv-org/invidious
+homepage: https://invidious.io
+documentation: https://docs.invidious.io

From 1a49e798c804e7841574838409dd0500010cbb16 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sat, 9 Nov 2024 21:52:06 +0100
Subject: [PATCH 51/61] Docker: Install tzdata in Dockerfile

---
 docker/Dockerfile       | 6 +++---
 docker/Dockerfile.arm64 | 9 +++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 3d9323fd..900c9e74 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM crystallang/crystal:1.12.1-alpine AS builder
+FROM crystallang/crystal:1.12.2-alpine AS builder
 
 RUN apk add --no-cache sqlite-static yaml-static
 
@@ -32,8 +32,8 @@ RUN if [[ "${release}" == 1 ]] ; then \
         --link-flags "-lxml2 -llzma"; \
     fi
 
-FROM alpine:3.18
-RUN apk add --no-cache rsvg-convert ttf-opensans tini
+FROM alpine:3.20
+RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious
diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64
index f054b326..ce9bab08 100644
--- a/docker/Dockerfile.arm64
+++ b/docker/Dockerfile.arm64
@@ -1,5 +1,6 @@
-FROM alpine:3.19 AS builder
-RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static
+FROM alpine:3.20 AS builder
+RUN apk add --no-cache 'crystal=1.12.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \
+       zlib-static openssl-libs-static openssl-dev musl-dev xz-static
 
 ARG release
 
@@ -32,8 +33,8 @@ RUN if [[ "${release}" == 1 ]] ; then \
         --link-flags "-lxml2 -llzma"; \
     fi
 
-FROM alpine:3.18
-RUN apk add --no-cache rsvg-convert ttf-opensans tini
+FROM alpine:3.20
+RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

From 8bf7e029782c86c3a1b7b4be1f7c6f3a9659d2eb Mon Sep 17 00:00:00 2001
From: syeopite <70992037+syeopite@users.noreply.github.com>
Date: Sat, 9 Nov 2024 13:04:10 -0800
Subject: [PATCH 52/61] Change authors section to reflect current state

---
 shard.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shard.yml b/shard.yml
index e6a10c38..4c3f7526 100644
--- a/shard.yml
+++ b/shard.yml
@@ -2,8 +2,8 @@ name: invidious
 version: 2.20240825.2-dev
 
 authors:
-  - Omar Roth <omarroth@protonmail.com>
   - Invidious team <contact@invidious.io>
+  - Contributors!
 
 description: |
    Invidious is an alternative front-end to YouTube 

From b9ad9bd72331e8a568bd11813cb0169bd9c2a831 Mon Sep 17 00:00:00 2001
From: Emilien <4016501+unixfox@users.noreply.github.com>
Date: Fri, 8 Nov 2024 21:47:52 +0100
Subject: [PATCH 53/61] use WEB when po_token + android test suite when no
 po_token

---
 src/invidious/videos/parser.cr | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index fb8935d9..b2744120 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -53,9 +53,9 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
-  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  # Use the WEB when po_token is configured
   if CONFIG.po_token
-    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+    client_config.client_type = YoutubeAPI::ClientType::Web
   end
 
   # Fetch data from the player endpoint
@@ -113,8 +113,8 @@ def extract_video_info(video_id : String)
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
 
-  # Don't use Android client if po_token is passed because po_token doesn't
-  # work for Android client.
+  # Don't use Android test suite client if po_token is passed because po_token doesn't
+  # work for Android test suite client.
   if reason.nil? && CONFIG.po_token.nil?
     # Fetch the video streams using an Android client in order to get the
     # decrypted URLs and maybe fix throttling issues (#2194). See the

From 82b1506cccc85ed4f3979d674a2754c35e0194fc Mon Sep 17 00:00:00 2001
From: Emilien <4016501+unixfox@users.noreply.github.com>
Date: Fri, 8 Nov 2024 21:56:24 +0100
Subject: [PATCH 54/61] remove usage of WebEmbeddedPlayer

---
 src/invidious/videos/parser.cr | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index b2744120..1f229df0 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -106,13 +106,6 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
-  # Second try in case WEB_CREATOR doesn't work with po_token.
-  # Only trigger if reason found and po_token configured.
-  if reason && CONFIG.po_token
-    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
-  end
-
   # Don't use Android test suite client if po_token is passed because po_token doesn't
   # work for Android test suite client.
   if reason.nil? && CONFIG.po_token.nil?

From f3e93ca83d21ab7e766d931c3985cf291e96ad3e Mon Sep 17 00:00:00 2001
From: Emilien <4016501+unixfox@users.noreply.github.com>
Date: Fri, 8 Nov 2024 21:56:48 +0100
Subject: [PATCH 55/61] revert back to www.youtube.com when
 client_config.screen embed

---
 src/invidious/yt_backend/youtube_api.cr | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index e0a3181f..8f5aa61d 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -300,9 +300,8 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
-      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.google.com/",
+        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
       } of String => String | Int64
     end
 

From 0f8f32bca8739dfc05edc70ac634bfebe763d927 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89milien=20=28perso=29?=
 <4016501+unixfox@users.noreply.github.com>
Date: Fri, 8 Nov 2024 23:33:19 +0100
Subject: [PATCH 56/61] remove explicit usage of WEB

---
 src/invidious/videos/parser.cr | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index 1f229df0..65ba8a3d 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -53,10 +53,6 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
-  # Use the WEB when po_token is configured
-  if CONFIG.po_token
-    client_config.client_type = YoutubeAPI::ClientType::Web
-  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)

From d2123b46829e57515e281ffd98b75dac3de6f379 Mon Sep 17 00:00:00 2001
From: Brahim Hadriche <brahim.hadriche@gmail.com>
Date: Sat, 9 Nov 2024 17:49:06 -0500
Subject: [PATCH 57/61] Sort channel shorts API

---
 src/invidious/routes/api/v1/channels.cr | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 2da76134..588bbc2a 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -197,6 +197,7 @@ module Invidious::Routes::API::V1::Channels
     get_channel()
 
     # Retrieve continuation from URL parameters
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
     continuation = env.params.query["continuation"]?
 
     if channel.is_age_gated
@@ -211,7 +212,7 @@ module Invidious::Routes::API::V1::Channels
     else
       begin
         videos, next_continuation = Channel::Tabs.get_shorts(
-          channel, continuation: continuation
+          channel, continuation: continuation, sort_by: sort_by
         )
       rescue ex
         return error_json(500, ex)

From 2150264d849771df8f15bab172ab6d87eeb80c55 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 10 Nov 2024 18:00:26 +0100
Subject: [PATCH 58/61] Update CHANGELOG.md

---
 CHANGELOG.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 061f977c..9cc36f12 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@
 
 ### Full list of pull requests merged since the last release (newest first)
 
+* API: Add "sort_by" parameter to channels/shorts endpoint ([#5071], thanks @iBicha)
+* Docker: Install tzdata in Dockerfile ([#5070], by @SamantazFox)
+* Videos: Stop using TVHTML5_SIMPLY_EMBEDDED_PLAYER ([#5063], thanks @unixfox)
+* Routing: Deprecate old channel API routes ([#5045], by @SamantazFox)
+* Videos: use WEB client instead of WEB CREATOR ([#4984], thanks @unixfox)
+* Parsers: Fix parsing live_now and premiere_timestamp ([#4934], thanks @absidue)
 * Stale bot updates ([#5060], thanks @syeopite)
 * Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox)
 * Channels: Fix for live videos ([#5027], thanks @iBicha)
@@ -52,15 +58,21 @@
 [#4928]: https://github.com/iv-org/invidious/pull/4928
 [#4930]: https://github.com/iv-org/invidious/pull/4930
 [#4931]: https://github.com/iv-org/invidious/pull/4931
+[#4934]: https://github.com/iv-org/invidious/pull/4934
 [#4942]: https://github.com/iv-org/invidious/pull/4942
+[#4984]: https://github.com/iv-org/invidious/pull/4984
 [#4991]: https://github.com/iv-org/invidious/pull/4991
 [#4993]: https://github.com/iv-org/invidious/pull/4993
 [#4995]: https://github.com/iv-org/invidious/pull/4995
 [#5027]: https://github.com/iv-org/invidious/pull/5027
 [#5034]: https://github.com/iv-org/invidious/pull/5034
+[#5045]: https://github.com/iv-org/invidious/pull/5045
 [#5046]: https://github.com/iv-org/invidious/pull/5046
 [#5059]: https://github.com/iv-org/invidious/pull/5059
 [#5060]: https://github.com/iv-org/invidious/pull/5060
+[#5063]: https://github.com/iv-org/invidious/pull/5063
+[#5070]: https://github.com/iv-org/invidious/pull/5070
+[#5071]: https://github.com/iv-org/invidious/pull/5071
 
 
 ## v2.20240825.2 (2024-08-26)

From 5d2dd40bc3e043caac7d85e7ddee3d2c5d40ccbb Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 10 Nov 2024 21:04:37 +0100
Subject: [PATCH 59/61] Release v2.20241110.0

---
 CHANGELOG.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 shard.yml    |  2 +-
 2 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9cc36f12..d8f3299c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,73 @@
 # CHANGELOG
 
-## vX.Y.0 (future)
+## v2.20241110.0
+
+### Wrap-up
+
+This release is most importantly here to fix to the annoying "Youtube API returned error 400"
+error that prevented all channel pages from loading.
+
+If you're updating from the previous release, it provides no improvements on the ability to play
+videos. If updating from a commit in-between release, it removes the "Please sign in" error caused
+by a previous attempt at restoring video playback on large instances.
+
+In the preferences, a new option allows for control of video preload. When enabled, this option
+tells the browser to load the video as soon as the page is loaded (this used to be the default).
+When disabled, the video starts loading only when the "play" button is pressed.
+
+New interface languages available: Bulgarian, Welsh and Lombard
+
+New dependency required: `tzdata`.
+
+An HTTP proxy can be configured directly in Invidious, if needed. \
+**NOTE:** In that case, it is recommended to comment out `force_resolve`.
+
+
+### New features & important changes
+
+#### For users
+
+* Channels: Fix "Youtube API returned error 400" error preventing channel pages from loading
+* Channels: Shorts can now be sorted by "newest", "oldest" and "popular"
+* Preferences: Addition of the new "preload" option
+* New interface languages available: Bulgarian, Welsh and Lombard
+* Added "Filipino (auto-generated)" to the list of caption languages available
+* Lots of new translations from Weblate
+
+#### For instance owners
+
+* Allow the configuration of an HTTP proxy to talk to Youtube
+* Invidious tries to reconnect to `inv_sig_helper` if the socket is closed
+* The instance list is downloaded in the background to improve redirection speed
+* New `colorize_logs` option makes each log level a different color
+
+#### For developpers
+
+* `/api/v1/channels/{id}/shorts` now supports the `sort-by` parameter with the following values:
+  `newest`, `oldest` and `popular`
+* Older `/api/v1/channels/xyz/{id}` (tab name before UCID) were removed
+* API/Search: New video metadata available: `isNew`, `is4k`, `is8k`, `isVr180`, `isVr360`,
+  `is3d` and `hasCaptions`
+
+### Bugs fixed
+
+#### User-side
+
+* Channels: The second page of shorts now loads as expected
+* Channels: Fixed intermittent empty "playlists" tab
+* Search: Fixed `youtu.be` URLs not being properly redirected to the watch page
+* Fixed `DB::MappingException` error on the subscriptions feed (due to missing `tzdata` in docker)
+* Switching to another instance is much faster
+* Fixed an "invalid byte sequence" error when subscribing to a playlist
+* Videos: Playback URLs were sometimes broken when cached and `inv_sig_helper` was used
+
+#### For instance owners
+
+* Fix `force_resolve` being ignored in some cases
+
+#### API
+
+* API/Videos: Fixed `live_now` and `premiere_timestamp` sometimes not having the right values
 
 
 ### Full list of pull requests merged since the last release (newest first)
diff --git a/shard.yml b/shard.yml
index 4c3f7526..2e4df045 100644
--- a/shard.yml
+++ b/shard.yml
@@ -1,5 +1,5 @@
 name: invidious
-version: 2.20240825.2-dev
+version: 2.20241110.0
 
 authors:
   - Invidious team <contact@invidious.io>

From 98926047586154269bb269d01e3e52e60e044035 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 10 Nov 2024 21:40:32 +0100
Subject: [PATCH 60/61] Prepare for next release

---
 CHANGELOG.md | 3 +++
 shard.yml    | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d8f3299c..5af38003 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## vX.Y.0 (future)
+
+
 ## v2.20241110.0
 
 ### Wrap-up
diff --git a/shard.yml b/shard.yml
index 2e4df045..af7e4186 100644
--- a/shard.yml
+++ b/shard.yml
@@ -1,5 +1,5 @@
 name: invidious
-version: 2.20241110.0
+version: 2.20241110.0-dev
 
 authors:
   - Invidious team <contact@invidious.io>

From b13f77b5afe72b5258d2a9484833a7b7e0fd0944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89milien=20=28perso=29?=
 <4016501+unixfox@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:21:28 +0100
Subject: [PATCH 61/61] Update bug report issue message

---
 .github/ISSUE_TEMPLATE/bug_report.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 4c1a6330..02bc3795 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -10,8 +10,10 @@ assignees: ''
 <!--
   BEFORE TRYING TO REPORT A BUG:
  
-  * Read the FAQ!
-  * Use the search function to check if there is already an issue open for your problem!
+  * Read the FAQ: https://docs.invidious.io/faq/!
+  * Use the search function to check if there is already an issue open for your problem: https://github.com/search?q=repo%3Aiv-org%2Finvidious+replace+me+with+your+bug&type=issues!
+
+  MAKE SURE TO FOLLOW THE TWO STEPS ABOVE BEFORE REPORTING A BUG. A BUG THAT ALREADY EXIST WILL IMMEDIATELY CLOSED.
 
   If you want to suggest a new feature please use "Feature request" instead
   If you want to suggest an enhancement to an existing feature please use "Enhancement" instead