Merge 36ed5d3418 into 53e8a5d62d

* Update to latest version WEB_CREATOR

* fix comment about using web embed as a fallback

.github/CODEOWNERS

@@ -6,7 +6,7 @@ docker/ @unixfox
 kubernetes/ @unixfox
 
 README.md @thefrenchghosty
-config/config.example.yml @thefrenchghosty @SamantazFox @unixfox
+config/config.example.yml @SamantazFox @unixfox
 
 scripts/ @syeopite
 shards.lock @syeopite

assets/css/default.css

@@ -160,7 +160,9 @@ body a.pure-button {
 button.pure-button-primary,
 body a.pure-button-primary,
 .channel-owner:hover,
-.channel-owner:focus {
+.channel-owner:focus,
+.chapter:hover,
+.chapter:focus {
   background-color: #a0a0a0;
   color: rgba(35, 35, 35, 1);
 }
@@ -814,5 +816,26 @@ h1, h2, h3, h4, h5, p,
 }
 
 #download_widget {
-    width: 100%;
+  width: 100%;
+}
+
+.description-chapters-section {
+  white-space: normal;
+}
+
+.description-chapters-content-container {
+  display: flex;
+  flex-direction: row;
+  gap: 5px;
+  overflow: scroll;
+
+  overflow-y: hidden;
+}
+
+.chapter {
+  padding: 3px;
+}
+
+.chapter .thumbnail {
+  width: 200px;
 }

assets/css/player.css

@@ -114,6 +114,10 @@ ul.vjs-menu-content::-webkit-scrollbar {
   order: 5;
 }
 
+.vjs-chapters-button {
+  order: 5;
+}
+
 .vjs-share-control {
   order: 6;
 }

assets/js/player.js

@@ -17,6 +17,7 @@ var options = {
             'remainingTimeDisplay',
             'Spacer',
             'captionsButton',
+            'ChaptersButton',
             'audioTrackButton',
             'qualitySelector',
             'playbackRateMenuButton',

assets/js/watch.js

@@ -191,3 +191,9 @@ addEventListener('load', function (e) {
         comments.innerHTML = '';
     }
 });
+
+const chapter_widget_buttons = document.getElementsByClassName("chapter-widget-buttons")
+Array.from(chapter_widget_buttons).forEach(e => e.addEventListener("click", function (event) {
+    event.preventDefault();
+    player.currentTime(e.getAttribute('data-jump-time'));
+}))

locales/en-US.json

@@ -496,5 +496,7 @@
     "toggle_theme": "Toggle Theme",
     "carousel_slide": "Slide {{current}} of {{total}}",
     "carousel_skip": "Skip the Carousel",
-    "carousel_go_to": "Go to slide `x`"
+    "carousel_go_to": "Go to slide `x`",
+    "video_chapters_label": "Chapters",
+    "video_chapters_auto_generated_label": "These chapters are auto-generated"
 }

src/invidious/jsonify/api_v1/video_json.cr

@@ -218,6 +218,14 @@ module Invidious::JSONify::APIv1
         end
       end
 
+      if !video.chapters.nil?
+        json.field "chapters" do
+          json.object do
+            video.chapters.to_json(json)
+          end
+        end
+      end
+
       if !video.music.empty?
         json.field "musicTracks" do
           json.array do

src/invidious/routes/api/v1/videos.cr

@@ -429,4 +429,41 @@ module Invidious::Routes::API::V1::Videos
       end
     end
   end
+
+  def self.chapters(env)
+    id = env.params.url["id"]
+    region = env.params.query["region"]? || env.params.body["region"]?
+
+    if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/)
+      return error_json(400, "Invalid video ID")
+    end
+
+    format = env.params.query["format"]?
+
+    begin
+      video = get_video(id, region: region)
+    rescue ex : NotFoundException
+      haltf env, 404
+    rescue ex
+      haltf env, 500
+    end
+
+    begin
+      chapters = video.chapters
+    rescue ex
+      haltf env, 500
+    end
+
+    if chapters.nil?
+      return error_json(404, "No chapters are defined in video \"#{id}\"")
+    end
+
+    if format == "json"
+      env.response.content_type = "application/json"
+      return chapters.to_json
+    else
+      env.response.content_type = "text/vtt; charset=UTF-8"
+      return chapters.to_vtt
+    end
+  end
 end

src/invidious/routes/images.cr

@@ -160,6 +160,12 @@ module Invidious::Routes::Images
     id = env.params.url["id"]
     name = env.params.url["name"]
 
+    # Some thumbnails such as the ones for chapters requires some additional queries.
+    query_params = HTTP::Params.new
+    {"sqp", "rs"}.each do |name|
+      query_params[name] = env.params.query[name] if env.params.query[name]?
+    end
+
     headers = HTTP::Headers.new
 
     if name == "maxres.jpg"
@@ -173,7 +179,7 @@ module Invidious::Routes::Images
       end
     end
 
-    url = "/vi/#{id}/#{name}"
+    url = "/vi/#{id}/#{name}?#{query_params}"
 
     REQUEST_HEADERS_WHITELIST.each do |header|
       if env.request.headers[header]?

src/invidious/routing.cr

@@ -236,6 +236,7 @@ module Invidious::Routing
       get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
       get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
       get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
+      get "/api/v1/chapters/:id", {{namespace}}::Videos, :chapters
 
       # Feeds
       get "/api/v1/trending", {{namespace}}::Feeds, :trending

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 2
+  SCHEMA_VERSION = 3
 
   property id : String
 
@@ -26,6 +26,9 @@ struct Video
   @[DB::Field(ignore: true)]
   @captions = [] of Invidious::Videos::Captions::Metadata
 
+  @[DB::Field(ignore: true)]
+  @chapters : Invidious::Videos::Chapters? = nil
+
   @[DB::Field(ignore: true)]
   property adaptive_fmts : Array(Hash(String, JSON::Any))?
 
@@ -197,6 +200,24 @@ struct Video
     return @captions
   end
 
+  def chapters
+    # As the chapters key is always present in @info we need to check that it is
+    # actually populated
+    if @chapters.nil?
+      chapters = @info["chapters"].as_a
+      return nil if chapters.empty?
+
+      @chapters = Invidious::Videos::Chapters.from_raw_chapters(
+        chapters,
+        self.length_seconds,
+        # Should never be nil but just in case
+        is_auto_generated: @info["autoGeneratedChapters"].as_bool? || false
+      )
+    end
+
+    return @chapters
+  end
+
   def hls_manifest_url : String?
     info.dig?("streamingData", "hlsManifestUrl").try &.as_s
   end

src/invidious/videos/chapters.cr

@@ -0,0 +1,108 @@
+module Invidious::Videos
+  # A `Chapters` struct represents an sequence of chapters for a given video
+  struct Chapters
+    record Chapter, start_ms : Time::Span, end_ms : Time::Span, title : String, thumbnails : Array(Hash(String, Int32 | String))
+    property? auto_generated : Bool
+
+    def initialize(@chapters : Array(Chapter), @auto_generated : Bool)
+    end
+
+    # Constructs a chapters object from InnerTube's JSON object for chapters
+    #
+    # Requires the length of the video the chapters are associated to in order to construct correct ending time
+    def Chapters.from_raw_chapters(raw_chapters : Array(JSON::Any), video_length : Int32, is_auto_generated : Bool = false)
+      video_length_milliseconds = video_length.seconds.total_milliseconds
+
+      parsed_chapters = [] of Chapter
+
+      raw_chapters.each_with_index do |chapter, index|
+        chapter = chapter["chapterRenderer"]
+
+        title = chapter["title"]["simpleText"].as_s
+
+        raw_thumbnails = chapter["thumbnail"]["thumbnails"].as_a
+        thumbnails = raw_thumbnails.map do |thumbnail|
+          {
+            "url"    => thumbnail["url"].as_s,
+            "width"  => thumbnail["width"].as_i,
+            "height" => thumbnail["height"].as_i,
+          }
+        end
+
+        start_ms = chapter["timeRangeStartMillis"].as_i
+
+        # To get the ending range we have to peek at the next chapter.
+        # If we're the last chapter then we need to calculate the end time through the video length.
+        if next_chapter = raw_chapters[index + 1]?
+          end_ms = next_chapter["chapterRenderer"]["timeRangeStartMillis"].as_i
+        else
+          end_ms = video_length_milliseconds.to_i
+        end
+
+        parsed_chapters << Chapter.new(
+          start_ms: start_ms.milliseconds,
+          end_ms: end_ms.milliseconds,
+          title: title,
+          thumbnails: thumbnails,
+        )
+      end
+
+      return Chapters.new(parsed_chapters, is_auto_generated)
+    end
+
+    # Calls the given block for each chapter and passes it as a parameter
+    def each(&)
+      @chapters.each { |c| yield c }
+    end
+
+    # Converts the sequence of chapters to a WebVTT representation
+    def to_vtt
+      vtt = WebVTT.build do |build|
+        self.each do |chapter|
+          build.cue(chapter.start_ms, chapter.end_ms, chapter.title)
+        end
+      end
+    end
+
+    # Dumps a JSON representation of the sequence of chapters to the given JSON::Builder
+    def to_json(json : JSON::Builder)
+      json.field "autoGenerated", @auto_generated.to_s
+      json.field "chapters" do
+        json.array do
+          @chapters.each do |chapter|
+            json.object do
+              json.field "title", chapter.title
+              json.field "startMs", chapter.start_ms.total_milliseconds
+              json.field "endMs", chapter.end_ms.total_milliseconds
+
+              json.field "thumbnails" do
+                json.array do
+                  chapter.thumbnails.each do |thumbnail|
+                    json.object do
+                      json.field "url", URI.parse(thumbnail["url"].as(String)).request_target
+                      json.field "width", thumbnail["width"]
+                      json.field "height", thumbnail["height"]
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    # Create a JSON representation of the sequence of chapters
+    def to_json
+      JSON.build do |json|
+        json.object do
+          json.field "chapters" do
+            json.object do
+              to_json(json)
+            end
+          end
+        end
+      end
+    end
+  end
+end

src/invidious/videos/parser.cr

@@ -53,6 +53,10 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  if CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
@@ -102,6 +106,13 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
+  # Second try in case WEB_CREATOR doesn't work with po_token.
+  # Only trigger if reason found and po_token configured.
+  if reason && CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
   if reason.nil? && CONFIG.po_token.nil?
@@ -114,10 +125,9 @@ def extract_video_info(video_id : String)
   end
 
   # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  # Only trigger if reason found or didn't work wth Android client.
+  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
+  if reason && CONFIG.po_token.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -185,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
-  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+  if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+    microformat = {} of String => JSON::Any
+  end
 
   raise BrokenTubeException.new("videoDetails") if !video_details
-  raise BrokenTubeException.new("microformat") if !microformat
 
   # Basic video infos
 
@@ -225,7 +236,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
-  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  family_friendly = microformat["isFamilySafe"]?.try &.as_bool
   is_listed = video_details["isCrawlable"]?.try &.as_bool
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 
@@ -248,11 +259,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     end
   end
 
+  player_overlays = player_response.dig?("playerOverlays", "playerOverlayRenderer")
+
   # If nothing was found previously, fall back to end screen renderer
   if related.empty?
     # Container for "endScreenVideoRenderer" items
-    player_overlays = player_response.dig?(
-      "playerOverlays", "playerOverlayRenderer",
+    end_screen_watch_next_array = player_overlays.try &.dig?(
       "endScreen", "watchNextEndScreenRenderer", "results"
     )
 
@@ -394,6 +406,32 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
       .try &.as_s.split(" ", 2)[0]
   end
 
+  # Chapters
+  chapters_array = [] of JSON::Any
+  chapters_auto_generated = nil
+
+  # Yes,`decoratedPlayerBarRenderer` is repeated twice.
+  if player_bar = player_overlays.try &.dig?("decoratedPlayerBarRenderer", "decoratedPlayerBarRenderer", "playerBar")
+    if markers = player_bar.dig?("multiMarkersPlayerBarRenderer", "markersMap")
+      potential_chapters_array = markers.as_a.find(&.["key"]?.try &.== "DESCRIPTION_CHAPTERS")
+
+      # Chapters that are manually created should have a higher precedence than automatically generated chapters
+      if !potential_chapters_array
+        potential_chapters_array = markers.as_a.find(&.["key"]?.try &.== "AUTO_CHAPTERS")
+      end
+
+      if potential_chapters_array
+        if potential_chapters_array["key"] == "AUTO_CHAPTERS"
+          chapters_auto_generated = true
+        else
+          chapters_auto_generated = false
+        end
+
+        chapters_array = potential_chapters_array["value"]["chapters"].as_a
+      end
+    end
+  end
+
   # Return data
 
   if live_now
@@ -414,13 +452,15 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     "lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
     "published"     => JSON::Any.new(published.to_rfc3339),
     # Extra video infos
-    "allowedRegions"   => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
-    "allowRatings"     => JSON::Any.new(allow_ratings || false),
-    "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
-    "isListed"         => JSON::Any.new(is_listed || false),
-    "isUpcoming"       => JSON::Any.new(is_upcoming || false),
-    "keywords"         => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
-    "isPostLiveDvr"    => JSON::Any.new(post_live_dvr),
+    "allowedRegions"        => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
+    "allowRatings"          => JSON::Any.new(allow_ratings || false),
+    "isFamilyFriendly"      => JSON::Any.new(family_friendly || false),
+    "isListed"              => JSON::Any.new(is_listed || false),
+    "isUpcoming"            => JSON::Any.new(is_upcoming || false),
+    "keywords"              => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
+    "isPostLiveDvr"         => JSON::Any.new(post_live_dvr),
+    "autoGeneratedChapters" => JSON::Any.new(chapters_auto_generated),
+    "chapters"              => JSON::Any.new(chapters_array),
     # Related videos
     "relatedVideos" => JSON::Any.new(related),
     # Description

src/invidious/views/components/description_chapters_widget.ecr

@@ -0,0 +1,34 @@
+<% if chapters = video.chapters %>
+    <div class="description-chapters-section">
+        <hr class="description-content-separator"/>
+        <h4><%=HTML.escape(translate(locale, "video_chapters_label"))%></h4>
+
+        <% if chapters.auto_generated? %> 
+          <h5><%=HTML.escape(translate(locale, "video_chapters_auto_generated_label"))%> </h5>
+        <% end %>
+
+        <div class="description-chapters-content-container">
+            <% chapters.each do | chapter | %>
+                <%- start_in_seconds = chapter.start_ms.total_seconds.to_i %>
+                <a href="/watch?v=<%-= video.id %>&t=<%=start_in_seconds %>" data-jump-time="<%=start_in_seconds%>" class="chapter-widget-buttons">
+                    <div class="chapter">
+                        <div class="thumbnail">
+                            <%- if !env.get("preferences").as(Preferences).thin_mode -%>
+                                <img loading="lazy" class="thumbnail" src="<%-=URI.parse(chapter.thumbnails[-1]["url"].to_s).request_target %>" alt="<%=chapter.title%>"/>
+                            <%- else -%>
+                                <div class="thumbnail-placeholder"></div>
+                            <%- end -%>
+                        </div>
+                        <%- if start_in_seconds > 0 -%>
+                            <p><%-= recode_length_seconds(start_in_seconds) -%></p>
+                        <%- else -%>
+                            <p>0:00</p>
+                        <%- end -%>
+                        <p><%-=chapter.title-%></p>
+                    </div>
+                </a>
+            <% end %>
+        </div>
+        <hr class="description-content-separator"/>
+    </div>
+<% end %>

src/invidious/views/components/player.ecr

@@ -63,6 +63,10 @@
         <% captions.each do |caption| %>
             <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
         <% end %>
+
+        <% if !video.chapters.nil? %>
+            <track kind="chapters" src="/api/v1/chapters/<%= video.id %>">
+        <% end %>
     <% end %>
 </video>
 

src/invidious/views/watch.ecr

@@ -254,10 +254,14 @@ we're going to need to do it here in order to allow for translations.
 
             <div id="description-box"> <!-- Description -->
                 <% if video.description.size < 200 || params.extend_desc %>
-                    <div id="descriptionWrapper"><%= video.description_html %></div>
+                    <div id="descriptionWrapper"><%-= video.description_html %>
+                        <%= rendered "components/description_chapters_widget" %>
+                    </div>
                 <% else %>
                     <input id="descexpansionbutton" type="checkbox"/>
-                    <div id="descriptionWrapper"><%= video.description_html %></div>
+                    <div id="descriptionWrapper"><%-= video.description_html %>
+                        <%= rendered "components/description_chapters_widget" %>
+                    </div>
                     <label for="descexpansionbutton">
                         <a></a>
                     </label>

src/invidious/yt_backend/youtube_api.cr

@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -80,6 +81,14 @@ module YoutubeAPI
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20240918.03.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -291,8 +300,9 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
+      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+        "embedUrl" => "https://www.google.com/",
       } of String => String | Int64
     end