Fix Ameba error

* Update to latest version WEB_CREATOR

* fix comment about using web embed as a fallback

.github/CODEOWNERS

@@ -6,7 +6,7 @@ docker/ @unixfox
 kubernetes/ @unixfox
 
 README.md @thefrenchghosty
-config/config.example.yml @thefrenchghosty @SamantazFox @unixfox
+config/config.example.yml @SamantazFox @unixfox
 
 scripts/ @syeopite
 shards.lock @syeopite

.github/workflows/build-stable-container.yml

@@ -1,6 +1,7 @@
 name: Build and release container
 
 on:
+  workflow_dispatch:
   push:
     tags:
       - "v*"
@@ -46,9 +47,11 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: quay.io/invidious/invidious
+          flavor: |
+            latest=false
           tags: |
             type=semver,pattern={{version}}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=latest
           labels: |
             quay.expires-after=12w
 
@@ -70,10 +73,11 @@ jobs:
         with:
           images: quay.io/invidious/invidious
           flavor: |
+            latest=false
             suffix=-arm64
           tags: |
             type=semver,pattern={{version}}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=latest
           labels: |
             quay.expires-after=12w
 

CHANGELOG.md

@@ -1,6 +1,189 @@
 # CHANGELOG
 
-## 2024-04-26
+
+## v2.20240825.2 (2024-08-26)
+
+This releases fixes the container tags pushed on quay.io.
+Previously, the ARM64 build was released under the `latest` tag, instead of `latest-arm64`.
+
+### Full list of pull requests merged since the last release (newest first)
+
+CI: Fix docker container tags ([#4883], by @SamantazFox)
+
+[#4877]: https://github.com/iv-org/invidious/pull/4877
+
+
+## v2.20240825.1 (2024-08-25)
+
+Add patch component to be [semver] compliant and make github actions happy.
+
+[semver]: https://semver.org/
+
+### Full list of pull requests merged since the last release (newest first)
+
+Allow manual trigger of release-container build ([#4877], thanks @syeopite)
+
+[#4877]: https://github.com/iv-org/invidious/pull/4877
+
+
+## v2.20240825.0 (2024-08-25)
+
+### New features & important changes
+
+#### For users
+
+* The search bar now has a button that you can click!
+* Youtube URLs can be pasted directly in the search bar. Prepend search query with a
+  backslash (`\`) to disable that feature (useful if you need to search for a video whose
+  title contains some youtube URL).
+* On the channel page the "streams" tab can be sorted by either: "newest", "oldest" or "popular"
+* Lots of translations have been updated (thanks to our contributors on Weblate!)
+* Videos embedded in local HTML files (e.g: a webpage saved from a blog) can now be played
+
+#### For instance owners
+
+* Invidious now has the ability to provide a `po_token` and `visitordata` to Youtube in order to
+  circumvent current Youtube restrictions.
+* Invidious can use an (optional) external signature server like [inv_sig_helper]. Please note that
+  some videos can't be played without that signature server.
+* The Helm charts were moved to a separate repo: https://github.com/iv-org/invidious-helm-chart
+* We have changed how containers are released: the `latest` tag now tracks tagged releases, whereas
+  the `master` tag tracks the most recent commits of the `master` branch ("nightly" builds).
+
+[inv_sig_helper]: https://github.com/iv-org/inv_sig_helper
+
+#### For developpers
+
+* The versions of Crystal that we test in CI/CD are now: `1.9.2`, `1.10.1`, `1.11.2`, `1.12.1`.
+  Please note that due to a bug in the `libxml` bindings (See [#4256]), versions prior to `1.10.0`
+  are not recommended to use.
+* Thanks to @syeopite, the code is now [ameba] compliant.
+* Ameba is part of our CI/CD pipeline, and its rules will be enforced in future PRs.
+* The transcript code has been rewritten to permit transcripts as a feature rather than being
+  only a workaround for captions. Trancripts feature is coming soon!
+* Various fixes regarding the logic interacting with Youtube
+* The `sort_by` parameter can be used on the `/api/v1/channels/{id}/streams` endpoint. Accepted
+  values are: "newest", "oldest" and "popular"
+
+[ameba]: https://github.com/crystal-ameba/ameba
+[#4256]: https://github.com/iv-org/invidious/issues/4256
+
+
+### Bugs fixed
+
+#### User-side
+
+* Channels: fixed broken "subscribers" and "views" counters
+* Watch page: playback position is reset at the end of a video, so that the next time this video
+  is watched, it will start from the beginning rather than 15 seconds before the end
+* Watch page: the items in the "add to playlist" drop down are now sorted alphabetically
+* Videos: the "genre" URL is now always pointing to a valid webpage
+* Playlists: Fixed `Could not parse N episodes` error on podcast playlists
+* All external links should now have the [`rel`] attibute set to `noreferrer noopener` for
+  increased privacy.
+* Preferences: Fixed the admin-only "modified source code" input being ignored
+* Watch/channel pages: use the full image URL in `og:image` and `twitter:image` meta tags
+
+[`rel`]: https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes/rel
+
+#### API
+
+* fixed the `local` parameter not applying to `formatStreams` on `/api/v1/videos/{id}`
+* fixed an `Index out of bounds` error hapenning when a playlist had no videos
+* fixed duplicated query parameters in proxied video URLs
+* Return actual video height/width/fps rather than hard coded values
+* Fixed the `/api/v1/popular` endpoint not returning a proper error code/message when the
+  popular page/endpoint are disabled.
+
+
+### Full list of pull requests merged since the last release (newest first)
+
+* HTML: Sort playlists alphabetically in watch page drop down ([#4853], by @SamantazFox)
+* Videos: Fix XSS vulnerability in description/comments ([#4852], thanks _anonymous_)
+* YtAPI: Bump client versions ([#4849], by @SamantazFox)
+* SigHelper: Fix inverted time comparison in 'check_update' ([#4845], by @SamantazFox)
+* Storyboards: Various fixes and code cleaning ([#4153], by SamantazFox)
+* Fix lint errors introduced in #4146 and #4295 ([#4876], thanks @syeopite)
+* Search: Add support for Youtube URLs ([#4146], by @SamantazFox)
+* Channel: Render age restricted channels ([#4295], thanks @ChunkyProgrammer)
+* Ameba: Miscellaneous fixes ([#4807], thanks @syeopite)
+* API: Proxy formatStreams URLs too ([#4859], thanks @colinleroy)
+* UI: Add search button to search bar ([#4706], thanks @thansk)
+* Add ability to set po_token and visitordata ID ([#4789], thanks @unixfox)
+* Add support for an external signature server ([#4772], by @SamantazFox)
+* Ameba: Fix Naming/VariableNames ([#4790], thanks @syeopite)
+* Translations update from Hosted Weblate ([#4659])
+* Ameba: Fix Lint/UselessAssign ([#4795], thanks @syeopite)
+* HTML: Add rel="noreferrer noopener" to external links ([#4667], thanks @ulmemxpoc)
+* Remove unused methods in Invidious::LogHandler ([#4812], thanks @syeopite)
+* Ameba: Fix Lint/NotNilAfterNoBang ([#4796], thanks @syeopite)
+* Ameba: Fix unused argument Lint warnings ([#4805], thanks @syeopite)
+* Ameba: i18next.cr fixes ([#4806], thanks @syeopite)
+* Ameba: Disable rules ([#4792], thanks @syeopite)
+* Channel: parse subscriber count and channel banner ([#4785], thanks @ChunkyProgrammer)
+* Player: Fix playback position of already watched videos ([#4731], thanks @Fijxu)
+* Videos: Fix genre url being unusable ([#4717], thanks @meatball133)
+* API: Fix out of bound error on empty playlists ([#4696], thanks @Fijxu)
+* Handle playlists cataloged as Podcast ([#4695], thanks @Fijxu)
+* API: Fix duplicated query parameters in proxied video URLs ([#4587], thanks @absidue)
+* API: Return actual stream height, width and fps ([#4586], thanks @absidue)
+* Preferences: Fix handling of modified source code URL ([#4437], thanks @nooptek)
+* API: Fix URL for vtt subtitles ([#4221], thanks @karelrooted)
+* Channels: Add sort options to streams ([#4224], thanks @src-tinkerer)
+* API: Fix error code for disabled popular endpoint ([#4296], thanks @iBicha)
+* Allow embedding videos in local HTML files ([#4450], thanks @tomasz1986)
+* CI: Bump Crystal version matrix ([#4654], by @SamantazFox)
+* YtAPI: Remove API keys like official clients ([#4655], by @SamantazFox)
+* HTML: Use full URL in the og:image property ([#4675], thanks @Fijxu)
+* Rewrite transcript logic to be more generic ([#4747], thanks @syeopite)
+* CI: Run Ameba ([#4753], thanks @syeopite)
+* CI: Add release based containers ([#4763], thanks @syeopite)
+* move helm chart to a dedicated github repository ([#4711], thanks @unixfox)
+
+[#4146]: https://github.com/iv-org/invidious/pull/4146
+[#4153]: https://github.com/iv-org/invidious/pull/4153
+[#4221]: https://github.com/iv-org/invidious/pull/4221
+[#4224]: https://github.com/iv-org/invidious/pull/4224
+[#4295]: https://github.com/iv-org/invidious/pull/4295
+[#4296]: https://github.com/iv-org/invidious/pull/4296
+[#4437]: https://github.com/iv-org/invidious/pull/4437
+[#4450]: https://github.com/iv-org/invidious/pull/4450
+[#4586]: https://github.com/iv-org/invidious/pull/4586
+[#4587]: https://github.com/iv-org/invidious/pull/4587
+[#4654]: https://github.com/iv-org/invidious/pull/4654
+[#4655]: https://github.com/iv-org/invidious/pull/4655
+[#4659]: https://github.com/iv-org/invidious/pull/4659
+[#4667]: https://github.com/iv-org/invidious/pull/4667
+[#4675]: https://github.com/iv-org/invidious/pull/4675
+[#4695]: https://github.com/iv-org/invidious/pull/4695
+[#4696]: https://github.com/iv-org/invidious/pull/4696
+[#4706]: https://github.com/iv-org/invidious/pull/4706
+[#4711]: https://github.com/iv-org/invidious/pull/4711
+[#4717]: https://github.com/iv-org/invidious/pull/4717
+[#4731]: https://github.com/iv-org/invidious/pull/4731
+[#4747]: https://github.com/iv-org/invidious/pull/4747
+[#4753]: https://github.com/iv-org/invidious/pull/4753
+[#4763]: https://github.com/iv-org/invidious/pull/4763
+[#4772]: https://github.com/iv-org/invidious/pull/4772
+[#4785]: https://github.com/iv-org/invidious/pull/4785
+[#4789]: https://github.com/iv-org/invidious/pull/4789
+[#4790]: https://github.com/iv-org/invidious/pull/4790
+[#4792]: https://github.com/iv-org/invidious/pull/4792
+[#4795]: https://github.com/iv-org/invidious/pull/4795
+[#4796]: https://github.com/iv-org/invidious/pull/4796
+[#4805]: https://github.com/iv-org/invidious/pull/4805
+[#4806]: https://github.com/iv-org/invidious/pull/4806
+[#4807]: https://github.com/iv-org/invidious/pull/4807
+[#4812]: https://github.com/iv-org/invidious/pull/4812
+[#4845]: https://github.com/iv-org/invidious/pull/4845
+[#4849]: https://github.com/iv-org/invidious/pull/4849
+[#4852]: https://github.com/iv-org/invidious/pull/4852
+[#4853]: https://github.com/iv-org/invidious/pull/4853
+[#4859]: https://github.com/iv-org/invidious/pull/4859
+[#4876]: https://github.com/iv-org/invidious/pull/4876
+
+
+## v2.20240427 (2024-04-27)
 
 Major bug fixes:
  * Videos: Use android test suite client (#4650, thanks @SamantazFox)

src/invidious/hashtag.cr

@@ -1,42 +1,89 @@
 module Invidious::Hashtag
   extend self
 
-  def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
+  struct HashtagPage
+    include DB::Serializable
+
+    property videos : Array(SearchItem) | Array(Video)
+    property header : SearchHashtag?
+    property has_next_continuation : Bool
+
+    def to_json(locale : String?, json : JSON::Builder)
+      json.object do
+        json.field "type", "hashtagPage"
+        if self.header != nil
+          json.field "header" do
+            self.header.try &.as(SearchHashtag).to_json(locale, json)
+          end
+        end
+        json.field "results" do
+          json.array do
+            self.videos.each do |item|
+              item.to_json(locale, json)
+            end
+          end
+        end
+        json.field "hasNextPage", self.has_next_continuation
+      end
+    end
+  end
+
+  def fetch(hashtag : String, page : Int, region : String? = nil) : HashtagPage
     cursor = (page - 1) * 60
-    ctoken = generate_continuation(hashtag, cursor)
+    header = nil
-
     client_config = YoutubeAPI::ClientConfig.new(region: region)
-    response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
+    item = generate_continuation(hashtag, cursor)
+    # item is a ctoken
+    if cursor > 0
+      response = YoutubeAPI.browse(continuation: item, client_config: client_config)
+    else
+      # item browses the first page (including metadata)
+      response = YoutubeAPI.browse("FEhashtag", params: item, client_config: client_config)
+      if item_contents = response.dig?("header")
+        header = parse_item(item_contents).try &.as(SearchHashtag)
+      end
+    end
 
-    items, _ = extract_items(response)
+    items, next_continuation = extract_items(response)
-    return items
+    return HashtagPage.new({
+      videos:                items,
+      header:                header,
+      has_next_continuation: next_continuation != nil,
+    })
   end
 
   def generate_continuation(hashtag : String, cursor : Int)
     object = {
-      "80226972:embedded" => {
+      "93:2:embedded" => {
-        "2:string" => "FEhashtag",
+        "1:string" => hashtag,
-        "3:base64" => {
+        "2:varint" => 0_i64,
-          "1:varint"  => 60_i64, # result count
+        "3:varint" => 1_i64,
-          "15:base64" => {
-            "1:varint" => cursor.to_i64,
-            "2:varint" => 0_i64,
-          },
-          "93:2:embedded" => {
-            "1:string" => hashtag,
-            "2:varint" => 0_i64,
-            "3:varint" => 1_i64,
-          },
-        },
-        "35:string" => "browse-feedFEhashtag",
       },
     }
+    if cursor > 0
+      object = {
+        "80226972:embedded" => {
+          "2:string" => "FEhashtag",
+          "3:base64" => {
+            "1:varint"  => 60_i64, # result count
+            "15:base64" => {
+              "1:varint" => cursor.to_i64,
+              "2:varint" => 0_i64,
+            },
+            "93:2:embedded" => {
+              "1:string" => hashtag,
+              "2:varint" => 0_i64,
+              "3:varint" => 1_i64,
+            },
+          },
+          "35:string" => "browse-feedFEhashtag",
+        },
+      }
+    end
 
-    continuation = object.try { |i| Protodec::Any.cast_json(i) }
+    return object.try { |i| Protodec::Any.cast_json(i) }
       .try { |i| Protodec::Any.from_json(i) }
       .try { |i| Base64.urlsafe_encode(i) }
       .try { |i| URI.encode_www_form(i) }
-
-    return continuation
   end
 end

src/invidious/routes/api/v1/search.cr

@@ -69,21 +69,13 @@ module Invidious::Routes::API::V1::Search
     env.response.content_type = "application/json"
 
     begin
-      results = Invidious::Hashtag.fetch(hashtag, page, region)
+      hashtag_page = Invidious::Hashtag.fetch(hashtag, page, region)
     rescue ex
       return error_json(400, ex)
     end
 
     JSON.build do |json|
-      json.object do
+      hashtag_page.to_json(locale, json)
-        json.field "results" do
-          json.array do
-            results.each do |item|
-              item.to_json(locale, json)
-            end
-          end
-        end
-      end
     end
   end
 end

src/invidious/routes/search.cr

@@ -101,7 +101,8 @@ module Invidious::Routes::Search
     end
 
     begin
-      items = Invidious::Hashtag.fetch(hashtag, page)
+      hashtag_page = Invidious::Hashtag.fetch(hashtag, page)
+      items = hashtag_page.videos
     rescue ex
       return error_template(500, ex)
     end
@@ -111,7 +112,7 @@ module Invidious::Routes::Search
     page_nav_html = Frontend::Pagination.nav_numeric(locale,
       base_url: "/hashtag/#{hashtag_encoded}",
       current_page: page,
-      show_next: (items.size >= 60)
+      show_next: hashtag_page.has_next_continuation
     )
 
     templated "hashtag"

src/invidious/videos/parser.cr

@@ -53,6 +53,10 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  if CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
@@ -102,6 +106,13 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
+  # Second try in case WEB_CREATOR doesn't work with po_token.
+  # Only trigger if reason found and po_token configured.
+  if reason && CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
   if reason.nil? && CONFIG.po_token.nil?
@@ -114,10 +125,9 @@ def extract_video_info(video_id : String)
   end
 
   # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
+  # Only trigger if reason found or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
+  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
-  # if the IP address is not blocked.
+  if reason && CONFIG.po_token.nil?
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -185,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
-  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+  if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+    microformat = {} of String => JSON::Any
+  end
 
   raise BrokenTubeException.new("videoDetails") if !video_details
-  raise BrokenTubeException.new("microformat") if !microformat
 
   # Basic video infos
 
@@ -225,7 +236,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
-  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  family_friendly = microformat["isFamilySafe"]?.try &.as_bool
   is_listed = video_details["isCrawlable"]?.try &.as_bool
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 

src/invidious/yt_backend/extractors.cr

@@ -217,15 +217,18 @@ private module Parsers
   #
   # A `hashtagTileRenderer` is a kind of search result.
   # It can be found when searching for any hashtag (e.g "#hi" or "#shorts")
+  #
+  # A `hashtagHeaderRenderer` is displayed on the first page of the hashtag page.
   module HashtagRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
-      if item_contents = item["hashtagTileRenderer"]?
+      if item_contents = (item["hashtagTileRenderer"]? || item["hashtagHeaderRenderer"]? || item["pageHeaderRenderer"]?)
         return self.parse(item_contents)
       end
     end
 
     private def self.parse(item_contents)
-      title = extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi"
+      title = item_contents.dig?("pageTitle").try &.as_s
+      title ||= extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi"
 
       # E.g "/hashtag/hi"
       url = item_contents.dig?("onTapCommand", "commandMetadata", "webCommandMetadata", "url").try &.as_s
@@ -236,8 +239,10 @@ private module Parsers
 
       # Fallback for video/channel counts
       if channel_count_txt.nil? || video_count_txt.nil?
+        info_text = (item_contents.dig?("content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0, "text", "content").try &.as_s ||
+                     extract_text(item_contents.dig?("hashtagInfoText"))).try &.split(" • ")
+
         # E.g: "203K videos • 81K channels"
-        info_text = extract_text(item_contents["hashtagInfoText"]?).try &.split(" • ")
 
         if info_text && info_text.size == 2
           video_count_txt ||= info_text[0]

src/invidious/yt_backend/youtube_api.cr

@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -80,6 +81,14 @@ module YoutubeAPI
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20240918.03.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -291,8 +300,9 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
+      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+        "embedUrl" => "https://www.google.com/",
       } of String => String | Int64
     end