Git-Mirrors/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-08-03 20:14:13 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #2956 from SamantazFox/search-filters

Browse source 
Overhaul search filters
This commit is contained in:
Samantaz Fox 2022-04-08 23:49:03 +02:00 • committed by GitHub
commit 4900ce24fa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 2529 additions and 1510 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/invidious.cr
View file

@ -35,6 +35,7 @@ require "./invidious/frontend/*"
require "./invidious/*"
require "./invidious/channels/*"
require "./invidious/user/*"
require "./invidious/search/*"
require "./invidious/routes/**"
require "./invidious/jobs/**"

8
src/invidious/exceptions.cr
View file

@ -1,3 +1,11 @@
# Exception used to hold the bogus UCID during a channel search.
class ChannelSearchException < InfoException
getter channel : String
def initialize(@channel)
end
end
# Exception used to hold the name of the missing item
# Should be used in all parsing functions
class BrokenTubeException < Exception

14
src/invidious/frontend/misc.cr Normal file
View file

@ -0,0 +1,14 @@
module Invidious::Frontend::Misc
extend self
def redirect_url(env : HTTP::Server::Context)
prefs = env.get("preferences").as(Preferences)
if prefs.automatic_instance_redirect
current_page = env.get?("current_page").as(String)
redirect_url = "/redirect?referer=#{current_page}"
else
redirect_url = "https://redirect.invidious.io#{env.request.resource}"
end
end
end

135
src/invidious/frontend/search_filters.cr Normal file
View file

@ -0,0 +1,135 @@
module Invidious::Frontend::SearchFilters
extend self
# Generate the search filters collapsable widget.
def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String
return String.build(8000) do |str|
str << "<div id='filters'>\n"
str << "\t<details id='filters-collapse'>"
str << "\t\t<summary>" << translate(locale, "search_filters_title") << "</summary>\n"
str << "\t\t<div id='filters-box'><form action='/search' method='get'>\n"
str << "\t\t\t<input type='hidden' name='q' value='" << HTML.escape(query) << "'>\n"
str << "\t\t\t<input type='hidden' name='page' value='" << page << "'>\n"
str << "\t\t\t<div id='filters-flex'>"
filter_wrapper(date)
filter_wrapper(type)
filter_wrapper(duration)
filter_wrapper(features)
filter_wrapper(sort)
str << "\t\t\t</div>\n"
str << "\t\t\t<div id='filters-apply'>"
str << "<button type='submit' class=\"pure-button pure-button-primary\">"
str << translate(locale, "search_filters_apply_button")
str << "</button></div>\n"
str << "\t\t</form></div>\n"
str << "\t</details>\n"
str << "</div>\n"
end
end
# Generate wrapper HTML (`<div>`, filter name, etc...) around the
# `<input>` elements of a search filter
macro filter_wrapper(name)
str << "\t\t\t\t<div class=\"filter-column\"><fieldset>\n"
str << "\t\t\t\t\t<legend><div class=\"filter-name underlined\">"
str << translate(locale, "search_filters_{{name}}_label")
str << "</div></legend>\n"
str << "\t\t\t\t\t<div class=\"filter-options\">\n"
make_{{name}}_filter_options(str, filters.{{name}}, locale)
str << "\t\t\t\t\t</div>"
str << "\t\t\t\t</fieldset></div>\n"
end
# Generates the HTML for the list of radio buttons of the "date" search filter
def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String)
{% for value in Invidious::Search::Filters::Date.constants %}
{% date = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='date' id='filter-date-{{date}}' value='{{date}}'"
str << " checked" if value.{{date}}?
str << '>'
str << "<label for='filter-date-{{date}}'>"
str << translate(locale, "search_filters_date_option_{{date}}")
str << "</label></div>\n"
{% end %}
end
# Generates the HTML for the list of radio buttons of the "type" search filter
def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String)
{% for value in Invidious::Search::Filters::Type.constants %}
{% type = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='type' id='filter-type-{{type}}' value='{{type}}'"
str << " checked" if value.{{type}}?
str << '>'
str << "<label for='filter-type-{{type}}'>"
str << translate(locale, "search_filters_type_option_{{type}}")
str << "</label></div>\n"
{% end %}
end
# Generates the HTML for the list of radio buttons of the "duration" search filter
def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String)
{% for value in Invidious::Search::Filters::Duration.constants %}
{% duration = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='duration' id='filter-duration-{{duration}}' value='{{duration}}'"
str << " checked" if value.{{duration}}?
str << '>'
str << "<label for='filter-duration-{{duration}}'>"
str << translate(locale, "search_filters_duration_option_{{duration}}")
str << "</label></div>\n"
{% end %}
end
# Generates the HTML for the list of checkboxes of the "features" search filter
def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String)
{% for value in Invidious::Search::Filters::Features.constants %}
{% if value.stringify != "All" && value.stringify != "None" %}
{% feature = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='checkbox' name='features' id='filter-features-{{feature}}' value='{{feature}}'"
str << " checked" if value.{{feature}}?
str << '>'
str << "<label for='filter-feature-{{feature}}'>"
str << translate(locale, "search_filters_features_option_{{feature}}")
str << "</label></div>\n"
{% end %}
{% end %}
end
# Generates the HTML for the list of radio buttons of the "sort" search filter
def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String)
{% for value in Invidious::Search::Filters::Sort.constants %}
{% sort = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='sort' id='filter-sort-{{sort}}' value='{{sort}}'"
str << " checked" if value.{{sort}}?
str << '>'
str << "<label for='filter-sort-{{sort}}'>"
str << translate(locale, "search_filters_sort_option_{{sort}}")
str << "</label></div>\n"
{% end %}
end
end

16
src/invidious/routes/api/v1/channels.cr
View file

@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
query = Invidious::Search::Query.new(env.params.query, :channel, region)
query = env.params.query["q"]?
query ||= ""
# Required because we can't (yet) pass multiple parameter to the
# `Search::Query` initializer (in this case, an URL segment)
query.channel = env.params.url["ucid"]
page = env.params.query["page"]?.try &.to_i?
page ||= 1
begin
search_results = query.process
rescue ex
return error_json(400, ex)
end
search_results = channel_search(query, page, ucid)
JSON.build do |json|
json.array do
search_results.each do |item|

24
src/invidious/routes/api/v1/search.cr
View file

@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search
env.response.content_type = "application/json"
query = env.params.query["q"]?
query ||= ""
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "relevance"
date = env.params.query["date"]?.try &.downcase
date ||= ""
duration = env.params.query["duration"]?.try &.downcase
duration ||= ""
features = env.params.query["features"]?.try &.split(",").map(&.downcase)
features ||= [] of String
content_type = env.params.query["type"]?.try &.downcase
content_type ||= "video"
query = Invidious::Search::Query.new(env.params.query, :regular, region)
begin
search_params = produce_search_params(page, sort_by, date, content_type, duration, features)
search_results = query.process
rescue ex
return error_json(400, ex)
end
search_results = search(query, search_params, region)
JSON.build do |json|
json.array do
search_results.each do |item|

18
src/invidious/routes/playlists.cr
View file

@ -212,7 +212,10 @@ module Invidious::Routes::Playlists
end
def self.add_playlist_items_page(env)
locale = env.get("preferences").as(Preferences).locale
prefs = env.get("preferences").as(Preferences)
locale = prefs.locale
region = env.params.query["region"]? || prefs.region
user = env.get? "user"
sid = env.get? "sid"
@ -236,15 +239,10 @@ module Invidious::Routes::Playlists
return env.redirect referer
end
query = env.params.query["q"]?
if query
begin
search_query, items, operators = process_search_query(query, page, user, region: nil)
videos = items.select(SearchVideo).map(&.as(SearchVideo))
rescue ex
videos = [] of SearchVideo
end
else
begin
query = Invidious::Search::Query.new(env.params.query, :playlist, region)
videos = query.process.select(SearchVideo).map(&.as(SearchVideo))
rescue ex
videos = [] of SearchVideo
end

24
src/invidious/routes/search.cr
View file

@ -37,37 +37,29 @@ module Invidious::Routes::Search
end
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
prefs = env.get("preferences").as(Preferences)
locale = prefs.locale
query = env.params.query["search_query"]?
query ||= env.params.query["q"]?
region = env.params.query["region"]? || prefs.region
if !query || query.empty?
query = Invidious::Search::Query.new(env.params.query, :regular, region)
if query.empty?
# Display the full page search box implemented in #1977
env.set "search", ""
templated "search_homepage", navbar_search: false
else
page = env.params.query["page"]?.try &.to_i?
page ||= 1
user = env.get? "user"
begin
search_query, videos, operators = process_search_query(query, page, user, region: region)
videos = query.process
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
return error_template(500, ex)
end
operator_hash = {} of String => String
operators.each do |operator|
key, value = operator.downcase.split(":")
operator_hash[key] = value
end
env.set "search", query
env.set "search", query.text
templated "search"
end
end

254
src/invidious/search.cr
View file

@ -1,254 +0,0 @@
class ChannelSearchException < InfoException
getter channel : String
def initialize(@channel)
end
end
def channel_search(query, page, channel) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{channel}")
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{channel}")
response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404
initial_data = extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(channel) if !ucid
else
ucid = channel
end
continuation = produce_channel_search_continuation(ucid, query, page)
response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
return items
end
def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem)
return [] of SearchItem if query.empty?
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
return extract_items(initial_data)
end
def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "",
duration : String = "", features : Array(String) = [] of String)
object = {
"1:varint" => 0_i64,
"2:embedded" => {} of String => Int64,
"9:varint" => ((page - 1) * 20).to_i64,
}
case sort
when "relevance"
object["1:varint"] = 0_i64
when "rating"
object["1:varint"] = 1_i64
when "upload_date", "date"
object["1:varint"] = 2_i64
when "view_count", "views"
object["1:varint"] = 3_i64
else
raise "No sort #{sort}"
end
case date
when "hour"
object["2:embedded"].as(Hash)["1:varint"] = 1_i64
when "today"
object["2:embedded"].as(Hash)["1:varint"] = 2_i64
when "week"
object["2:embedded"].as(Hash)["1:varint"] = 3_i64
when "month"
object["2:embedded"].as(Hash)["1:varint"] = 4_i64
when "year"
object["2:embedded"].as(Hash)["1:varint"] = 5_i64
else nil # Ignore
end
case content_type
when "video"
object["2:embedded"].as(Hash)["2:varint"] = 1_i64
when "channel"
object["2:embedded"].as(Hash)["2:varint"] = 2_i64
when "playlist"
object["2:embedded"].as(Hash)["2:varint"] = 3_i64
when "movie"
object["2:embedded"].as(Hash)["2:varint"] = 4_i64
when "show"
object["2:embedded"].as(Hash)["2:varint"] = 5_i64
when "all"
#
else
object["2:embedded"].as(Hash)["2:varint"] = 1_i64
end
case duration
when "short"
object["2:embedded"].as(Hash)["3:varint"] = 1_i64
when "long"
object["2:embedded"].as(Hash)["3:varint"] = 2_i64
else nil # Ignore
end
features.each do |feature|
case feature
when "hd"
object["2:embedded"].as(Hash)["4:varint"] = 1_i64
when "subtitles"
object["2:embedded"].as(Hash)["5:varint"] = 1_i64
when "creative_commons", "cc"
object["2:embedded"].as(Hash)["6:varint"] = 1_i64
when "3d"
object["2:embedded"].as(Hash)["7:varint"] = 1_i64
when "live", "livestream"
object["2:embedded"].as(Hash)["8:varint"] = 1_i64
when "purchased"
object["2:embedded"].as(Hash)["9:varint"] = 1_i64
when "4k"
object["2:embedded"].as(Hash)["14:varint"] = 1_i64
when "360"
object["2:embedded"].as(Hash)["15:varint"] = 1_i64
when "location"
object["2:embedded"].as(Hash)["23:varint"] = 1_i64
when "hdr"
object["2:embedded"].as(Hash)["25:varint"] = 1_i64
else nil # Ignore
end
end
if object["2:embedded"].as(Hash).empty?
object.delete("2:embedded")
end
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return params
end
def produce_channel_search_continuation(ucid, query, page)
if page <= 1
idx = 0_i64
else
idx = 30_i64 * (page - 1)
end
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"15:base64" => {
"3:varint" => idx,
},
"23:varint" => 0_i64,
},
"11:string" => query,
"35:string" => "browse-feed#{ucid}search",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def process_search_query(query, page, user, region)
if user
user = user.as(Invidious::User)
view_name = "subscriptions_#{sha256(user.email)}"
end
channel = nil
content_type = "all"
date = ""
duration = ""
features = [] of String
sort = "relevance"
subscriptions = nil
operators = query.split(" ").select(&.match(/\w+:[\w,]+/))
operators.each do |operator|
key, value = operator.downcase.split(":")
case key
when "channel", "user"
channel = operator.split(":")[-1]
when "content_type", "type"
content_type = value
when "date"
date = value
when "duration"
duration = value
when "feature", "features"
features = value.split(",")
when "sort"
sort = value
when "subscriptions"
subscriptions = value == "true"
else
operators.delete(operator)
end
end
search_query = (query.split(" ") - operators).join(" ")
if channel
items = channel_search(search_query, page, channel)
elsif subscriptions
if view_name
items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM (
SELECT *,
to_tsvector(#{view_name}.title) ||
to_tsvector(#{view_name}.author)
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo)
else
items = [] of ChannelVideo
end
else
search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type,
duration: duration, features: features)
items = search(search_query, search_params, region)
end
# Light processing to flatten search results out of Categories.
# They should ideally be supported in the future.
items_without_category = [] of SearchItem | ChannelVideo
items.each do |i|
if i.is_a? Category
i.contents.each do |nest_i|
if !nest_i.is_a? Video
items_without_category << nest_i
end
end
else
items_without_category << i
end
end
{search_query, items_without_category, operators}
end

32
src/invidious/search/ctoken.cr Normal file
View file

@ -0,0 +1,32 @@
def produce_channel_search_continuation(ucid, query, page)
if page <= 1
idx = 0_i64
else
idx = 30_i64 * (page - 1)
end
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"15:base64" => {
"3:varint" => idx,
},
"23:varint" => 0_i64,
},
"11:string" => query,
"35:string" => "browse-feed#{ucid}search",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end

376
src/invidious/search/filters.cr Normal file
View file

@ -0,0 +1,376 @@
require "protodec/utils"
require "http/params"
module Invidious::Search
struct Filters
# Values correspond to { "2:embedded": { "1:varint": <X> }}
# except for "None" which is only used by us (= nothing selected)
enum Date
None = 0
Hour = 1
Today = 2
Week = 3
Month = 4
Year = 5
end
# Values correspond to { "2:embedded": { "2:varint": <X> }}
# except for "All" which is only used by us (= nothing selected)
enum Type
All = 0
Video = 1
Channel = 2
Playlist = 3
Movie = 4
# Has it been removed?
# (Not available on youtube's UI)
Show = 5
end
# Values correspond to { "2:embedded": { "3:varint": <X> }}
# except for "None" which is only used by us (= nothing selected)
enum Duration
None = 0
Short = 1 # "Under 4 minutes"
Long = 2 # "Over 20 minutes"
Medium = 3 # "4 - 20 minutes"
end
# Note: flag enums automatically generate
# "none" and "all" members
@[Flags]
enum Features
Live
FourK # "4K"
HD
Subtitles # "Subtitles/CC"
CCommons # "Creative Commons"
ThreeSixty # "360°"
VR180
ThreeD # "3D"
HDR
Location
Purchased
end
# Values correspond to { "1:varint": <X> }
enum Sort
Relevance = 0
Rating = 1
Date = 2
Views = 3
end
# Parameters are sorted as on Youtube
property date : Date
property type : Type
property duration : Duration
property features : Features
property sort : Sort
def initialize(
*, # All parameters must be named
@date : Date = Date::None,
@type : Type = Type::All,
@duration : Duration = Duration::None,
@features : Features = Features::None,
@sort : Sort = Sort::Relevance
)
end
def default? : Bool
return @date.none? && @type.all? && @duration.none? && \
@features.none? && @sort.relevance?
end
# -------------------
# Invidious params
# -------------------
def self.parse_features(raw : Array(String)) : Features
# Initialize return variable
features = Features.new(0)
raw.each do |ft|
case ft.downcase
when "live", "livestream"
features = features | Features::Live
when "4k" then features = features | Features::FourK
when "hd" then features = features | Features::HD
when "subtitles" then features = features | Features::Subtitles
when "creative_commons", "commons", "cc"
features = features | Features::CCommons
when "360" then features = features | Features::ThreeSixty
when "vr180" then features = features | Features::VR180
when "3d" then features = features | Features::ThreeD
when "hdr" then features = features | Features::HDR
when "location" then features = features | Features::Location
when "purchased" then features = features | Features::Purchased
end
end
return features
end
def self.format_features(features : Features) : String
# Directly return an empty string if there are no features
return "" if features.none?
# Initialize return variable
str = [] of String
str << "live" if features.live?
str << "4k" if features.four_k?
str << "hd" if features.hd?
str << "subtitles" if features.subtitles?
str << "commons" if features.c_commons?
str << "360" if features.three_sixty?
str << "vr180" if features.vr180?
str << "3d" if features.three_d?
str << "hdr" if features.hdr?
str << "location" if features.location?
str << "purchased" if features.purchased?
return str.join(',')
end
def self.from_legacy_filters(str : String) : {Filters, String, String, Bool}
# Split search query on spaces
members = str.split(' ')
# Output variables
channel = ""
filters = Filters.new
subscriptions = false
# Array to hold the non-filter members
query = [] of String
# Parse!
members.each do |substr|
# Separator operators
operators = substr.split(':')
case operators[0]
when "user", "channel"
next if operators.size != 2
channel = operators[1]
#
when "type", "content_type"
next if operators.size != 2
type = Type.parse?(operators[1])
filters.type = type if !type.nil?
#
when "date"
next if operators.size != 2
date = Date.parse?(operators[1])
filters.date = date if !date.nil?
#
when "duration"
next if operators.size != 2
duration = Duration.parse?(operators[1])
filters.duration = duration if !duration.nil?
#
when "feature", "features"
next if operators.size != 2
features = parse_features(operators[1].split(','))
filters.features = features if !features.nil?
#
when "sort"
next if operators.size != 2
sort = Sort.parse?(operators[1])
filters.sort = sort if !sort.nil?
#
when "subscriptions"
next if operators.size != 2
subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1])
#
else
query << substr
end
end
# Re-assemble query (without filters)
cleaned_query = query.join(' ')
return {filters, channel, cleaned_query, subscriptions}
end
def self.from_iv_params(params : HTTP::Params) : Filters
# Temporary variables
filters = Filters.new
if type = params["type"]?
filters.type = Type.parse?(type) || Type::All
params.delete("type")
end
if date = params["date"]?
filters.date = Date.parse?(date) || Date::None
params.delete("date")
end
if duration = params["duration"]?
filters.duration = Duration.parse?(duration) || Duration::None
params.delete("duration")
end
features = params.fetch_all("features")
if !features.empty?
# Un-array input so it can be treated as a comma-separated list
features = features[0].split(',') if features.size == 1
filters.features = parse_features(features) || Features::None
params.delete_all("features")
end
if sort = params["sort"]?
filters.sort = Sort.parse?(sort) || Sort::Relevance
params.delete("sort")
end
return filters
end
def to_iv_params : HTTP::Params
# Temporary variables
raw_params = {} of String => Array(String)
raw_params["date"] = [@date.to_s.underscore] if !@date.none?
raw_params["type"] = [@type.to_s.underscore] if !@type.all?
raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance?
if !@duration.none?
raw_params["duration"] = [@duration.to_s.underscore]
end
if !@features.none?
raw_params["features"] = [Filters.format_features(@features)]
end
return HTTP::Params.new(raw_params)
end
# -------------------
# Youtube params
# -------------------
# Produce the youtube search parameters for the
# innertube API (base64-encoded protobuf object).
def to_yt_params(page : Int = 1) : String
# Initialize the embedded protobuf object
embedded = {} of String => Int64
# Add these field only if associated parameter is selected
embedded["1:varint"] = @date.to_i64 if !@date.none?
embedded["2:varint"] = @type.to_i64 if !@type.all?
embedded["3:varint"] = @duration.to_i64 if !@duration.none?
if !@features.none?
# All features have a value of "1" when enabled, and
# the field is omitted when the feature is no selected.
embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD)
embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles)
embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons)
embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD)
embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live)
embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased)
embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK)
embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty)
embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location)
embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR)
embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180)
end
# Initialize an empty protobuf object
object = {} of String => (Int64 | String | Hash(String, Int64))
# As usual, everything can be omitted if it has no value
object["2:embedded"] = embedded if !embedded.empty?
# Default sort is "relevance", so when this option is selected,
# the associated field can be omitted.
if !@sort.relevance?
object["1:varint"] = @sort.to_i64
end
# Add page number (if provided)
if page > 1
object["9:varint"] = ((page - 1) * 20).to_i64
end
# If the object is empty, return an empty string,
# otherwise encode to protobuf then to base64
return "" if object.empty?
return object
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
end
# Function to parse the `sp` URL parameter from Youtube
# search page. It's a base64-encoded protobuf object.
def self.from_yt_params(params : HTTP::Params) : Filters
# Initialize output variable
filters = Filters.new
# Get parameter, and check emptyness
search_params = params["sp"]?
if search_params.nil? || search_params.empty?
return filters
end
# Decode protobuf object
object = search_params
.try { |i| URI.decode_www_form(i) }
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
# Parse items from embedded object
if embedded = object["2:0:embedded"]?
# All the following fields (date, type, duration) are optional.
if date = embedded["1:0:varint"]?
filters.date = Date.from_value?(date.as_i) || Date::None
end
if type = embedded["2:0:varint"]?
filters.type = Type.from_value?(type.as_i) || Type::All
end
if duration = embedded["3:0:varint"]?
filters.duration = Duration.from_value?(duration.as_i) || Duration::None
end
# All features should have a value of "1" when enabled, and
# the field should be omitted when the feature is no selected.
features = 0
features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0
features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0
features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0
features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0
features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0
features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0
features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0
features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0
features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0
features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0
features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0
filters.features = Features.from_value?(features) || Features::None
end
if sort = object["1:0:varint"]?
filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance
end
# Remove URL parameter and return result
params.delete("sp")
return filters
end
end
end

64
src/invidious/search/processors.cr Normal file
View file

@ -0,0 +1,64 @@
module Invidious::Search
module Processors
extend self
# Regular search (`/search` endpoint)
def regular(query : Query) : Array(SearchItem)
search_params = query.filters.to_yt_params(page: query.page)
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
return extract_items(initial_data)
end
# Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{query.channel}")
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
initial_data = extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
else
ucid = query.channel
end
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
return items
end
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
SELECT *,
to_tsvector(#{view_name}.title) ||
to_tsvector(#{view_name}.author)
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end
end
end

148
src/invidious/search/query.cr Normal file
View file

@ -0,0 +1,148 @@
module Invidious::Search
class Query
enum Type
# Types related to YouTube
Regular # Youtube search page
Channel # Youtube channel search box
# Types specific to Invidious
Subscriptions # Search user subscriptions
Playlist # "Add playlist item" search
end
@type : Type = Type::Regular
@raw_query : String
@query : String = ""
property filters : Filters = Filters.new
property page : Int32
property region : String?
property channel : String = ""
# Return true if @raw_query is either `nil` or empty
private def empty_raw_query?
return @raw_query.empty?
end
# Same as `empty_raw_query?`, but named for external use
def empty?
return self.empty_raw_query?
end
# Getter for the query string.
# It is named `text` to reduce confusion (`search_query.text` makes more
# sense than `search_query.query`)
def text
return @query
end
# Initialize a new search query.
# Parameters are used to get the query string, the page number
# and the search filters (if any). Type tells this function
# where it is being called from (See `Type` above).
def initialize(
params : HTTP::Params,
@type : Type = Type::Regular,
@region : String? = nil
)
# Get the raw search query string (common to all search types). In
# Regular search mode, also look for the `search_query` URL parameter
if @type.regular?
@raw_query = params["q"]? || params["search_query"]? || ""
else
@raw_query = params["q"]? || ""
end
# Get the page number (also common to all search types)
@page = params["page"]?.try &.to_i? || 1
# Stop here is raw query in empty
# NOTE: maybe raise in the future?
return if self.empty_raw_query?
# Specific handling
case @type
when .playlist?, .channel?
# In "add playlist item" mode, filters are parsed from the query
# string itself (legacy), and the channel is ignored.
#
# In "channel search" mode, filters are ignored, but we still parse
# the query prevent transmission of legacy filters to youtube.
#
@filters, @query, @channel, _ = Filters.from_legacy_filters(@raw_query || "")
#
when .subscriptions?, .regular?
if params["sp"]?
# Parse the `sp` URL parameter (youtube compatibility)
@filters = Filters.from_yt_params(params)
@query = @raw_query || ""
else
# Parse invidious URL parameters (sort, date, etc...)
@filters = Filters.from_iv_params(params)
@channel = params["channel"]? || ""
if @filters.default? && @raw_query.includes?(':')
# Parse legacy filters from query
@filters, @query, @channel, subs = Filters.from_legacy_filters(@raw_query || "")
else
@query = @raw_query || ""
end
if !@channel.empty?
# Switch to channel search mode (filters will be ignored)
@type = Type::Channel
elsif subs
# Switch to subscriptions search mode
@type = Type::Subscriptions
end
end
end
end
# Run the search query using the corresponding search processor.
# Returns either the results or an empty array of `SearchItem`.
def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo)
items = [] of SearchItem
# Don't bother going further if search query is empty
return items if self.empty_raw_query?
case @type
when .regular?, .playlist?
items = unnest_items(Processors.regular(self))
#
when .channel?
items = Processors.channel(self)
#
when .subscriptions?
if user
items = Processors.subscriptions(self, user.as(Invidious::User))
end
end
return items
end
# TODO: clean code
private def unnest_items(all_items) : Array(SearchItem)
items = [] of SearchItem
# Light processing to flatten search results out of Categories.
# They should ideally be supported in the future.
all_items.each do |i|
if i.is_a? Category
i.contents.each do |nest_i|
if !nest_i.is_a? Video
items << nest_i
end
end
else
items << i
end
end
return items
end
end
end

11
src/invidious/views/add_playlist_items.ecr
View file

@ -11,7 +11,9 @@
<legend><a href="/playlist?list=<%= playlist.id %>"><%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %></a></legend>
<fieldset>
<input class="pure-input-1" type="search" name="q" <% if query %>value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>>
<input class="pure-input-1" type="search" name="q"
<% if query %>value="<%= HTML.escape(query.text) %>"<% end %>
placeholder="<%= translate(locale, "Search for videos") %>">
<input type="hidden" name="list" value="<%= plid %>">
</fieldset>
</form>
@ -38,10 +40,11 @@
</div>
<% if query %>
<%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page - 1 %>">
<% if query.page > 1 %>
<a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page - 1 %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
@ -49,7 +52,7 @@
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if videos.size >= 20 %>
<a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>">
<a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>

163
src/invidious/views/search.ecr
View file

@ -1,147 +1,62 @@
<% content_for "header" do %>
<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title>
<title><%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "&hellip;" : HTML.escape(query.text) %> - Invidious</title>
<link rel="stylesheet" href="/css/search.css?v=<%= ASSET_COMMIT %>">
<% end %>
<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %>
<%-
search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true)
filter_params = query.filters.to_iv_params
url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}"
url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-%>
<!-- Search redirection and filtering UI -->
<% if videos.size == 0 %>
<h3 style="text-align: center">
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a>
</h3>
<% else %>
<details id="filters">
<summary>
<h3 style="display:inline"> <%= translate(locale, "filter") %> </h3>
</summary>
<div id="filters" class="pure-g h-box">
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "date") %></b>
<hr/>
<% ["hour", "today", "week", "month", "year"].each do |date| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("date", "all") == date %>
<b><%= translate(locale, date) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>">
<%= translate(locale, date) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "content_type") %></b>
<hr/>
<% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("content_type", "all") == content_type %>
<b><%= translate(locale, content_type) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>">
<%= translate(locale, content_type) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "duration") %></b>
<hr/>
<% ["short", "long"].each do |duration| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("duration", "all") == duration %>
<b><%= translate(locale, duration) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>">
<%= translate(locale, duration) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "features") %></b>
<hr/>
<% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("features", "all").includes?(feature) %>
<b><%= translate(locale, feature) %></b>
<% elsif operator_hash.has_key?("features") %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>">
<%= translate(locale, feature) %>
</a>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil! + " features:" + feature) %>&page=<%= page %>">
<%= translate(locale, feature) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "sort") %></b>
<hr/>
<% ["relevance", "rating", "date", "views"].each do |sort| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("sort", "relevance") == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>">
<%= translate(locale, sort) %>
</a>
<% end %>
</div>
<% end %>
</div>
</div>
</details>
<% end %>
<% if videos.size == 0 %>
<hr style="margin: 0;"/>
<% else %>
<hr/>
<% end %>
<%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %>
<hr/>
<div class="pure-g h-box v-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
<%- if query.page > 1 -%>
<a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
<%- end -%>
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if videos.size >= 20 %>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
<%- if videos.size >= 20 -%>
<a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
<%- end -%>
</div>
</div>
<div class="pure-g">
<% videos.each do |item| %>
<%= rendered "components/item" %>
<% end %>
<%- if videos.empty? -%>
<div class="h-box no-results-error">
<div>
<%= translate(locale, "search_message_no_results") %><br/><br/>
<%= translate(locale, "search_message_change_filters_or_query") %><br/><br/>
<%= translate(locale, "search_message_use_another_instance", redirect_url) %>
</div>
</div>
<%- else -%>
<div class="pure-g">
<%- videos.each do |item| -%>
<%= rendered "components/item" %>
<%- end -%>
</div>
<%- end -%>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
<%- if query.page > 1 -%>
<a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
<%- end -%>
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if videos.size >= 20 %>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
<%- if videos.size >= 20 -%>
<a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
<%- end -%>
</div>
</div>