Cleanup channel helpers code

This commit is contained in:
matthewmcgarvey 2022-01-21 21:27:50 -06:00
parent 508f137b30
commit a82d21ff78
7 changed files with 20 additions and 31 deletions

View File

@ -114,8 +114,9 @@ class ChannelRedirect < Exception
end end
end end
def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10) def get_batch_channels(channels)
finished_channel = Channel(String | Nil).new finished_channel = Channel(String | Nil).new
max_threads = 10
spawn do spawn do
active_threads = 0 active_threads = 0
@ -130,7 +131,7 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th
active_threads += 1 active_threads += 1
spawn do spawn do
begin begin
get_channel(ucid, refresh, pull_all_videos) get_channel(ucid)
finished_channel.send(ucid) finished_channel.send(ucid)
rescue ex rescue ex
finished_channel.send(nil) finished_channel.send(nil)
@ -151,23 +152,18 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th
return final return final
end end
def get_channel(id, refresh = true, pull_all_videos = true) def get_channel(id)
if channel = Invidious::Database::Channels.select(id) channel = Invidious::Database::Channels.select(id)
if refresh && Time.utc - channel.updated > 10.minutes return channel if channel
channel = fetch_channel(id, pull_all_videos: pull_all_videos)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
end
else
channel = fetch_channel(id, pull_all_videos: pull_all_videos)
Invidious::Database::Channels.insert(channel)
end
channel = fetch_channel(id, pull_all_videos: false)
Invidious::Database::Channels.insert(channel)
return channel return channel
end end
def fetch_channel(ucid, pull_all_videos = true, locale = nil) def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}") LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body

View File

@ -10,7 +10,7 @@ module Invidious::Database::Channels
# Insert / delete # Insert / delete
# ------------------- # -------------------
def insert(channel : InvidiousChannel, update_on_conflict : Bool = false) def insert(channel : InvidiousChannel)
channel_array = channel.to_a channel_array = channel.to_a
request = <<-SQL request = <<-SQL
@ -18,13 +18,6 @@ module Invidious::Database::Channels
VALUES (#{arg_array(channel_array)}) VALUES (#{arg_array(channel_array)})
SQL SQL
if update_on_conflict
request += <<-SQL
ON CONFLICT (id) DO UPDATE
SET author = $2, updated = $3
SQL
end
PG_DB.exec(request, args: channel_array) PG_DB.exec(request, args: channel_array)
end end

View File

@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
spawn do spawn do
begin begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, CONFIG.full_refresh) channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
lim_fibers = max_fibers lim_fibers = max_fibers

View File

@ -92,7 +92,7 @@ module Invidious::Routes::API::V1::Authenticated
ucid = env.params.url["ucid"] ucid = env.params.url["ucid"]
if !user.subscriptions.includes? ucid if !user.subscriptions.includes? ucid
get_channel(ucid, false, false) get_channel(ucid)
Invidious::Database::Users.subscribe_channel(user, ucid) Invidious::Database::Users.subscribe_channel(user, ucid)
end end

View File

@ -327,7 +327,7 @@ module Invidious::Routes::PreferencesRoute
user.subscriptions += body["subscriptions"].as_a.map(&.as_s) user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
user.subscriptions.uniq! user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, false, false) user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user) Invidious::Database::Users.update_subscriptions(user)
end end
@ -409,7 +409,7 @@ module Invidious::Routes::PreferencesRoute
end end
user.subscriptions.uniq! user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, false, false) user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user) Invidious::Database::Users.update_subscriptions(user)
when "import_freetube" when "import_freetube"
@ -418,7 +418,7 @@ module Invidious::Routes::PreferencesRoute
end end
user.subscriptions.uniq! user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, false, false) user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user) Invidious::Database::Users.update_subscriptions(user)
when "import_newpipe_subscriptions" when "import_newpipe_subscriptions"
@ -437,7 +437,7 @@ module Invidious::Routes::PreferencesRoute
end end
user.subscriptions.uniq! user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, false, false) user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user) Invidious::Database::Users.update_subscriptions(user)
when "import_newpipe" when "import_newpipe"
@ -456,7 +456,7 @@ module Invidious::Routes::PreferencesRoute
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
user.subscriptions.uniq! user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, false, false) user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user) Invidious::Database::Users.update_subscriptions(user)

View File

@ -51,7 +51,7 @@ module Invidious::Routes::Subscriptions
case action case action
when "action_create_subscription_to_channel" when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id if !user.subscriptions.includes? channel_id
get_channel(channel_id, false, false) get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id)
end end
when "action_remove_subscriptions" when "action_remove_subscriptions"

View File

@ -74,7 +74,7 @@ def fetch_user(sid, headers)
end end
end end
channels = get_batch_channels(channels, false, false) channels = get_batch_channels(channels)
email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
if email if email