From aee7a52b1f20bcf90b5151a4dc94a8ec6a4af098 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:13:11 -0500 Subject: [PATCH 1/3] Ensure IO is properly closed when importing NewPipe subscriptions --- src/invidious/user/imports.cr | 52 ++++++++++++++++------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 533c18d9..84794991 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -290,42 +290,38 @@ struct Invidious::User end def from_newpipe(user : User, body : String) : Bool - io = IO::Memory.new(body) - - Compress::Zip::File.open(io) do |file| + Compress::Zip::File.open(IO::Memory.new(body), true) do |file| file.entries.each do |entry| entry.open do |file_io| + next if entry.filename != "newpipe.db" + # Ensure max size of 4MB io_sized = IO::Sized.new(file_io, 0x400000) - next if entry.filename != "newpipe.db" + temp = File.tempfile(".db") do |tempfile| + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end - tempfile = File.tempfile(".db") + DB.open("sqlite3://" + tempfile.path) do |db| + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) - begin - File.write(tempfile.path, io_sized.gets_to_end) - rescue - return false + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end end - - db = DB.open("sqlite3://" + tempfile.path) - - user.watched += db.query_all("SELECT url FROM streams", as: String) - .map(&.lchop("https://www.youtube.com/watch?v=")) - - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) - .map(&.lchop("https://www.youtube.com/channel/")) - - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) - - db.close - tempfile.delete + temp.delete end end end From 7b6b914c7bdb40dbaa810068b518ccb5ea21f96f Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Tue, 30 Apr 2024 19:45:29 -0400 Subject: [PATCH 2/3] Put temp.delete inside ensure block --- src/invidious/user/imports.cr | 47 +++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 84794991..163a8c74 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -298,30 +298,33 @@ struct Invidious::User # Ensure max size of 4MB io_sized = IO::Sized.new(file_io, 0x400000) - temp = File.tempfile(".db") do |tempfile| - begin - File.write(tempfile.path, io_sized.gets_to_end) - rescue - return false - end - - DB.open("sqlite3://" + tempfile.path) do |db| - user.watched += db.query_all("SELECT url FROM streams", as: String) - .map(&.lchop("https://www.youtube.com/watch?v=")) - - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) - .map(&.lchop("https://www.youtube.com/channel/")) - - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) + begin + temp = File.tempfile(".db") do |tempfile| + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end + + DB.open("sqlite3://" + tempfile.path) do |db| + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) + + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end end + ensure + temp.delete if !temp.nil? end - temp.delete end end end From 2d1437f6ff924c5cf17112a0c3900c9a89bdd05b Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Wed, 10 Jul 2024 09:26:31 -0400 Subject: [PATCH 3/3] Use a find instead of an each loop --- src/invidious/user/imports.cr | 60 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 163a8c74..007eb666 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -291,40 +291,38 @@ struct Invidious::User def from_newpipe(user : User, body : String) : Bool Compress::Zip::File.open(IO::Memory.new(body), true) do |file| - file.entries.each do |entry| - entry.open do |file_io| - next if entry.filename != "newpipe.db" + entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" } + return false if entry.nil? + entry.open do |file_io| + # Ensure max size of 4MB + io_sized = IO::Sized.new(file_io, 0x400000) - # Ensure max size of 4MB - io_sized = IO::Sized.new(file_io, 0x400000) - - begin - temp = File.tempfile(".db") do |tempfile| - begin - File.write(tempfile.path, io_sized.gets_to_end) - rescue - return false - end - - DB.open("sqlite3://" + tempfile.path) do |db| - user.watched += db.query_all("SELECT url FROM streams", as: String) - .map(&.lchop("https://www.youtube.com/watch?v=")) - - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) - .map(&.lchop("https://www.youtube.com/channel/")) - - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) - end + begin + temp = File.tempfile(".db") do |tempfile| + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end + + DB.open("sqlite3://" + tempfile.path) do |db| + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) + + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) end - ensure - temp.delete if !temp.nil? end + ensure + temp.delete if !temp.nil? end end end