mirror of
https://github.com/iv-org/invidious.git
synced 2024-12-24 14:59:29 -05:00
Merge pull request #2545 from bbielsa/csv-subscriptions-import
Add CSV Subscriptions Import
This commit is contained in:
commit
6fab5d0554
51
spec/invidious/user/imports_spec.cr
Normal file
51
spec/invidious/user/imports_spec.cr
Normal file
@ -0,0 +1,51 @@
|
||||
require "spectator"
|
||||
require "../../../src/invidious/user/imports"
|
||||
|
||||
Spectator.configure do |config|
|
||||
config.fail_blank
|
||||
config.randomize
|
||||
end
|
||||
|
||||
def csv_sample
|
||||
return <<-CSV
|
||||
Kanal-ID,Kanal-URL,Kanaltitel
|
||||
UC0hHW5Y08ggq-9kbrGgWj0A,http://www.youtube.com/channel/UC0hHW5Y08ggq-9kbrGgWj0A,Matias Marolla
|
||||
UC0vBXGSyV14uvJ4hECDOl0Q,http://www.youtube.com/channel/UC0vBXGSyV14uvJ4hECDOl0Q,Techquickie
|
||||
UC1sELGmy5jp5fQUugmuYlXQ,http://www.youtube.com/channel/UC1sELGmy5jp5fQUugmuYlXQ,Minecraft
|
||||
UC9kFnwdCRrX7oTjqKd6-tiQ,http://www.youtube.com/channel/UC9kFnwdCRrX7oTjqKd6-tiQ,LUMOX - Topic
|
||||
UCBa659QWEk1AI4Tg--mrJ2A,http://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A,Tom Scott
|
||||
UCGu6_XQ64rXPR6nuitMQE_A,http://www.youtube.com/channel/UCGu6_XQ64rXPR6nuitMQE_A,Callcenter Fun
|
||||
UCGwu0nbY2wSkW8N-cghnLpA,http://www.youtube.com/channel/UCGwu0nbY2wSkW8N-cghnLpA,Jaiden Animations
|
||||
UCQ0OvZ54pCFZwsKxbltg_tg,http://www.youtube.com/channel/UCQ0OvZ54pCFZwsKxbltg_tg,Methos
|
||||
UCRE6itj4Jte4manQEu3Y7OA,http://www.youtube.com/channel/UCRE6itj4Jte4manQEu3Y7OA,Chipflake
|
||||
UCRLc6zsv_d0OEBO8OOkz-DA,http://www.youtube.com/channel/UCRLc6zsv_d0OEBO8OOkz-DA,Kegy
|
||||
UCSl5Uxu2LyaoAoMMGp6oTJA,http://www.youtube.com/channel/UCSl5Uxu2LyaoAoMMGp6oTJA,Atomic Shrimp
|
||||
UCXuqSBlHAE6Xw-yeJA0Tunw,http://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw,Linus Tech Tips
|
||||
UCZ5XnGb-3t7jCkXdawN2tkA,http://www.youtube.com/channel/UCZ5XnGb-3t7jCkXdawN2tkA,Discord
|
||||
CSV
|
||||
end
|
||||
|
||||
Spectator.describe "Invidious::User::Imports" do
|
||||
it "imports CSV" do
|
||||
subscriptions = parse_subscription_export_csv(csv_sample)
|
||||
|
||||
expect(subscriptions).to be_an(Array(String))
|
||||
expect(subscriptions.size).to eq(13)
|
||||
|
||||
expect(subscriptions).to contain_exactly(
|
||||
"UC0hHW5Y08ggq-9kbrGgWj0A",
|
||||
"UC0vBXGSyV14uvJ4hECDOl0Q",
|
||||
"UC1sELGmy5jp5fQUugmuYlXQ",
|
||||
"UC9kFnwdCRrX7oTjqKd6-tiQ",
|
||||
"UCBa659QWEk1AI4Tg--mrJ2A",
|
||||
"UCGu6_XQ64rXPR6nuitMQE_A",
|
||||
"UCGwu0nbY2wSkW8N-cghnLpA",
|
||||
"UCQ0OvZ54pCFZwsKxbltg_tg",
|
||||
"UCRE6itj4Jte4manQEu3Y7OA",
|
||||
"UCRLc6zsv_d0OEBO8OOkz-DA",
|
||||
"UCSl5Uxu2LyaoAoMMGp6oTJA",
|
||||
"UCXuqSBlHAE6Xw-yeJA0Tunw",
|
||||
"UCZ5XnGb-3t7jCkXdawN2tkA",
|
||||
).in_order
|
||||
end
|
||||
end
|
@ -746,6 +746,8 @@ post "/data_control" do |env|
|
||||
|
||||
HTTP::FormData.parse(env.request) do |part|
|
||||
body = part.body.gets_to_end
|
||||
type = part.headers["Content-Type"]
|
||||
|
||||
next if body.empty?
|
||||
|
||||
# TODO: Unify into single import based on content-type
|
||||
@ -816,19 +818,29 @@ post "/data_control" do |env|
|
||||
end
|
||||
end
|
||||
when "import_youtube"
|
||||
if body[0..4] == "<opml"
|
||||
filename = part.filename || ""
|
||||
extension = filename.split(".").last
|
||||
|
||||
if extension == "xml" || type == "application/xml" || type == "text/xml"
|
||||
subscriptions = XML.parse(body)
|
||||
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
|
||||
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||
end
|
||||
else
|
||||
elsif extension == "json" || type == "application/json"
|
||||
subscriptions = JSON.parse(body)
|
||||
user.subscriptions += subscriptions.as_a.compact_map do |entry|
|
||||
entry["snippet"]["resourceId"]["channelId"].as_s
|
||||
end
|
||||
elsif extension == "csv" || type == "text/csv"
|
||||
subscriptions = parse_subscription_export_csv(body)
|
||||
user.subscriptions += subscriptions
|
||||
else
|
||||
halt(env, status_code: 415,
|
||||
response: error_template(415, "Invalid subscription file uploaded")
|
||||
)
|
||||
end
|
||||
user.subscriptions.uniq!
|
||||
|
||||
user.subscriptions.uniq!
|
||||
user.subscriptions = get_batch_channels(user.subscriptions, false, false)
|
||||
|
||||
Invidious::Database::Users.update_subscriptions(user)
|
||||
|
27
src/invidious/user/imports.cr
Normal file
27
src/invidious/user/imports.cr
Normal file
@ -0,0 +1,27 @@
|
||||
require "csv"
|
||||
|
||||
def parse_subscription_export_csv(csv_content : String)
|
||||
rows = CSV.new(csv_content, headers: true)
|
||||
subscriptions = Array(String).new
|
||||
|
||||
# Counter to limit the amount of imports.
|
||||
# This is intended to prevent DoS.
|
||||
row_counter = 0
|
||||
|
||||
rows.each do |row|
|
||||
# Limit to 1200
|
||||
row_counter += 1
|
||||
break if row_counter > 1_200
|
||||
|
||||
# Channel ID is the first column in the csv export we can't use the header
|
||||
# name, because the header name is localized depending on the
|
||||
# language the user has set on their account
|
||||
channel_id = row[0].strip
|
||||
|
||||
next if channel_id.empty?
|
||||
|
||||
subscriptions << channel_id
|
||||
end
|
||||
|
||||
return subscriptions
|
||||
end
|
Loading…
Reference in New Issue
Block a user