Merge pull request #2545 from bbielsa/csv-subscriptions-import

Add CSV Subscriptions Import

spec/invidious/user/imports_spec.cr

@@ -0,0 +1,51 @@
+require "spectator"
+require "../../../src/invidious/user/imports"
+
+Spectator.configure do |config|
+  config.fail_blank
+  config.randomize
+end
+
+def csv_sample
+  return <<-CSV
+  Kanal-ID,Kanal-URL,Kanaltitel
+  UC0hHW5Y08ggq-9kbrGgWj0A,http://www.youtube.com/channel/UC0hHW5Y08ggq-9kbrGgWj0A,Matias Marolla
+  UC0vBXGSyV14uvJ4hECDOl0Q,http://www.youtube.com/channel/UC0vBXGSyV14uvJ4hECDOl0Q,Techquickie
+  UC1sELGmy5jp5fQUugmuYlXQ,http://www.youtube.com/channel/UC1sELGmy5jp5fQUugmuYlXQ,Minecraft
+  UC9kFnwdCRrX7oTjqKd6-tiQ,http://www.youtube.com/channel/UC9kFnwdCRrX7oTjqKd6-tiQ,LUMOX - Topic
+  UCBa659QWEk1AI4Tg--mrJ2A,http://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A,Tom Scott
+  UCGu6_XQ64rXPR6nuitMQE_A,http://www.youtube.com/channel/UCGu6_XQ64rXPR6nuitMQE_A,Callcenter Fun
+  UCGwu0nbY2wSkW8N-cghnLpA,http://www.youtube.com/channel/UCGwu0nbY2wSkW8N-cghnLpA,Jaiden Animations
+  UCQ0OvZ54pCFZwsKxbltg_tg,http://www.youtube.com/channel/UCQ0OvZ54pCFZwsKxbltg_tg,Methos
+  UCRE6itj4Jte4manQEu3Y7OA,http://www.youtube.com/channel/UCRE6itj4Jte4manQEu3Y7OA,Chipflake
+  UCRLc6zsv_d0OEBO8OOkz-DA,http://www.youtube.com/channel/UCRLc6zsv_d0OEBO8OOkz-DA,Kegy
+  UCSl5Uxu2LyaoAoMMGp6oTJA,http://www.youtube.com/channel/UCSl5Uxu2LyaoAoMMGp6oTJA,Atomic Shrimp
+  UCXuqSBlHAE6Xw-yeJA0Tunw,http://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw,Linus Tech Tips
+  UCZ5XnGb-3t7jCkXdawN2tkA,http://www.youtube.com/channel/UCZ5XnGb-3t7jCkXdawN2tkA,Discord
+  CSV
+end
+
+Spectator.describe "Invidious::User::Imports" do
+  it "imports CSV" do
+    subscriptions = parse_subscription_export_csv(csv_sample)
+
+    expect(subscriptions).to be_an(Array(String))
+    expect(subscriptions.size).to eq(13)
+
+    expect(subscriptions).to contain_exactly(
+      "UC0hHW5Y08ggq-9kbrGgWj0A",
+      "UC0vBXGSyV14uvJ4hECDOl0Q",
+      "UC1sELGmy5jp5fQUugmuYlXQ",
+      "UC9kFnwdCRrX7oTjqKd6-tiQ",
+      "UCBa659QWEk1AI4Tg--mrJ2A",
+      "UCGu6_XQ64rXPR6nuitMQE_A",
+      "UCGwu0nbY2wSkW8N-cghnLpA",
+      "UCQ0OvZ54pCFZwsKxbltg_tg",
+      "UCRE6itj4Jte4manQEu3Y7OA",
+      "UCRLc6zsv_d0OEBO8OOkz-DA",
+      "UCSl5Uxu2LyaoAoMMGp6oTJA",
+      "UCXuqSBlHAE6Xw-yeJA0Tunw",
+      "UCZ5XnGb-3t7jCkXdawN2tkA",
+    ).in_order
+  end
+end

src/invidious.cr

@@ -746,6 +746,8 @@ post "/data_control" do |env|
 
     HTTP::FormData.parse(env.request) do |part|
       body = part.body.gets_to_end
+      type = part.headers["Content-Type"]
+
       next if body.empty?
 
       # TODO: Unify into single import based on content-type
@@ -816,19 +818,29 @@ post "/data_control" do |env|
           end
         end
       when "import_youtube"
-        if body[0..4] == "<opml"
+        filename = part.filename || ""
+        extension = filename.split(".").last
+
+        if extension == "xml" || type == "application/xml" || type == "text/xml"
           subscriptions = XML.parse(body)
           user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
             channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
           end
-        else
+        elsif extension == "json" || type == "application/json"
           subscriptions = JSON.parse(body)
           user.subscriptions += subscriptions.as_a.compact_map do |entry|
             entry["snippet"]["resourceId"]["channelId"].as_s
           end
+        elsif extension == "csv" || type == "text/csv"
+          subscriptions = parse_subscription_export_csv(body)
+          user.subscriptions += subscriptions
+        else
+          halt(env, status_code: 415,
+            response: error_template(415, "Invalid subscription file uploaded")
+          )
         end
-        user.subscriptions.uniq!
 
+        user.subscriptions.uniq!
         user.subscriptions = get_batch_channels(user.subscriptions, false, false)
 
         Invidious::Database::Users.update_subscriptions(user)

src/invidious/user/imports.cr

@@ -0,0 +1,27 @@
+require "csv"
+
+def parse_subscription_export_csv(csv_content : String)
+  rows = CSV.new(csv_content, headers: true)
+  subscriptions = Array(String).new
+
+  # Counter to limit the amount of imports.
+  # This is intended to prevent DoS.
+  row_counter = 0
+
+  rows.each do |row|
+    # Limit to 1200
+    row_counter += 1
+    break if row_counter > 1_200
+
+    # Channel ID is the first column in the csv export we can't use the header
+    # name, because the header name is localized depending on the
+    # language the user has set on their account
+    channel_id = row[0].strip
+
+    next if channel_id.empty?
+
+    subscriptions << channel_id
+  end
+
+  return subscriptions
+end