Use fibers to try to speed up importing of channels

src/invidious/channels.cr

@@ -21,6 +21,33 @@ class ChannelVideo
   })
 end
 
+def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
+  active_threads = 0
+  active_channel = Channel(String | Nil).new
+
+  final = [] of String
+  channels.map do |ucid|
+    if active_threads >= max_threads
+      if response = active_channel.receive
+        active_threads -= 1
+        final << response
+      end
+    end
+
+    active_threads += 1
+    spawn do
+      begin
+        get_channel(ucid, db, refresh, pull_all_videos)
+        active_channel.send(ucid)
+      rescue ex
+        active_channel.send(nil)
+      end
+    end
+  end
+
+  return final
+end
+
 def get_channel(id, db, refresh = true, pull_all_videos = true)
   client = make_client(YT_URL)
 

src/invidious/users.cr

@@ -177,19 +177,16 @@ def fetch_user(sid, headers, db)
   feed = XML.parse_html(feed.body)
 
   channels = [] of String
-  feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).each do |channel|
-    if !{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
-      channel_id = channel["href"].lstrip("/channel/")
-
-      begin
-        channel = get_channel(channel_id, db, false, false)
-        channels << channel.id
-      rescue ex
-        next
-      end
+  channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
+    if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
+      nil
+    else
+      channel["href"].lstrip("/channel/")
     end
   end
 
+  channels = get_batch_channels(channels, db, false, false)
+
   email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
   if email
     email = email.content.strip