Add select_all for channels and use in refresh job

src/invidious/database/channels.cr

@@ -76,6 +76,14 @@ module Invidious::Database::Channels
 
     return PG_DB.query_all(request, as: InvidiousChannel)
   end
+
+  def select_all : Array(InvidiousChannel)
+    request = <<-SQL
+      SELECT * FROM channels
+    SQL
+
+    return PG_DB.query_all(rqeuest, as: InvidiousChannel)
+  end
 end
 
 #

src/invidious/jobs/refresh_channels_job.cr

@@ -13,47 +13,44 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
 
     loop do
       LOGGER.debug("RefreshChannelsJob: Refreshing all channels")
-      PG_DB.query("SELECT id FROM channels ORDER BY updated") do |rs|
+      Invidious::Database::Channels.select_all.each do |channel|
-        rs.each do
+        id = channel.id
-          id = rs.read(String)
+        if active_fibers >= lim_fibers
-
+          LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...")
-          if active_fibers >= lim_fibers
+          if active_channel.receive
-            LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...")
+            LOGGER.trace("RefreshChannelsJob: Fiber limit ok, continuing")
-            if active_channel.receive
+            active_fibers -= 1
-              LOGGER.trace("RefreshChannelsJob: Fiber limit ok, continuing")
-              active_fibers -= 1
-            end
           end
+        end
 
-          LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber")
+        LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber")
-          active_fibers += 1
+        active_fibers += 1
-          spawn do
+        spawn do
-            begin
+          begin
-              LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
+            LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
-              channel = fetch_channel(id, CONFIG.full_refresh)
+            channel = fetch_channel(id, CONFIG.full_refresh)
 
-              lim_fibers = max_fibers
+            lim_fibers = max_fibers
 
-              LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
+            LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
-              Invidious::Database::Channels.update_author(id, channel.author)
+            Invidious::Database::Channels.update_author(id, channel.author)
-            rescue ex
+          rescue ex
-              LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
+            LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
-              if ex.message == "Deleted or invalid channel"
+            if ex.message == "Deleted or invalid channel"
-                Invidious::Database::Channels.update_mark_deleted(id)
+              Invidious::Database::Channels.update_mark_deleted(id)
+            else
+              lim_fibers = 1
+              LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")
+              sleep backoff
+              if backoff < 1.days
+                backoff += backoff
               else
-                lim_fibers = 1
+                backoff = 1.days
-                LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")
-                sleep backoff
-                if backoff < 1.days
-                  backoff += backoff
-                else
-                  backoff = 1.days
-                end
               end
-            ensure
-              LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done")
-              active_channel.send(true)
             end
+          ensure
+            LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done")
+            active_channel.send(true)
           end
         end
       end