diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index 134cf59d..1075398a 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -76,6 +76,14 @@ module Invidious::Database::Channels return PG_DB.query_all(request, as: InvidiousChannel) end + + def select_all : Array(InvidiousChannel) + request = <<-SQL + SELECT * FROM channels + SQL + + return PG_DB.query_all(rqeuest, as: InvidiousChannel) + end end # diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 941089c1..e083ebac 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -13,47 +13,44 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob loop do LOGGER.debug("RefreshChannelsJob: Refreshing all channels") - PG_DB.query("SELECT id FROM channels ORDER BY updated") do |rs| - rs.each do - id = rs.read(String) - - if active_fibers >= lim_fibers - LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...") - if active_channel.receive - LOGGER.trace("RefreshChannelsJob: Fiber limit ok, continuing") - active_fibers -= 1 - end + Invidious::Database::Channels.select_all.each do |channel| + id = channel.id + if active_fibers >= lim_fibers + LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...") + if active_channel.receive + LOGGER.trace("RefreshChannelsJob: Fiber limit ok, continuing") + active_fibers -= 1 end + end - LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber") - active_fibers += 1 - spawn do - begin - LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, CONFIG.full_refresh) + LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber") + active_fibers += 1 + spawn do + begin + LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") + channel = fetch_channel(id, CONFIG.full_refresh) - lim_fibers = max_fibers + lim_fibers = max_fibers - LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB") - Invidious::Database::Channels.update_author(id, channel.author) - rescue ex - LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") - if ex.message == "Deleted or invalid channel" - Invidious::Database::Channels.update_mark_deleted(id) + LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB") + Invidious::Database::Channels.update_author(id, channel.author) + rescue ex + LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") + if ex.message == "Deleted or invalid channel" + Invidious::Database::Channels.update_mark_deleted(id) + else + lim_fibers = 1 + LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") + sleep backoff + if backoff < 1.days + backoff += backoff else - lim_fibers = 1 - LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") - sleep backoff - if backoff < 1.days - backoff += backoff - else - backoff = 1.days - end + backoff = 1.days end - ensure - LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done") - active_channel.send(true) end + ensure + LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done") + active_channel.send(true) end end end