mirrors/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-08-23 13:08:31 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Extract channel refresh code from async code

Browse Source
This commit is contained in:
matthewmcgarvey 2022-01-22 10:26:00 -06:00
parent 314f49b687
commit bcb9441bd8
1 changed files with 31 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/invidious/jobs/refresh_channels_job.cr
View File

@ -9,7 +9,6 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
loop do loop do
LOGGER.debug("RefreshChannelsJob: Refreshing all channels") LOGGER.debug("RefreshChannelsJob: Refreshing all channels")
Invidious::Database::Channels.select_all.each do |channel| Invidious::Database::Channels.select_all.each do |channel|
id = channel.id
if active_fibers >= lim_fibers if active_fibers >= lim_fibers
LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...") LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...")
if active_channel.receive if active_channel.receive
@ -18,24 +17,14 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
end end
end end
LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber") LOGGER.debug("RefreshChannelsJob: #{channel.id} : Spawning fiber")
active_fibers += 1 active_fibers += 1
spawn do spawn do
begin if refresh_channel(channel)
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, CONFIG.full_refresh)
lim_fibers = max_fibers lim_fibers = max_fibers
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
Invidious::Database::Channels.update_author(id, channel.author)
rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
Invidious::Database::Channels.update_mark_deleted(id)
else else
lim_fibers = 1 lim_fibers = 1
LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") LOGGER.error("RefreshChannelsJob: #{channel.id} fiber : backing off for #{backoff}s")
sleep backoff sleep backoff
if backoff < 1.days if backoff < 1.days
backoff += backoff backoff += backoff
@ -43,12 +32,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
backoff = 1.days backoff = 1.days
end end
end end
ensure
LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done") LOGGER.debug("RefreshChannelsJob: #{channel.id} fiber : Done")
active_channel.send(true) active_channel.send(true)
end end
end end
end
# TODO: make this configurable # TODO: make this configurable
LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes") LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes")
@ -56,4 +44,23 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
Fiber.yield Fiber.yield
end end
end end
private def refresh_channel(channel : InvidiousChannel) : Bool
id = channel.id
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, CONFIG.full_refresh)
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
Invidious::Database::Channels.update_author(id, channel.author)
return true
rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
Invidious::Database::Channels.update_mark_deleted(id) if id
return true
else
return false
end
end
end end