mirrors/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-08-17 18:18:30 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Add select_all for channels and use in refresh job

Browse Source
This commit is contained in:
matthewmcgarvey 2022-01-22 09:53:24 -06:00
parent 8b0ed8b110
commit 7e25ad5e4e
2 changed files with 39 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/invidious/database/channels.cr
View File

@ -76,6 +76,14 @@ module Invidious::Database::Channels
return PG_DB.query_all(request, as: InvidiousChannel)
end
def select_all : Array(InvidiousChannel)
request = <<-SQL
SELECT * FROM channels
SQL
return PG_DB.query_all(rqeuest, as: InvidiousChannel)
end
end
#

65
src/invidious/jobs/refresh_channels_job.cr
View File

@ -13,47 +13,44 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
loop do
LOGGER.debug("RefreshChannelsJob: Refreshing all channels")
PG_DB.query("SELECT id FROM channels ORDER BY updated") do |rs|
rs.each do
id = rs.read(String)
if active_fibers >= lim_fibers
LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...")
if active_channel.receive
LOGGER.trace("RefreshChannelsJob: Fiber limit ok, continuing")
active_fibers -= 1
end
Invidious::Database::Channels.select_all.each do |channel|
id = channel.id
if active_fibers >= lim_fibers
LOGGER.trace("RefreshChannelsJob: Fiber limit reached, waiting...")
if active_channel.receive
LOGGER.trace("RefreshChannelsJob: Fiber limit ok, continuing")
active_fibers -= 1
end
end
LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber")
active_fibers += 1
spawn do
begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, CONFIG.full_refresh)
LOGGER.debug("RefreshChannelsJob: #{id} : Spawning fiber")
active_fibers += 1
spawn do
begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, CONFIG.full_refresh)
lim_fibers = max_fibers
lim_fibers = max_fibers
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
Invidious::Database::Channels.update_author(id, channel.author)
rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
Invidious::Database::Channels.update_mark_deleted(id)
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
Invidious::Database::Channels.update_author(id, channel.author)
rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
Invidious::Database::Channels.update_mark_deleted(id)
else
lim_fibers = 1
LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")
sleep backoff
if backoff < 1.days
backoff += backoff
else
lim_fibers = 1
LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")
sleep backoff
if backoff < 1.days
backoff += backoff
else
backoff = 1.days
end
backoff = 1.days
end
ensure
LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done")
active_channel.send(true)
end
ensure
LOGGER.debug("RefreshChannelsJob: #{id} fiber : Done")
active_channel.send(true)
end
end
end