Thread safety and not try none-working backends (#140)

Co-authored-by: Kevinf100 <12779728+kevinf100@users.noreply.github.com>

commit 0fe9e0a28a
Author: Kevinf100 <12779728+kevinf100@users.noreply.github.com>
Date:   Thu Jun 5 11:41:02 2025 -0400

    More thread safety and speed improvements

commit 54ea7f04fd
Author: Kevinf100 <12779728+kevinf100@users.noreply.github.com>
Date:   Tue Jun 3 23:37:20 2025 -0400

    Many thread safe updates.
    Updating instances timesout after 10 seconds.
    Now waits until checking all instances finishes before starting the timer for next.
    More random debugging logs

commit 4dcd6dd2f4
Author: Kevinf100 <12779728+kevinf100@users.noreply.github.com>
Date:   Tue Jun 3 10:34:56 2025 -0400

    Update API to only try working backends with companion

src/invidious/helpers/backend_info.cr

@@ -3,61 +3,104 @@ module BackendInfo
   @@exvpp_url : Array(String) = Array.new(CONFIG.invidious_companion.size, "")
   @@status : Array(Int32) = Array.new(CONFIG.invidious_companion.size, 0)
   @@csp : Array(String) = Array.new(CONFIG.invidious_companion.size, "")
-  @@mutex : Mutex = Mutex.new
+  @@working_ends : Array(Int32) = Array(Int32).new(0)
+  @@csp_mutex : Mutex = Mutex.new
+  @@check_mutex : Mutex = Mutex.new
+
+  enum BackendStatus
+      Dead
+      Problems
+      Working
+  end
 
   def check_backends
     check_companion()
+    LOGGER.debug("Invidious companion: New working_ends \"#{@@working_ends}\"")
+    LOGGER.debug("Invidious companion: New status \"#{@@status}\"")
   end
 
   private def check_companion
+    # Create Channels the size of CONFIG.invidious_companion
+    comp_size = CONFIG.invidious_companion.size
+    channels = Channel(Nil).new(comp_size)
+    updated_ends = Array(Int32).new(0)
+    updated_status = Array(Int32).new(CONFIG.invidious_companion.size, 0)
+    LOGGER.debug("Invidious companion: comp_size \"#{comp_size}\"")
     CONFIG.invidious_companion.each_with_index do |companion, index|
       spawn do
         begin
-          response = HTTP::Client.get "#{companion.private_url}/healthz"
+          client = HTTP::Client.new(companion.private_url)
+          client.connect_timeout = 10.seconds
+          response = client.get("/healthz")
           if response.status_code == 200
-            check_videoplayback_proxy(companion, index)
+            check_videoplayback_proxy(companion, index, updated_status, updated_ends)
             generate_csp([companion.public_url.to_s, companion.i2p_public_url.to_s], @@exvpp_url[index], index)
           else
-            @@status[index] = 0
+            @@check_mutex.synchronize do
+              updated_status[index] = BackendStatus::Dead.to_i
+            end
           end
         rescue
-          @@status[index] = 0
+          @@check_mutex.synchronize do
+            updated_status[index] = BackendStatus::Dead.to_i
+          end
+        ensure
+          LOGGER.debug("Invidious companion: Done Index: \"#{index}\"")
+          channels.send(nil)
         end
       end
     end
+    # Wait until we receive a signal from them all
+    LOGGER.debug("Invidious companion: Updating working_ends")
+    comp_size.times { channels.receive }
+    @@working_ends = updated_ends
+    @@status = updated_status
   end
 
-  private def check_videoplayback_proxy(companion : Config::CompanionConfig, index : Int32)
+  private def check_videoplayback_proxy(companion : Config::CompanionConfig, index : Int32, updated_status : Array(Int32), updated_ends : Array(Int32))
     begin
       info = HTTP::Client.get "#{companion.private_url}/info"
       exvpp_url = JSON.parse(info.body)["external_videoplayback_proxy"]?.try &.to_s
     rescue JSON::ParseException
-      @@status[index] = 2
+      @@check_mutex.synchronize do
+        updated_status[index] = BackendStatus::Working.to_i
+        updated_ends.push(index)
+      end
       return
     end
 
     exvpp_url = "" if exvpp_url.nil?
     @@exvpp_url[index] = exvpp_url
     if exvpp_url.empty?
-      @@status[index] = 2
+      @@check_mutex.synchronize do
+        updated_status[index] = BackendStatus::Working.to_i
+        updated_ends.push(index)
+      end
       return
     else
       begin
         exvpp_health = HTTP::Client.get "#{exvpp_url}/health"
         if exvpp_health.status_code == 200
-          @@status[index] = 2
+            @@check_mutex.synchronize do
+              updated_status[index] = BackendStatus::Working.to_i
+              updated_ends.push(index)
+            end
           return exvpp_url
         else
-          @@status[index] = 1
+          @@check_mutex.synchronize do
+            updated_status[index] = BackendStatus::Problems.to_i
+          end
         end
       rescue
-        @@status[index] = 1
+        @@check_mutex.synchronize do
+          updated_status[index] = BackendStatus::Problems.to_i
+        end
       end
     end
   end
 
   private def generate_csp(companion_url : Array(String), exvpp_url : String? = nil, index : Int32? = nil)
-    @@mutex.synchronize do
+    @@csp_mutex.synchronize do
       @@csp[index] = ""
       companion_url.each do |url|
         @@csp[index] += " #{url}"
@@ -67,9 +110,15 @@ module BackendInfo
   end
 
   def get_status
+    # Shouldn't need to lock since we never edit this array, only change the pointer.
     return @@status
   end
 
+  def get_working_ends
+    # Shouldn't need to lock since we never edit this array, only change the pointer.
+    return @@working_ends
+  end
+
   def get_exvpp
     return @@exvpp_url
   end
@@ -78,7 +127,7 @@ module BackendInfo
     # A little mutex to prevent sending a partial CSP header
     # Not sure if this is necessary. But if the @@csp[index] is being assigned
     # at the same time when it's being accessed, a data race will appear
-    @@mutex.synchronize do
+    @@csp_mutex.synchronize do
       return @@csp[index], @@csp[index]
     end
   end

src/invidious/jobs/backend_checker.cr

@@ -4,6 +4,7 @@ class Invidious::Jobs::CheckBackend < Invidious::Jobs::BaseJob
 
   def begin
     loop do
+      LOGGER.info("Backend Checker: Starting")
       BackendInfo.check_backends
       LOGGER.info("Backend Checker: Done, sleeping for #{CONFIG.check_backends_interval} seconds")
       sleep CONFIG.check_backends_interval.seconds

src/invidious/routes/before_all.cr

@@ -46,7 +46,8 @@ module Invidious::Routes::BeforeAll
         begin
           current_companion = env.request.cookies[CONFIG.server_id_cookie_name].value.try &.to_i
         rescue
-          current_companion = rand(CONFIG.invidious_companion.size)
+          working_ends = BackendInfo.get_working_ends
+          current_companion = working_ends.sample
         end
 
         if current_companion > CONFIG.invidious_companion.size

src/invidious/yt_backend/youtube_api.cr

@@ -689,7 +689,8 @@ module YoutubeAPI
 
     begin
       if env.nil?
-        current_companion = rand(CONFIG.invidious_companion.size)
+        working_ends = BackendInfo.get_working_ends
+        current_companion = working_ends.sample
       else
         current_companion = env.get("current_companion").as(Int32)
       end