Many thread safe updates.

Updating instances timesout after 10 seconds.
Now waits until checking all instances finishes before starting the timer for next.
More random debugging logs

src/invidious/helpers/backend_info.cr

@@ -5,35 +5,54 @@ module BackendInfo
   @@csp : Array(String) = Array.new(CONFIG.invidious_companion.size, "")
   @@working_ends : Array(Int32) = Array(Int32).new(0)
   @@mutex : Mutex = Mutex.new
+  @@working_mutex : Mutex = Mutex.new
 
   def check_backends
     check_companion()
     LOGGER.debug("Invidious companion: Updating working_ends")
-    @@working_ends.clear
+    updated_ends = Array(Int32).new(0)
     @@status.each_with_index do |_, index|
       if @@status[index] == 2
-          @@working_ends.push(index)
+          updated_ends.push(index)
       end
     end
+    @@working_mutex.synchronize do
+      @@working_ends = updated_ends
+    end
     LOGGER.debug("Invidious companion: New working_ends \"#{@@working_ends}\"")
   end
 
   private def check_companion
+    # Create Channels the size of CONFIG.invidious_companion
+    comp_size = CONFIG.invidious_companion.size
+    channels = Channel(Nil).new(comp_size)
+    LOGGER.debug("Invidious companion: comp_size \"#{comp_size}\"")
     CONFIG.invidious_companion.each_with_index do |companion, index|
       spawn do
         begin
-          response = HTTP::Client.get "#{companion.private_url}/healthz"
+          client = HTTP::Client.new(companion.private_url)
+          client.connect_timeout = 10.seconds
+          response = client.get("/healthz")
           if response.status_code == 200
             check_videoplayback_proxy(companion, index)
             generate_csp([companion.public_url.to_s, companion.i2p_public_url.to_s], @@exvpp_url[index], index)
           else
-            @@status[index] = 0
+            @@mutex.synchronize do
+              @@status[index] = 0
+            end
           end
         rescue
-          @@status[index] = 0
+          @@mutex.synchronize do
+            @@status[index] = 0
+          end
+        ensure
+          LOGGER.debug("Invidious companion: Done Index: \"#{index}\"")
+          channels.send(nil)
         end
       end
     end
+    # Wait until we receive a signal from them all
+    comp_size.times { channels.receive }
   end
 
   private def check_videoplayback_proxy(companion : Config::CompanionConfig, index : Int32)
@@ -41,26 +60,36 @@ module BackendInfo
       info = HTTP::Client.get "#{companion.private_url}/info"
       exvpp_url = JSON.parse(info.body)["external_videoplayback_proxy"]?.try &.to_s
     rescue JSON::ParseException
-      @@status[index] = 2
+      @@mutex.synchronize do
+        @@status[index] = 2
+      end
       return
     end
 
     exvpp_url = "" if exvpp_url.nil?
     @@exvpp_url[index] = exvpp_url
     if exvpp_url.empty?
-      @@status[index] = 2
+      @@mutex.synchronize do
+        @@status[index] = 2
+      end
       return
     else
       begin
         exvpp_health = HTTP::Client.get "#{exvpp_url}/health"
         if exvpp_health.status_code == 200
-          @@status[index] = 2
+          @@mutex.synchronize do
+            @@status[index] = 2
+          end
           return exvpp_url
         else
-          @@status[index] = 1
+          @@mutex.synchronize do
+            @@status[index] = 1
+          end
         end
       rescue
-        @@status[index] = 1
+        @@mutex.synchronize do
+          @@status[index] = 1
+        end
       end
     end
   end
@@ -80,7 +109,11 @@ module BackendInfo
   end
 
   def get_working_ends
-    return @@working_ends
+    # We need to stall this if we are updating the array
+    # Not doing this can cause this to return weird values while being updated
+    @@working_mutex.synchronize do
+      @@working_ends.dup
+    end
   end
 
   def get_exvpp

src/invidious/jobs/backend_checker.cr

@@ -4,6 +4,7 @@ class Invidious::Jobs::CheckBackend < Invidious::Jobs::BaseJob
 
   def begin
     loop do
+      LOGGER.info("Backend Checker: Starting")
       BackendInfo.check_backends
       LOGGER.info("Backend Checker: Done, sleeping for #{CONFIG.check_backends_interval} seconds")
       sleep CONFIG.check_backends_interval.seconds

src/invidious/routes/before_all.cr

@@ -46,7 +46,9 @@ module Invidious::Routes::BeforeAll
         begin
           current_companion = env.request.cookies[CONFIG.server_id_cookie_name].value.try &.to_i
         rescue
-          current_companion = BackendInfo.get_working_ends.sample
+          working_ends = BackendInfo.get_working_ends
+          LOGGER.debug("Invidious companion youtube_api: get_working_ends: \"#{working_ends}\"")
+          current_companion = working_ends.sample
         end
 
         if current_companion > CONFIG.invidious_companion.size

src/invidious/yt_backend/youtube_api.cr

@@ -689,11 +689,13 @@ module YoutubeAPI
 
     begin
       if env.nil?
-        current_companion = BackendInfo.get_working_ends.sample
+        working_ends = BackendInfo.get_working_ends
+        LOGGER.debug("Invidious companion youtube_api: get_working_ends: \"#{working_ends}\"")
+        current_companion = working_ends.sample
       else
         current_companion = env.get("current_companion").as(Int32)
       end
-      LOGGER.trace("Invidious companion: current_companion: #{current_companion}")
+      LOGGER.debug("Invidious companion: current_companion: #{current_companion}")
       response = COMPANION_POOL[current_companion].client &.post(endpoint, headers: headers, body: data.to_json)
       body = response.body
       if (response.status_code != 200)