Merge pull request #30 from catspeed-cc/new-dev-add-proxy

Merge: new-dev-add-proxy into new-development

README.md

@@ -68,7 +68,9 @@
 ## Features
 
 **Patches**
+
 - revert d9df90b
+- token updater patch (mooleshacat)
 
 **User features**
 - Lightweight

config/config.example.yml

@@ -173,6 +173,17 @@ https_only: false
 ##
 #force_resolve:
 
+##
+## Configuration for using a HTTP proxy
+##
+## If unset, then no HTTP proxy will be used.
+## 
+http_proxy:
+  user:
+  password:
+  host:
+  port:
+
 
 ##
 ## Use Innertube's transcripts API instead of timedtext for closed captions

shard.lock

@@ -10,7 +10,7 @@ shards:
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
-    version: 1.2.1
+    version: 1.2.2
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
@@ -20,6 +20,10 @@ shards:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.2.2
 
+  http_proxy:
+    git: https://github.com/mamantoha/http_proxy.git
+    version: 0.10.3
+
   kemal:
     git: https://github.com/kemalcr/kemal.git
     version: 1.1.2
@@ -42,7 +46,7 @@ shards:
 
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
-    version: 0.10.4
+    version: 0.10.6
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git

shard.yml

@@ -28,6 +28,9 @@ dependencies:
   athena-negotiation:
     github: athena-framework/negotiation
     version: ~> 0.1.1
+  http_proxy:
+    github: mamantoha/http_proxy
+    version: ~> 0.10.3
 
 development_dependencies:
   spectator:

src/invidious.cr

@@ -23,6 +23,7 @@ require "kilt"
 require "./ext/kemal_content_for.cr"
 require "./ext/kemal_static_file_handler.cr"
 
+require "http_proxy"
 require "athena-negotiation"
 require "openssl/hmac"
 require "option_parser"
@@ -189,6 +190,8 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL
 
 Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
 
+Invidious::Jobs.register Invidious::Jobs::MonitorCfgTokensJob.new()
+
 Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
 
 Invidious::Jobs.start_all

src/invidious/config.cr

@@ -55,6 +55,15 @@ struct ConfigPreferences
   end
 end
 
+struct HTTPProxyConfig
+  include YAML::Serializable
+
+  property user : String
+  property password : String
+  property host : String
+  property port : Int32
+end
+
 class Config
   include YAML::Serializable
 
@@ -129,6 +138,8 @@ class Config
   property host_binding : String = "0.0.0.0"
   # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
   property pool_size : Int32 = 100
+  # HTTP Proxy configuration
+  property http_proxy : HTTPProxyConfig? = nil
 
   # Use Innertube's transcripts API instead of timedtext for closed captions
   property use_innertube_for_captions : Bool = false

src/invidious/helpers/crystal_class_overrides.cr

@@ -18,6 +18,40 @@ end
 class HTTP::Client
   property family : Socket::Family = Socket::Family::UNSPEC
 
+  # Override stdlib to automatically initialize proxy if configured
+  #
+  # Accurate as of crystal 1.12.1
+
+  def initialize(@host : String, port = nil, tls : TLSContext = nil)
+    check_host_only(@host)
+
+    {% if flag?(:without_openssl) %}
+      if tls
+        raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time"
+      end
+      @tls = nil
+    {% else %}
+      @tls = case tls
+             when true
+               OpenSSL::SSL::Context::Client.new
+             when OpenSSL::SSL::Context::Client
+               tls
+             when false, nil
+               nil
+             end
+    {% end %}
+
+    @port = (port || (@tls ? 443 : 80)).to_i
+
+    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  end
+
+  def initialize(@io : IO, @host = "", @port = 80)
+    @reconnect = false
+
+    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  end
+
   private def io
     io = @io
     return io if io

src/invidious/jobs/token_monitor.cr

@@ -0,0 +1,17 @@
+
+class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob
+  include Invidious
+  def begin
+    loop do
+        
+      LOGGER.info("jobs: running MonitorCfgTokensJob job")
+    
+      ReloadTokens.get_tokens
+    
+      LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadTokens.pot.as(String))
+      LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadTokens.vdata.as(String))
+    
+      sleep 15.seconds
+    end
+  end
+end

src/invidious/reloadtoken.cr

@@ -0,0 +1,83 @@
+class ReloadTokens
+
+  @@instance = new
+  
+  def self.pot
+    @@pot
+  end
+  
+  def self.vdata
+    @@vdata
+  end  
+  
+  def initialize
+    
+    @@pot = "error"
+    @@vdata = "error"
+    
+  end
+  
+  def self.get_tokens
+
+    # Load config from file or YAML string env var
+    env_config_file = "INVIDIOUS_CONFIG_FILE"
+    env_config_yaml = "INVIDIOUS_CONFIG"
+
+    config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
+    config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
+
+    config = Config.from_yaml(config_yaml)
+    
+    
+    
+    
+    # Update config from env vars (upcased and prefixed with "INVIDIOUS_")
+    {% for ivar in Config.instance_vars %}
+        {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
+
+        if ENV.has_key?({{env_id}})
+            env_value = ENV.fetch({{env_id}})
+            success = false
+
+            # Use YAML converter if specified
+            {% ann = ivar.annotation(::YAML::Field) %}
+            {% if ann && ann[:converter] %}
+                config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
+                success = true
+
+            # Use regular YAML parser otherwise
+            {% else %}
+                {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %}
+                # Sort types to avoid parsing nulls and numbers as strings
+                {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %}
+                {{ivar_types}}.each do |ivar_type|
+                    if !success
+                        begin
+                            config.{{ivar.id}} = ivar_type.from_yaml(env_value)
+                            success = true
+                        rescue
+                            # nop
+                        end
+                    end
+                end
+            {% end %}
+
+            # Exit on fail
+            if !success
+                puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}})
+                exit(1)
+            end
+        end
+    {% end %}
+
+
+    @@pot = config.po_token
+    @@vdata = config.visitor_data
+
+  end
+  
+  def self.get_instance
+    return @@instance
+  end
+  
+end

src/invidious/yt_backend/connection_pool.cr

@@ -26,12 +26,16 @@ struct YoutubeConnectionPool
 
   def client(&)
     conn = pool.checkout
+    # Proxy needs to be reinstated every time we get a client from the pool
+    conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+
     begin
       response = yield conn
     rescue ex
       conn.close
-      conn = HTTP::Client.new(url)
 
+      conn = HTTP::Client.new(url)
+      conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@@ -77,3 +81,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
     client.close
   end
 end
+
+def make_configured_http_proxy_client
+  # This method is only called when configuration for an HTTP proxy are set
+  config_proxy = CONFIG.http_proxy.not_nil!
+
+  return HTTP::Proxy::Client.new(
+    config_proxy.host,
+    config_proxy.port,
+
+    username: config_proxy.user,
+    password: config_proxy.password,
+  )
+end

src/invidious/yt_backend/youtube_api.cr

@@ -320,8 +320,8 @@ module YoutubeAPI
       client_context["client"]["platform"] = platform
     end
 
-    if CONFIG.visitor_data.is_a?(String)
+    if ReloadTokens.vdata.is_a?(String)
-      client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)
+      client_context["client"]["visitorData"] = ReloadTokens.vdata.as(String)
     end
 
     return client_context
@@ -482,7 +482,7 @@ module YoutubeAPI
         "contentPlaybackContext" => playback_ctx,
       },
       "serviceIntegrityDimensions" => {
-        "poToken" => CONFIG.po_token,
+        "poToken" => ReloadTokens.pot.as(String),
       },
     }
 
@@ -616,8 +616,8 @@ module YoutubeAPI
       headers["User-Agent"] = user_agent
     end
 
-    if CONFIG.visitor_data.is_a?(String)
+    if ReloadTokens.vdata.is_a?(String)
-      headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)
+      headers["X-Goog-Visitor-Id"] = ReloadTokens.vdata.as(String)
     end
 
     # Logging