Merge remote-tracking branch 'upstream/master'

.github/workflows/ci.yml

@@ -99,7 +99,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Use ARM64 Dockerfile if ARM64
-        if: ${{ matrix.name }} == "ARM64"
+        if: ${{ matrix.name == 'ARM64' }}
         run: sed -i 's/Dockerfile/Dockerfile.arm64/' docker-compose.yml
 
       - name: Build Docker

.github/workflows/stale.yml

@@ -10,7 +10,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v9
+    - uses: actions/stale@v10
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 730

config/config.example.yml

@@ -61,31 +61,32 @@ db:
 ## When this setting is commented out, Invidious companion is not used.
 ## Otherwise, Invidious will proxy the requests to Invidious companion.
 ## 
-## Note: multiple URL can be configured. In this case, invidious will
+## Note: multiple URL can be configured. In this case, Invidious will
 ## randomly pick one every time video data needs to be retrieved. This
 ## URL is then kept in the video metadata cache to allow video playback
 ## to work. Once said cache has expired, requesting that video's data
 ## again will cause a new companion URL to be picked.
 ##
-## The parameter private_url needs to be configured for the internal
-## communication between the companion and Invidious.
-## And public_url is the public URL from which companion is listening
-## to the requests from the user(s).
+## The parameter private_url is required for the internal communication
+## between Invidious companion and Invidious.
 ##
-## If you are using a reverse proxy then you will probably need to
-## configure the public_url to be the same as the domain used for Invidious.
-## Also apply when used from an external IP address (without a domain).
-## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282
-##
-## Both parameter can have identical URL when Invidious is hosted in
-## an internal network or at home or locally (localhost).
+## The optional parameter public_url is the public URL from which 
+## Invidious companion is listening to the requests from the user(s).
+## When this setting is commented out, Invidious proxy all requests to
+## Invidious companion. Useful for simple setups.
+## Otherwise, requests from the user(s) will reach Invidious companion directly.
+## And you will need to configure a reverse proxy with separate routes
+## for Invidious and Invidious companion.
+## Read the post-install documentation for advanced reverse proxy
+## documentation: https://docs.invidious.io/installation/#post-install-configuration
 ##
 ## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
 ## Default: <none>
 ##
 #invidious_companion:
-#  - private_url: "http://localhost:8282"
-#    public_url: "http://localhost:8282"
+#  - private_url: "http://localhost:8282/companion"
+#  # Uncomment for advanced reverse proxy configuration (see above).
+#  # public_url: "http://localhost:8282/companion"
 
 ##
 ## API key for Invidious companion, used for securing the communication

locales/en-US.json

@@ -125,6 +125,8 @@
     "preferences_hidden_channels": "Hidden channels",
     "preferences_hidden_channels_label": "(Experimental) Channel ID list separated by ENTER key. This only hides channels from the popular page for now. You can get the ID of the channel clicking on the channel and copying the part that starts with 'UC' in the channel link. Example: /channel/<b>UCw-aR42z5gUcarpPGN5OKfA</b>",
     "preferences_default_trending_type": "Default trending page: ",
+    "preferences_default_playlist": "Default playlist: ",
+    "preferences_default_playlist_none": "No default playlist set",
     "published": "published",
     "published - reverse": "published - reverse",
     "alphabetically": "alphabetically",

locales/es.json

@@ -81,6 +81,8 @@
     "preferences_hidden_channels": "Canales ocultos",
     "preferences_hidden_channels_label": "(Experimental) Lista de IDs de canales separados por la tecla ENTER. Por ahora, esto solo oculta canales de la pagina popular. Puedes conseguir la ID del canal entrando al canal y copiando la parte que empieza por 'UC' en el enlace. Ejemplo: /channel/<b>UCw-aR42z5gUcarpPGN5OKfA</b>",
     "preferences_default_trending_type": "Página de tendencias por defecto: ",
+    "preferences_default_playlist": "Lista de reproducción por defecto: ",
+    "preferences_default_playlist_none": "Ninguna lista de reproducción por defecto establecida",
     "published": "fecha de publicación",
     "published - reverse": "fecha de publicación: orden inverso",
     "alphabetically": "alfabéticamente",

src/invidious/config.cr

@@ -52,6 +52,8 @@ struct ConfigPreferences
   property vr_mode : Bool = true
   property show_nick : Bool = true
   property save_player_pos : Bool = false
+  @[YAML::Field(ignore: true)]
+  property default_playlist : String? = nil
   property enable_dearrow : Bool = false
   @[YAML::Field(ignore: true)]
   property hidden_channels : Array(String)? = nil
@@ -93,6 +95,9 @@ class Config
 
     property note : String = ""
     property domain : Array(String) = [] of String
+
+    # Indicates if this companion instance uses the built-in proxy
+    property builtin_proxy : Bool = false
   end
 
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
@@ -399,6 +404,14 @@ class Config
         puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 characters."
         exit(1)
       end
+
+      # Set public_url to built-in proxy path when omitted
+      config.invidious_companion.each do |companion|
+        if companion.public_url.to_s.empty?
+          companion.public_url = URI.parse("/companion")
+          companion.builtin_proxy = true
+        end
+      end
     elsif config.signature_server
       puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/companion-installation/")
     else

src/invidious/helpers/handlers.cr

@@ -61,28 +61,13 @@ class Kemal::ExceptionHandler
   end
 end
 
-class FilteredCompressHandler < Kemal::Handler
+class FilteredCompressHandler < HTTP::CompressHandler
   exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/sb/*", "/ggpht/*", "/api/v1/auth/notifications"]
   exclude ["/api/v1/auth/notifications", "/data_control"], "POST"
 
-  def call(env)
-    return call_next env if exclude_match? env
-
-    {% if flag?(:without_zlib) %}
-      call_next env
-    {% else %}
-      request_headers = env.request.headers
-
-      if request_headers.includes_word?("Accept-Encoding", "gzip")
-        env.response.headers["Content-Encoding"] = "gzip"
-        env.response.output = Compress::Gzip::Writer.new(env.response.output, sync_close: true)
-      elsif request_headers.includes_word?("Accept-Encoding", "deflate")
-        env.response.headers["Content-Encoding"] = "deflate"
-        env.response.output = Compress::Deflate::Writer.new(env.response.output, sync_close: true)
-      end
-
-      call_next env
-    {% end %}
+  def call(context)
+    return call_next context if exclude_match? context
+    super
   end
 end
 

src/invidious/routes/before_all.cr

@@ -137,6 +137,7 @@ module Invidious::Routes::BeforeAll
                 "/videoplayback",
                 "/latest_version",
                 "/download",
+                "/companion/",
               }.any? { |r| env.request.resource.starts_with? r }
 
     if env.request.cookies.has_key? "SID"

src/invidious/routes/companion.cr

@@ -0,0 +1,47 @@
+module Invidious::Routes::Companion
+  # /companion
+  def self.get_companion(env)
+    current_companion = env.get("current_companion").as(Int32)
+
+    url = env.request.path
+    if env.request.query
+      url += "?#{env.request.query}"
+    end
+
+    begin
+      COMPANION_POOL[current_companion].client do |wrapper|
+        wrapper.client.get(url, env.request.headers) do |resp|
+          return self.proxy_companion(env, resp)
+        end
+      end
+    rescue ex
+    end
+  end
+
+  def self.options_companion(env)
+    current_companion = env.get("current_companion").as(Int32)
+
+    url = env.request.path
+    if env.request.query
+      url += "?#{env.request.query}"
+    end
+
+    begin
+      COMPANION_POOL[current_companion].client do |wrapper|
+        wrapper.client.options(url, env.request.headers) do |resp|
+          return self.proxy_companion(env, resp)
+        end
+      end
+    rescue ex
+    end
+  end
+
+  private def self.proxy_companion(env, response)
+    env.response.status_code = response.status_code
+    response.headers.each do |key, value|
+      env.response.headers[key] = value
+    end
+
+    return IO.copy response.body_io, env.response
+  end
+end

src/invidious/routes/embed.cr

@@ -210,6 +210,18 @@ module Invidious::Routes::Embed
     if CONFIG.invidious_companion.present?
       current_companion = env.get("current_companion").as(Int32)
       invidious_companion = CONFIG.invidious_companion[current_companion]
+
+      invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
+        uri =
+          "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
+      end.join(" ")
+
+      if !invidious_companion_urls.empty?
+        env.response.headers["Content-Security-Policy"] =
+          env.response.headers["Content-Security-Policy"]
+            .gsub("media-src", "media-src #{invidious_companion_urls}")
+            .gsub("connect-src", "connect-src #{invidious_companion_urls}")
+      end
     end
 
     rendered "embed"

src/invidious/routes/preferences.cr

@@ -144,6 +144,8 @@ module Invidious::Routes::PreferencesRoute
     notifications_only ||= "off"
     notifications_only = notifications_only == "on"
 
+    default_playlist = env.params.body["default_playlist"]?.try &.as(String)
+
     hidden_channels = env.params.body["hidden_channels"]?.try &.as(String)
     if hidden_channels
       hidden_channels = hidden_channels.split("\n")
@@ -171,6 +173,7 @@ module Invidious::Routes::PreferencesRoute
     default_trending_type = env.params.body["default_trending_type"]?.try &.as(String)
     default_trending_type ||= Invidious::Routes::Feeds::TrendingTypes::Default
 
+
     # Convert to JSON and back again to take advantage of converters used for compatibility
     preferences = Preferences.from_json({
       annotations:                 annotations,
@@ -207,6 +210,7 @@ module Invidious::Routes::PreferencesRoute
       vr_mode:                     vr_mode,
       show_nick:                   show_nick,
       save_player_pos:             save_player_pos,
+      default_playlist:            default_playlist,
       hidden_channels:             hidden_channels,
       default_trending_type:       default_trending_type,
     }.to_json)

src/invidious/routes/watch.cr

@@ -279,6 +279,18 @@ module Invidious::Routes::Watch
     if CONFIG.invidious_companion.present?
       current_companion = env.get("current_companion").as(Int32)
       invidious_companion = CONFIG.invidious_companion[current_companion]
+
+      invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
+        uri =
+          "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
+      end.join(" ")
+
+      if !invidious_companion_urls.empty?
+        env.response.headers["Content-Security-Policy"] =
+          env.response.headers["Content-Security-Policy"]
+            .gsub("media-src", "media-src #{invidious_companion_urls}")
+            .gsub("connect-src", "connect-src #{invidious_companion_urls}")
+      end
     end
 
     templated "watch"

src/invidious/routing.cr

@@ -47,6 +47,7 @@ module Invidious::Routing
     self.register_api_v1_routes
     self.register_api_manifest_routes
     self.register_video_playback_routes
+    self.register_companion_routes
   end
 
   # -------------------
@@ -191,7 +192,7 @@ module Invidious::Routing
   end
 
   # -------------------
-  #  Media proxy routes
+  #  Proxy routes
   # -------------------
 
   def register_api_manifest_routes
@@ -226,6 +227,13 @@ module Invidious::Routing
     get "/vi/:id/:name", Routes::Images, :thumbnails
   end
 
+  def register_companion_routes
+    if CONFIG.invidious_companion.present?
+      get "/companion/*", Routes::Companion, :get_companion
+      options "/companion/*", Routes::Companion, :options_companion
+    end
+  end
+
   # -------------------
   #  API routes
   # -------------------

src/invidious/user/preferences.cr

@@ -58,6 +58,7 @@ struct Preferences
   property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
   property hidden_channels : Array(String)? = nil
   property default_trending_type : Invidious::Routes::Feeds::TrendingTypes = Invidious::Routes::Feeds::TrendingTypes::Default
+  property default_playlist : String? = nil
 
   module BoolToString
     def self.to_json(value : String, json : JSON::Builder)

src/invidious/videos/parser.cr

@@ -102,6 +102,9 @@ def extract_video_info(video_id : String, env : HTTP::Server::Context | Nil = ni
   # Don't fetch the next endpoint if the video is unavailable.
   if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
     next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+    # Remove the microformat returned by the /next endpoint on some videos
+    # to prevent player_response microformat from being overwritten.
+    next_response.delete("microformat")
     player_response = player_response.merge(next_response)
   end
 

src/invidious/views/components/player.ecr

@@ -68,12 +68,18 @@
             <% end %>
         <% end %>
 
-        <% preferred_captions.each do |caption| %>
-            <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
+        <% preferred_captions.each do |caption|
+            api_captions_url = "/api/v1/captions/"
+            api_captions_url = invidious_companion.public_url.to_s + api_captions_url if (invidious_companion)
+        %>
+            <track kind="captions" src="<%= api_captions_url %><%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
         <% end %>
 
-        <% captions.each do |caption| %>
-            <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
+        <% captions.each do |caption|
+            api_captions_url = "/api/v1/captions/"
+            api_captions_url = invidious_companion.public_url.to_s + api_captions_url if (invidious_companion)
+        %>
+            <track kind="captions" src="<%= api_captions_url %><%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
         <% end %>
     <% end %>
 </video>

src/invidious/views/user/preferences.ecr

@@ -129,6 +129,19 @@
                 <input name="save_player_pos" id="save_player_pos" type="checkbox" <% if preferences.save_player_pos %>checked<% end %>>
             </div>
 
+            <% if user = env.get?("user").try &.as(User) %>
+                <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %>
+                <div class="pure-control-group">
+                    <label for="default_playlist"><%= translate(locale, "preferences_default_playlist") %></label>
+                    <select name="default_playlist" id="default_playlist">
+                        <option value=""><%= translate(locale, "preferences_default_playlist_none") %></option>
+                        <% playlists.each do |plid, playlist_title| %>
+                            <option value="<%= plid %>" <%= "selected" if user.preferences.default_playlist == plid %>><%= HTML.escape(playlist_title) %></option>
+                        <% end %>
+                    </select>
+                </div>
+            <% end %>
+
             <legend><%= translate(locale, "preferences_category_visual") %></legend>
 
             <div class="pure-control-group">

src/invidious/views/watch.ecr

@@ -190,7 +190,7 @@ we're going to need to do it here in order to allow for translations.
                             <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label>
                             <select style="width:100%" name="playlist_id" id="playlist_id">
                                 <% playlists.each do |plid, playlist_title| %>
-                                    <option data-plid="<%= plid %>" value="<%= plid %>"><%= HTML.escape(playlist_title) %></option>
+                                    <option data-plid="<%= plid %>" value="<%= plid %>" <%= "selected" if user.preferences.default_playlist == plid %>><%= HTML.escape(playlist_title) %></option>
                                 <% end %>
                             </select>
                         </div>

src/invidious/yt_backend/connection_pool.cr

@@ -46,11 +46,30 @@ struct YoutubeConnectionPool
   end
 end
 
-class CompanionConnectionPool
-  property pool : DB::Pool(HTTP::Client)
-  property companion : URI
+# Packages a `HTTP::Client` to an Invidious companion instance alongside the configuration for that instance.
+#
+# This is used as the resource for the `CompanionPool` as to allow the ability to
+# proxy the requests to Invidious companion from Invidious directly.
+# Instead of setting up routes in a reverse proxy.
+struct CompanionWrapper
+  property client : HTTP::Client
+  property companion : Config::CompanionConfig
 
-  def initialize(companion, capacity = 5, timeout = 5.0)
+  def initialize(companion : Config::CompanionConfig)
+    @companion = companion
+    @client = make_client(companion.private_url, use_http_proxy: false)
+  end
+
+  def close
+    @client.close
+  end
+end
+
+class CompanionConnectionPool
+  property pool : DB::Pool(CompanionWrapper)
+  property companion : Config::CompanionConfig
+
+  def initialize(@companion, capacity = 5, timeout = 5.0)
     options = DB::Pool::Options.new(
       initial_pool_size: 0,
       max_pool_size: capacity,
@@ -58,26 +77,26 @@ class CompanionConnectionPool
       checkout_timeout: timeout
     )
 
-    @companion = companion.private_url
-
-    @pool = DB::Pool(HTTP::Client).new(options) do
-      next make_client(@companion, use_http_proxy: false)
+    @pool = DB::Pool(CompanionWrapper).new(options) do
+      make_client(@companion.private_url, use_http_proxy: false)
+      CompanionWrapper.new(companion: @companion)
     end
   end
 
   def client(&)
-    conn = pool.checkout
+    wrapper = pool.checkout
 
     begin
-      response = yield conn
+      response = yield wrapper
     rescue ex
-      conn.close
+      wrapper.close
 
-      conn = make_client(@companion, use_http_proxy: false)
+      make_client(@companion.private_url, use_http_proxy: false)
+      wrapper = CompanionWrapper.new(companion: @companion)
 
-      response = yield conn
+      response = yield wrapper
     ensure
-      pool.release(conn)
+      pool.release(wrapper)
     end
 
     response

src/invidious/yt_backend/youtube_api.cr

@@ -709,22 +709,20 @@ module YoutubeAPI
       else
         current_companion = env.get("current_companion").as(Int32)
       end
-      response = COMPANION_POOL[current_companion].client &.post(endpoint, headers: headers, body: data.to_json)
-      body = response.body
-      if (response.status_code != 200)
-        raise Exception.new(
-          "Error while communicating with Invidious companion: \
-          status code: #{response.status_code} and body: #{body.dump}"
-        )
+      response_body = Hash(String, JSON::Any).new
+
+      COMPANION_POOL[current_companion].client do |wrapper|
+        companion_base_url = wrapper.companion.private_url.path
+
+        wrapper.client.post("#{companion_base_url}#{endpoint}", headers: headers, body: data.to_json) do |response|
+          response_body = JSON.parse(response.body_io).as_h
         end
+      end
+
+      return response_body
     rescue ex
       raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
     end
-
-    # Convert result to Hash
-    initial_data = JSON.parse(body).as_h
-
-    return initial_data
   end
 
   ####################################################################