From 65463333f32384d966c288edb46294336445a498 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 11 Dec 2025 17:28:20 -0300 Subject: [PATCH 01/17] Display "Erroneous CAPTCHA" for invalid captchas (#5508) --- src/invidious/routes/login.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index e7de5018..674f0a46 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -98,6 +98,8 @@ module Invidious::Routes::Login begin validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) + rescue ex : InfoException + return error_template(400, InfoException.new("Erroneous CAPTCHA")) rescue ex return error_template(400, ex) end From 994c25de2ec437c0c9c24f9d5d7e982a5811a951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20H=C3=A4drich?= <11225821+shaedrich@users.noreply.github.com> Date: Sun, 14 Dec 2025 23:30:52 +0100 Subject: [PATCH 02/17] Add link to GitHub release/tag/commit in footer (#4702) * Add link to GitHub release/tag/commit in footer * Only show tag if there is one Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --------- Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious.cr | 1 + src/invidious/views/template.ecr | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index 7fa0725e..4dd5d1dd 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -84,6 +84,7 @@ HTTP_CHUNK_SIZE = 10485760 # ~10MB CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} +CURRENT_TAG = {{ "#{`git tag --points-at HEAD`.strip}" }} # This is used to determine the `?v=` on the end of file URLs (for cache busting). We # only need to expire modified assets, so we can use this to find the last commit that changes diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 9bf33918..0e0f2e16 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -150,7 +150,24 @@ <%= translate(locale, "footer_donate_page") %> - <%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %> + + <%= translate(locale, "Current version: ") %> + <% if CONFIG.modified_source_code_url %> + <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> + <% else %> + <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> + <% end %> + @ <%= CURRENT_BRANCH %> + <% if CURRENT_TAG != "" %> + ( + <% if CONFIG.modified_source_code_url %> + <%= CURRENT_TAG %> + <% else %> + <%= CURRENT_TAG %> + <% end %> + ) + <% end %> + From aba31a8e20edf9cea632e72ab5ff42c04270a271 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 15 Dec 2025 04:21:55 -0300 Subject: [PATCH 03/17] Set Kemal `max_request_line_size` to 16384 for large channel continuation query parameters. (#5566) * feat: Add configurable max_request_line_size to handle long URLs This commit adds a new configuration option `max_request_line_size` that allows users to increase the HTTP request line size limit. This is particularly useful for handling very long continuation tokens that can cause 414 (URI Too Long) errors. Changes: - Add `max_request_line_size` property to Config class - Configure Kemal server to use the custom limit if specified - Document the option in config.example.yml with recommendations - Add examples in docker-compose.yml for both YAML and env var configuration The default behavior remains unchanged (8KB limit) unless explicitly configured. This provides a solution for users experiencing 414 errors without affecting existing installations. * Hardcode max_request_line_size to 16384 --------- Co-authored-by: Sunghyun Kim --- src/invidious.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/invidious.cr b/src/invidious.cr index 4dd5d1dd..2edc4702 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -250,6 +250,8 @@ Kemal.config.app_name = "Invidious" {% end %} Kemal.run do |config| + config.server.not_nil!.max_request_line_size = 16384 + if socket_binding = CONFIG.socket_binding File.delete?(socket_binding.path) # Create a socket and set its desired permissions From cf52a353662cce1ff97d294a14e2903ace52206b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 22:49:01 +0100 Subject: [PATCH 04/17] Bump actions/cache from 4 to 5 (#5569) Bumps [actions/cache](https://github.com/actions/cache) from 4 to 5. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/cache dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff82a5bd..b28873d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: crystal: ${{ matrix.crystal }} - name: Cache Shards - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ./lib @@ -139,7 +139,7 @@ jobs: crystal: latest - name: Cache Shards - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ./lib From eed8f25a3d91f63a91d4d9ce87454ee58f47c14a Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 18 Dec 2025 06:16:15 -0300 Subject: [PATCH 05/17] dockerfile: compile openssl instead of using the one bundled on the crystal alpine image. (#5441) * dockerfile: compile openssl instead of using the one bundled on the crystal alpine image. * fix formatting * CI: add --no-cache to openssl-builder * CI: add Dockerfile.arm64 version * add comment why we compile openssl ourselves * fix wrong position of comment * oopsie * verify openssl checksums * set nproc for openssl make * use ARG for openssl sha256 checksum --- docker/Dockerfile | 31 ++++++++++++++++++++++++++++++- docker/Dockerfile.arm64 | 31 +++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4cfc3c72..3e0d2f7f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,29 @@ -FROM crystallang/crystal:1.16.3-alpine AS builder +# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2 +ARG OPENSSL_VERSION='3.5.2' +ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec' + +FROM crystallang/crystal:1.16.3-alpine AS dependabot-crystal + +# We compile openssl ourselves due to a memory leak in how crystal interacts +# with openssl +# Reference: https://github.com/iv-org/invidious/issues/1438#issuecomment-3087636228 +FROM dependabot-crystal AS openssl-builder +RUN apk add --no-cache curl perl linux-headers + +WORKDIR / + +ARG OPENSSL_VERSION +ARG OPENSSL_SHA256 +RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" --output openssl-${OPENSSL_VERSION}.tar.gz +RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c +RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz + +RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc) + +FROM dependabot-crystal AS builder RUN apk add --no-cache sqlite-static yaml-static +RUN apk del openssl-dev openssl-libs-static ARG release @@ -21,12 +44,18 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml RUN crystal spec --warnings all \ --link-flags "-lxml2 -llzma" + +ARG OPENSSL_VERSION +COPY --from=openssl-builder /openssl-${OPENSSL_VERSION} /openssl-${OPENSSL_VERSION} + RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --release \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ else \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 758e7950..b02cc8ce 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,6 +1,28 @@ -FROM alpine:3.21 AS builder +# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2 +ARG OPENSSL_VERSION='3.5.2' +ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec' + +FROM alpine:3.21 AS dependabot-alpine + +# We compile openssl ourselves due to a memory leak in how crystal interacts +# with openssl +# Reference: https://github.com/iv-org/invidious/issues/1438#issuecomment-3087636228 +FROM dependabot-alpine AS openssl-builder +RUN apk add --no-cache curl perl linux-headers build-base + +WORKDIR / + +ARG OPENSSL_VERSION +ARG OPENSSL_SHA256 +RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" --output openssl-${OPENSSL_VERSION}.tar.gz +RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c +RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz + +RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc) + +FROM dependabot-alpine AS builder RUN apk add --no-cache 'crystal=1.14.0-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \ - zlib-static openssl-libs-static openssl-dev musl-dev xz-static + zlib-static musl-dev xz-static ARG release @@ -22,12 +44,17 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml RUN crystal spec --warnings all \ --link-flags "-lxml2 -llzma" +ARG OPENSSL_VERSION +COPY --from=openssl-builder /openssl-${OPENSSL_VERSION} /openssl-${OPENSSL_VERSION} + RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --release \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ else \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ From d2be57a4546b67679b2507241b6d9f3f6c880244 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 07:50:21 -0700 Subject: [PATCH 06/17] Replace `Kemal::StaticFileHandler` on Crystal < 1.17.0 Kemal's subclass of the stdlib `HTTP::StaticFileHandler` is not as maintained as its parent, and so misses out on many enhancements and bug fixes from upstream, which unfortunately also includes the patches for security vulnerabilities... Though this isn't necessarily Kemal's fault since the bulk of the stdlib handler's logic was done in a single big method, making any changes hard to maintain. This was fixed in Crystal 1.17.0 where the handler was refactored into many private methods, making it easier for an inheriting type to implement custom behaviors while still leveraging much of the pre-existing code. Since we don't actually use any of the Kemal specific features added by `Kemal::StaticFileHandler`, there really isn't a reason to not just create a new handler based upon the stdlib implementation instead which will address the problems mentioned above. This PR implements a new handler which inherits from the stdlib variant and overrides the helper methods added in Crystal 1.17.0 to add the caching behavior with minimal code changes. Since this new handler depends on the code in Crystal 1.17.0, it will only be applied on versions greater than or equal to 1.17.0. On older versions we'll fallback to the current monkey patched `Kemal::StaticFileHandler` --- src/ext/kemal_static_file_handler.cr | 21 +++ src/invidious.cr | 18 ++- .../http_server/static_assets_handler.cr | 138 ++++++++++++++++++ 3 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 src/invidious/http_server/static_assets_handler.cr diff --git a/src/ext/kemal_static_file_handler.cr b/src/ext/kemal_static_file_handler.cr index a5f42261..c6b9a27d 100644 --- a/src/ext/kemal_static_file_handler.cr +++ b/src/ext/kemal_static_file_handler.cr @@ -1,3 +1,24 @@ +{% if compare_versions(Crystal::VERSION, "1.17.0") >= 0 %} + # Strip StaticFileHandler from the binary + # + # This allows us to compile on 1.17.0 as the compiler won't try to + # semantically check the outdated upstream code. + class Kemal::Config + private def setup_static_file_handler + end + end + + # Nullify `Kemal::StaticFileHandler` + # + # Needed until the next release of Kemal after 1.7 + class Kemal::StaticFileHandler < HTTP::StaticFileHandler + def call(context : HTTP::Server::Context) + end + end + + {% skip_file %} +{% end %} + # Since systems have a limit on number of open files (`ulimit -a`), # we serve them from memory to avoid 'Too many open files' without needing # to modify ulimit. diff --git a/src/invidious.cr b/src/invidious.cr index 2edc4702..ea5b9c63 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -223,19 +223,25 @@ error 500 do |env, exception| error_template(500, exception) end -static_headers do |env| - env.response.headers.add("Cache-Control", "max-age=2629800") -end - # Init Kemal -public_folder "assets" - Kemal.config.powered_by_header = false add_handler FilteredCompressHandler.new add_handler APIHandler.new add_handler AuthHandler.new add_handler DenyFrame.new + +{% if compare_versions(Crystal::VERSION, "1.17.0") >= 0 %} + Kemal.config.serve_static = false + add_handler Invidious::HttpServer::StaticAssetsHandler.new("assets", directory_listing: false) +{% else %} + public_folder "assets" + + static_headers do |env| + env.response.headers.add("Cache-Control", "max-age=2629800") + end +{% end %} + add_context_storage_type(Array(String)) add_context_storage_type(Preferences) add_context_storage_type(Invidious::User) diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr new file mode 100644 index 00000000..c6137775 --- /dev/null +++ b/src/invidious/http_server/static_assets_handler.cr @@ -0,0 +1,138 @@ +{% skip_file if compare_versions(Crystal::VERSION, "1.17.0") < 0 %} + +module Invidious::HttpServer + class StaticAssetsHandler < HTTP::StaticFileHandler + # In addition to storing the actual data of a file, it also implements the required + # getters needed for the object to imitate a `File::Stat` within `StaticFileHandler`. + # + # Since the `File::Stat` is created once in `#call` and then passed around to the + # rest of the class's methods, imitating the object allows us to only lookup + # the cache hash once for every request. + # + private record CachedFile, data : Bytes, size : Int64, modification_time : Time + + CACHE_LIMIT = 5_000_000 # 5MB + @@cached_files = {} of Path => CachedFile + + # A simplified version of `#call` for Invidious to improve performance. + # + # This is basically the same as what we inherited but just with the directory listing + # features stripped out. This removes some conditional checks and calls which improves + # performance slightly but otherwise is entirely unneeded. + # + # Really, all the cache feature actually needs is to override the much simplifier `file_info` + # method to return a `CachedFile` or `File::Stat` depending on whether the file is cached. + def call(context) : Nil + check_request_method!(context) || return + + request_path = request_path(context) + + check_request_path!(context, request_path) || return + + request_path = Path.posix(request_path) + expanded_path = request_path.expand("/") + + # The path normalization can be simplified to just this since + # we don't need to care about normalizing directory urls. + if request_path != expanded_path + redirect_to context, expanded_path + end + + file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native)) + + if cached_info = @@cached_files[file_path]? + return serve_file_with_cache(context, cached_info, file_path) + end + + file_info = File.info?(file_path) + + return call_next(context) unless file_info + + if file_info.file? + # Actually means to serve file *with cache headers* + # The actual logic for serving the file is done in `#serve_file` + serve_file_with_cache(context, file_info, file_path) + else # Not a normal file (FIFO/device/socket) + call_next(context) + end + end + + # Add "Cache-Control" header to the response + private def add_cache_headers(response_headers : HTTP::Headers, last_modified : Time) : Nil + super; response_headers["Cache-Control"] = "max-age=2629800" + end + + # Serves and caches the file at the given path. + # + # This is an override of `serve_file` to allow serving a file from memory, and to cache it + # it as needed. + private def serve_file(context : HTTP::Server::Context, file_info, file_path : Path, original_file_path : Path, last_modified : Time) + context.response.content_type = MIME.from_filename(original_file_path.to_s, "application/octet-stream") + + range_header = context.request.headers["Range"]? + + if !file_info.is_a? CachedFile + retrieve_bytes_from = IO::Memory.new + + File.open(file_path) do |file| + # We cannot cache partial data so we'll rewind and read from the start + if range_header + dispatch_serve(context, file, file_info, range_header) + IO.copy(file.rewind, retrieve_bytes_from) + else + context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true) + dispatch_serve(context, file, file_info, range_header) + end + end + + return flush_io_to_cache(retrieve_bytes_from, file_path, file_info) + else + return dispatch_serve(context, file_info.data, file_info, range_header) + end + end + + # Writes file data to the cache + private def flush_io_to_cache(io, file_path, file_info) + if @@cached_files.sum(&.[1].size) + (size = file_info.size) < CACHE_LIMIT + data_slice = io.to_slice + @@cached_files[file_path] = CachedFile.new(data_slice, file_info.size, file_info.modification_time) + end + end + + # Either send the file in full, or just fragments of it depending on the request + private def dispatch_serve(context, file, file_info, range_header) + if range_header + # an IO is needed for `serve_file_range` + file = file.is_a?(Bytes) ? IO::Memory.new(file, writeable: false) : file + serve_file_range(context, file, range_header, file_info) + else + context.response.headers["Accept-Ranges"] = "bytes" + serve_file_full(context, file, file_info) + end + end + + # Skips the stdlib logic for serving pre-gzipped files + private def serve_file_compressed(context : HTTP::Server::Context, file_info, file_path : Path, last_modified : Time) + serve_file(context, file_info, file_path, file_path, last_modified) + end + + # If we're serving the full file right away then there's no need for an IO at all. + private def serve_file_full(context : HTTP::Server::Context, file : Bytes, file_info) + context.response.status = :ok + context.response.content_length = file_info.size + context.response.write file + end + + # Serves segments of a file based on the `Range header` + # + # An override of `serve_file_range` to allow using a generic IO rather than a `File`. + # Literally the same code as what we inherited but just with the `file` argument's type + # being set to `IO` rather than `File` + # + # Can be removed once https://github.com/crystal-lang/crystal/issues/15817 is fixed. + private def serve_file_range(context : HTTP::Server::Context, file : IO, range_header : String, file_info) + # Paste in the body of inherited serve_file_range + {{@type.superclass.methods.select(&.name.==("serve_file_range"))[0].body}} + end + end +end From ddfbed68f7e01d16d6807dd1544a6ac340e85a93 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 09:04:33 -0700 Subject: [PATCH 07/17] Simplify `StaticAssetsHandler` implementation Overriding `#call` or patching out `serve_file_compressed` provides only minimal benefits over the ease of maintenance granted by only overriding what we need to for the caching behavior. --- .../http_server/static_assets_handler.cr | 61 ++++++------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr index c6137775..7ea26dad 100644 --- a/src/invidious/http_server/static_assets_handler.cr +++ b/src/invidious/http_server/static_assets_handler.cr @@ -9,52 +9,30 @@ module Invidious::HttpServer # rest of the class's methods, imitating the object allows us to only lookup # the cache hash once for every request. # - private record CachedFile, data : Bytes, size : Int64, modification_time : Time + private record CachedFile, data : Bytes, size : Int64, modification_time : Time do + def directory? + false + end + + def file? + true + end + end CACHE_LIMIT = 5_000_000 # 5MB @@cached_files = {} of Path => CachedFile - # A simplified version of `#call` for Invidious to improve performance. + # Returns metadata for the requested file # - # This is basically the same as what we inherited but just with the directory listing - # features stripped out. This removes some conditional checks and calls which improves - # performance slightly but otherwise is entirely unneeded. + # If the requested file is cached, a `CachedFile` is returned instead of a `File::Stat`. + # This represents the metadata info of a cached file and implements all the methods of `File::Stat` that + # is used by the `StaticAssetsHandler`. # - # Really, all the cache feature actually needs is to override the much simplifier `file_info` - # method to return a `CachedFile` or `File::Stat` depending on whether the file is cached. - def call(context) : Nil - check_request_method!(context) || return - - request_path = request_path(context) - - check_request_path!(context, request_path) || return - - request_path = Path.posix(request_path) - expanded_path = request_path.expand("/") - - # The path normalization can be simplified to just this since - # we don't need to care about normalizing directory urls. - if request_path != expanded_path - redirect_to context, expanded_path - end - + # The `CachedFile` also stores the raw bytes of the cached file, and this method serves as the place where + # the cached file is retrieved if it exists. Though the data will only be read in `#serve_file` + private def file_info(expanded_path : Path) file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native)) - - if cached_info = @@cached_files[file_path]? - return serve_file_with_cache(context, cached_info, file_path) - end - - file_info = File.info?(file_path) - - return call_next(context) unless file_info - - if file_info.file? - # Actually means to serve file *with cache headers* - # The actual logic for serving the file is done in `#serve_file` - serve_file_with_cache(context, file_info, file_path) - else # Not a normal file (FIFO/device/socket) - call_next(context) - end + {@@cached_files[file_path]? || File.info?(file_path), file_path} end # Add "Cache-Control" header to the response @@ -111,11 +89,6 @@ module Invidious::HttpServer end end - # Skips the stdlib logic for serving pre-gzipped files - private def serve_file_compressed(context : HTTP::Server::Context, file_info, file_path : Path, last_modified : Time) - serve_file(context, file_info, file_path, file_path, last_modified) - end - # If we're serving the full file right away then there's no need for an IO at all. private def serve_file_full(context : HTTP::Server::Context, file : Bytes, file_info) context.response.status = :ok From 6fd1cb3585fed1faf0ea5edbfcdabe1337186fdc Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 09:23:28 -0700 Subject: [PATCH 08/17] Compare against 1.17.0-dev until full release --- src/ext/kemal_static_file_handler.cr | 2 +- src/invidious.cr | 2 +- src/invidious/http_server/static_assets_handler.cr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ext/kemal_static_file_handler.cr b/src/ext/kemal_static_file_handler.cr index c6b9a27d..16cb84fb 100644 --- a/src/ext/kemal_static_file_handler.cr +++ b/src/ext/kemal_static_file_handler.cr @@ -1,4 +1,4 @@ -{% if compare_versions(Crystal::VERSION, "1.17.0") >= 0 %} +{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %} # Strip StaticFileHandler from the binary # # This allows us to compile on 1.17.0 as the compiler won't try to diff --git a/src/invidious.cr b/src/invidious.cr index ea5b9c63..a61f91a9 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -231,7 +231,7 @@ add_handler APIHandler.new add_handler AuthHandler.new add_handler DenyFrame.new -{% if compare_versions(Crystal::VERSION, "1.17.0") >= 0 %} +{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %} Kemal.config.serve_static = false add_handler Invidious::HttpServer::StaticAssetsHandler.new("assets", directory_listing: false) {% else %} diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr index 7ea26dad..243d6a8d 100644 --- a/src/invidious/http_server/static_assets_handler.cr +++ b/src/invidious/http_server/static_assets_handler.cr @@ -1,4 +1,4 @@ -{% skip_file if compare_versions(Crystal::VERSION, "1.17.0") < 0 %} +{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %} module Invidious::HttpServer class StaticAssetsHandler < HTTP::StaticFileHandler From 9e482b48078a5e8646cd0df2999531a9ce3e12e5 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 16:35:40 -0700 Subject: [PATCH 09/17] Add specs for the new StaticAssetsHandler --- .../handlers/static_assets_handler/test.txt | 1 + .../handlers/static_assets_handler_spec.cr | 205 ++++++++++++++++++ .../http_server/static_assets_handler.cr | 7 + 3 files changed, 213 insertions(+) create mode 100644 spec/http_server/handlers/static_assets_handler/test.txt create mode 100644 spec/http_server/handlers/static_assets_handler_spec.cr diff --git a/spec/http_server/handlers/static_assets_handler/test.txt b/spec/http_server/handlers/static_assets_handler/test.txt new file mode 100644 index 00000000..70c379b6 --- /dev/null +++ b/spec/http_server/handlers/static_assets_handler/test.txt @@ -0,0 +1 @@ +Hello world \ No newline at end of file diff --git a/spec/http_server/handlers/static_assets_handler_spec.cr b/spec/http_server/handlers/static_assets_handler_spec.cr new file mode 100644 index 00000000..89c53014 --- /dev/null +++ b/spec/http_server/handlers/static_assets_handler_spec.cr @@ -0,0 +1,205 @@ +{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %} + +require "http" +require "spectator" +require "../../../src/invidious/http_server/static_assets_handler.cr" + +private def get_static_assets_handler + return Invidious::HttpServer::StaticAssetsHandler.new "spec/http_server/handlers/static_assets_handler", directory_listing: false +end + +# Slightly modified version of `handle` function from +# +# https://github.com/crystal-lang/crystal/blob/3f369d2c721e9462d9f6126cb0bcd4c6992f0225/spec/std/http/server/handlers/static_file_handler_spec.cr#L5 + +private def handle(request, handler : HTTP::Handler? = nil, decompress : Bool = false) + io = IO::Memory.new + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) + + if !handler + handler = get_static_assets_handler + get_static_assets_handler.call context + else + handler.call(context) + end + + response.close + io.rewind + + HTTP::Client::Response.from_io(io, decompress: decompress) +end + +# Makes and yields a temporary file with the given prefix +private def make_temporary_file(prefix, contents = nil, &) + tempfile = File.tempfile(prefix, "static_assets_handler_spec", dir: "spec/http_server/handlers/static_assets_handler") + yield tempfile +ensure + tempfile.try &.delete +end + +# Get relative file path to a file within the static_assets_handler folder +macro get_file_path(basename) + "spec/http_server/handlers/static_assets_handler/#{ {{basename}} }" +end + +Spectator.describe StaticAssetsHandler do + it "Can serve a file" do + response = handle HTTP::Request.new("GET", "/test.txt") + expect(response.status_code).to eq(200) + expect(response.body).to eq(File.read(get_file_path("test.txt"))) + end + + it "Can serve cached file" do + make_temporary_file("cache_test") do |temporary_file| + temporary_file.rewind << "foo" + temporary_file.flush + expect(temporary_file.rewind.gets_to_end).to eq("foo") + + file_link = "/#{File.basename(temporary_file.path)}" + + # Should get cached by the first run + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("foo") + + # Update temporary file to "bar" + temporary_file.rewind << "bar" + temporary_file.flush + expect(temporary_file.rewind.gets_to_end).to eq("bar") + + # Second request should still return "foo" + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("foo") + end + end + + it "Adds cache headers" do + response = handle HTTP::Request.new("GET", "/test.txt") + expect(response.headers["cache_control"]).to eq("max-age=2629800") + end + + context "Can handle range requests" do + it "Can serve range request" do + headers = HTTP::Headers{"Range" => "bytes=0-2"} + response = handle HTTP::Request.new("GET", "/test.txt", headers) + + expect(response.status_code).to eq(206) + expect(response.headers["Content-Range"]?).to eq "bytes 0-2/11" + expect(response.body).to eq "Hel" + end + + it "Will cache entire file even if doing partial requests" do + make_temporary_file("range_cache") do |temporary_file| + temporary_file << "Hello world" + temporary_file.flush.rewind + file_link = "/#{File.basename(temporary_file.path)}" + + # Make request + headers = HTTP::Headers{"Range" => "bytes=0-2"} + response = handle HTTP::Request.new("GET", file_link, headers) + + # Mutate file on disk + temporary_file << "Something else" + temporary_file.flush.rewind + + # Second request shouldn't have changed + headers = HTTP::Headers{"Range" => "bytes=3-8"} + response = handle HTTP::Request.new("GET", file_link, headers) + expect(response.status_code).to eq(206) + expect(response.body).to eq "lo wor" + end + end + end + + context "Is able to support compression" do + def decompressed(string : String) + decompressed = Compress::Gzip::Reader.open(IO::Memory.new(string)) do |gzip| + gzip.gets_to_end + end + + return expect(decompressed) + end + + it "For full file requests" do + handler = HTTP::CompressHandler.new + handler.next = get_static_assets_handler() + + make_temporary_file("check decompression handler") do |temporary_file| + temporary_file << "Hello world" + temporary_file.flush.rewind + file_link = "/#{File.basename(temporary_file.path)}" + + # Can send from disk? + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + decompressed(response.body).to eq("Hello world") + + temporary_file << "Hello world" + temporary_file.flush.rewind + file_link = "/#{File.basename(temporary_file.path)}" + + # Are cached requests working? + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + decompressed(response.body).to eq("Hello world") + + # Able to retrieve non gzipped file? + response = handle HTTP::Request.new("GET", file_link), handler: handler + expect(response.body).to eq("Hello world") + expect(response.headers).to_not have_key("Content-Encoding") + end + end + + # Inspired by the equivalent tests from upstream + it "For partial file requests" do + handler = HTTP::CompressHandler.new + handler.next = get_static_assets_handler() + + make_temporary_file("check_decompression_handler_on_partial_requests") do |temporary_file| + temporary_file << "Hello world this is a very long string" + temporary_file.flush.rewind + file_link = "/#{File.basename(temporary_file.path)}" + + range_response_results = { + "10-20/38" => "d this is a", + "0-0/38" => "H", + "5-9/38" => " worl", + } + + range_request_header_value = {"10-20", "5-9", "0-0"}.join(',') + range_response_header_value = range_response_results.keys + + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Range" => "bytes=#{range_request_header_value}", "Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + + # Decompress response + response = HTTP::Client::Response.new( + status: response.status, + headers: response.headers, + body_io: Compress::Gzip::Reader.new(IO::Memory.new(response.body)), + ) + + count = 0 + MIME::Multipart.parse(response) do |headers, part| + part_range = headers["Content-Range"][6..] + expect(part_range).to be_within(range_response_header_value) + expect(part.gets_to_end).to eq(range_response_results[part_range]) + count += 1 + end + + expect(count).to eq(3) + + # Is the file cached? + temporary_file << "Something else" + temporary_file.flush.rewind + + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + decompressed(response.body).to eq("Hello world this is a very long string") + end + end + end + + after_each { Invidious::HttpServer::StaticAssetsHandler.clear_cache } +end diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr index 243d6a8d..8f2c1b7e 100644 --- a/src/invidious/http_server/static_assets_handler.cr +++ b/src/invidious/http_server/static_assets_handler.cr @@ -107,5 +107,12 @@ module Invidious::HttpServer # Paste in the body of inherited serve_file_range {{@type.superclass.methods.select(&.name.==("serve_file_range"))[0].body}} end + + # Clear cached files. + # + # This is only used in the specs to clear the cache before each handler test + def self.clear_cache + return @@cached_files.clear + end end end From 7749ea1956401b622d65743271fe2622106bfb72 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 16:39:59 -0700 Subject: [PATCH 10/17] Isolate static assets handler spec from others Running `crystal spec` without a file argument essentially produces one big program that combines every single spec file, their imports, and the files that those imports themselves depend on. Most of the types within this combined program will get ignored by the compiler due to a lack of any calls to them from the spec files. But for some types, partially the HTTP module ones, using them within the spec files will suddenly make the compiler enable a bunch of previously ignored code. And those code will suddenly require the presence of additional types, constants, etc. This not only make it annoying for getting the specs working but also makes it difficult to isolate behaviors for testing. The `static_assets_handler_spec.cr` causes this issue and so will be marked as an isolated spec for now. In the future all of the tests should be organized into independent groupings similar to how the Crystal compiler splits their tests into std, compiler, primitives and interpreter. --- .../handlers/static_assets_handler_spec.cr | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spec/http_server/handlers/static_assets_handler_spec.cr b/spec/http_server/handlers/static_assets_handler_spec.cr index 89c53014..9b7a363e 100644 --- a/spec/http_server/handlers/static_assets_handler_spec.cr +++ b/spec/http_server/handlers/static_assets_handler_spec.cr @@ -1,4 +1,13 @@ -{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %} +# Due to the way that specs are handled this file cannot be run together with +# everything else without causing a compile time error that'll be incredibly +# annoying to resolve. +# +# TODO: Create different spec categories that can then be ran through make. +# An implementation of this can be seen with the tests for the Crystal compiler itself. +# +# For now run this with `crystal spec spec/http_server/handlers/static_assets_handler_spec.cr -Drunning_by_self` + +{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 || !flag?(:running_by_self) %} require "http" require "spectator" From 89a0761a19f48551ed37d9df0f512aceff76f5dc Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 16:40:35 -0700 Subject: [PATCH 11/17] Fix Ameba Lint/UselessAssign --- spec/http_server/handlers/static_assets_handler_spec.cr | 3 +-- src/invidious/http_server/static_assets_handler.cr | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/spec/http_server/handlers/static_assets_handler_spec.cr b/spec/http_server/handlers/static_assets_handler_spec.cr index 9b7a363e..373d59fd 100644 --- a/spec/http_server/handlers/static_assets_handler_spec.cr +++ b/spec/http_server/handlers/static_assets_handler_spec.cr @@ -106,8 +106,7 @@ Spectator.describe StaticAssetsHandler do file_link = "/#{File.basename(temporary_file.path)}" # Make request - headers = HTTP::Headers{"Range" => "bytes=0-2"} - response = handle HTTP::Request.new("GET", file_link, headers) + handle HTTP::Request.new("GET", file_link, HTTP::Headers{"Range" => "bytes=0-2"}) # Mutate file on disk temporary_file << "Something else" diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr index 8f2c1b7e..94add5a8 100644 --- a/src/invidious/http_server/static_assets_handler.cr +++ b/src/invidious/http_server/static_assets_handler.cr @@ -71,7 +71,7 @@ module Invidious::HttpServer # Writes file data to the cache private def flush_io_to_cache(io, file_path, file_info) - if @@cached_files.sum(&.[1].size) + (size = file_info.size) < CACHE_LIMIT + if @@cached_files.sum(&.[1].size) + file_info.size < CACHE_LIMIT data_slice = io.to_slice @@cached_files[file_path] = CachedFile.new(data_slice, file_info.size, file_info.modification_time) end From 7f9cfe1aa201e0c40255ecb0e9296cd6b40d4696 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 17:07:51 -0700 Subject: [PATCH 12/17] Refactor logic for updating temp files in tests --- .../handlers/static_assets_handler_spec.cr | 125 ++++++++---------- 1 file changed, 57 insertions(+), 68 deletions(-) diff --git a/spec/http_server/handlers/static_assets_handler_spec.cr b/spec/http_server/handlers/static_assets_handler_spec.cr index 373d59fd..4b50171a 100644 --- a/spec/http_server/handlers/static_assets_handler_spec.cr +++ b/spec/http_server/handlers/static_assets_handler_spec.cr @@ -42,11 +42,21 @@ end # Makes and yields a temporary file with the given prefix private def make_temporary_file(prefix, contents = nil, &) tempfile = File.tempfile(prefix, "static_assets_handler_spec", dir: "spec/http_server/handlers/static_assets_handler") - yield tempfile + file_link = "/#{File.basename(tempfile.path)}" + yield tempfile, file_link ensure tempfile.try &.delete end +# Changes the contents of the temporary file after yield +private def cycle_temporary_file_contents(temporary_file, initial, &) + temporary_file.rewind << initial + temporary_file.rewind.flush + yield + temporary_file.rewind << "something else" + temporary_file.rewind.flush +end + # Get relative file path to a file within the static_assets_handler folder macro get_file_path(basename) "spec/http_server/handlers/static_assets_handler/#{ {{basename}} }" @@ -60,24 +70,19 @@ Spectator.describe StaticAssetsHandler do end it "Can serve cached file" do - make_temporary_file("cache_test") do |temporary_file| - temporary_file.rewind << "foo" - temporary_file.flush - expect(temporary_file.rewind.gets_to_end).to eq("foo") + make_temporary_file("cache_test") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "foo") do + expect(temporary_file.rewind.gets_to_end).to eq("foo") - file_link = "/#{File.basename(temporary_file.path)}" + # Should get cached by the first run + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("foo") + end - # Should get cached by the first run - response = handle HTTP::Request.new("GET", file_link) - expect(response.status_code).to eq(200) - expect(response.body).to eq("foo") - - # Update temporary file to "bar" - temporary_file.rewind << "bar" - temporary_file.flush - expect(temporary_file.rewind.gets_to_end).to eq("bar") - - # Second request should still return "foo" + # Temporary file is updated after `cycle_temporary_file_contents` is called + # but if the file is successfully cached then we'll only get the original + # contents. response = handle HTTP::Request.new("GET", file_link) expect(response.status_code).to eq(200) expect(response.body).to eq("foo") @@ -100,17 +105,10 @@ Spectator.describe StaticAssetsHandler do end it "Will cache entire file even if doing partial requests" do - make_temporary_file("range_cache") do |temporary_file| - temporary_file << "Hello world" - temporary_file.flush.rewind - file_link = "/#{File.basename(temporary_file.path)}" - - # Make request - handle HTTP::Request.new("GET", file_link, HTTP::Headers{"Range" => "bytes=0-2"}) - - # Mutate file on disk - temporary_file << "Something else" - temporary_file.flush.rewind + make_temporary_file("range_cache") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "Hello world") do + handle HTTP::Request.new("GET", file_link, HTTP::Headers{"Range" => "bytes=0-2"}) + end # Second request shouldn't have changed headers = HTTP::Headers{"Range" => "bytes=3-8"} @@ -134,19 +132,12 @@ Spectator.describe StaticAssetsHandler do handler = HTTP::CompressHandler.new handler.next = get_static_assets_handler() - make_temporary_file("check decompression handler") do |temporary_file| - temporary_file << "Hello world" - temporary_file.flush.rewind - file_link = "/#{File.basename(temporary_file.path)}" - - # Can send from disk? - response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler - expect(response.headers["Content-Encoding"]).to eq("gzip") - decompressed(response.body).to eq("Hello world") - - temporary_file << "Hello world" - temporary_file.flush.rewind - file_link = "/#{File.basename(temporary_file.path)}" + make_temporary_file("check decompression handler") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "Hello world") do + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + decompressed(response.body).to eq("Hello world") + end # Are cached requests working? response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler @@ -165,40 +156,38 @@ Spectator.describe StaticAssetsHandler do handler = HTTP::CompressHandler.new handler.next = get_static_assets_handler() - make_temporary_file("check_decompression_handler_on_partial_requests") do |temporary_file| - temporary_file << "Hello world this is a very long string" - temporary_file.flush.rewind - file_link = "/#{File.basename(temporary_file.path)}" + make_temporary_file("check_decompression_handler_on_partial_requests") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "Hello world this is a very long string") do + range_response_results = { + "10-20/38" => "d this is a", + "0-0/38" => "H", + "5-9/38" => " worl", + } - range_response_results = { - "10-20/38" => "d this is a", - "0-0/38" => "H", - "5-9/38" => " worl", - } + range_request_header_value = {"10-20", "5-9", "0-0"}.join(',') + range_response_header_value = range_response_results.keys - range_request_header_value = {"10-20", "5-9", "0-0"}.join(',') - range_response_header_value = range_response_results.keys + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Range" => "bytes=#{range_request_header_value}", "Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") - response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Range" => "bytes=#{range_request_header_value}", "Accept-Encoding" => "gzip"}), handler: handler - expect(response.headers["Content-Encoding"]).to eq("gzip") + # Decompress response + response = HTTP::Client::Response.new( + status: response.status, + headers: response.headers, + body_io: Compress::Gzip::Reader.new(IO::Memory.new(response.body)), + ) - # Decompress response - response = HTTP::Client::Response.new( - status: response.status, - headers: response.headers, - body_io: Compress::Gzip::Reader.new(IO::Memory.new(response.body)), - ) + count = 0 + MIME::Multipart.parse(response) do |headers, part| + part_range = headers["Content-Range"][6..] + expect(part_range).to be_within(range_response_header_value) + expect(part.gets_to_end).to eq(range_response_results[part_range]) + count += 1 + end - count = 0 - MIME::Multipart.parse(response) do |headers, part| - part_range = headers["Content-Range"][6..] - expect(part_range).to be_within(range_response_header_value) - expect(part.gets_to_end).to eq(range_response_results[part_range]) - count += 1 + expect(count).to eq(3) end - expect(count).to eq(3) - # Is the file cached? temporary_file << "Something else" temporary_file.flush.rewind From 21049518d603da7ea1ba13feb98058e1355fdad4 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 3 Jun 2025 17:10:10 -0700 Subject: [PATCH 13/17] Improve cache size check to be more performant Summing the sizes of each cached file every time is very inefficient. Instead we can simply store the cache size in an constant and increase it everytime a file is added into the cache. --- .../handlers/static_assets_handler_spec.cr | 31 +++++++++++++++++++ .../http_server/static_assets_handler.cr | 7 +++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/spec/http_server/handlers/static_assets_handler_spec.cr b/spec/http_server/handlers/static_assets_handler_spec.cr index 4b50171a..76dc7be7 100644 --- a/spec/http_server/handlers/static_assets_handler_spec.cr +++ b/spec/http_server/handlers/static_assets_handler_spec.cr @@ -198,5 +198,36 @@ Spectator.describe StaticAssetsHandler do end end + it "Will not cache additional files if the cache limit is reached" do + 5.times do |times| + data = "a" * 1_000_000 + + make_temporary_file("test cache size limit #{times}") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, data) do + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq(data) + end + + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq(data) + end + end + + # Cache should be 5 mb so no more files will be cached. + make_temporary_file("test cache size limit uncached") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "a") do + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("a") + end + + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to_not eq("a") + end + end + after_each { Invidious::HttpServer::StaticAssetsHandler.clear_cache } end diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr index 94add5a8..e086ac3b 100644 --- a/src/invidious/http_server/static_assets_handler.cr +++ b/src/invidious/http_server/static_assets_handler.cr @@ -20,6 +20,7 @@ module Invidious::HttpServer end CACHE_LIMIT = 5_000_000 # 5MB + @@current_cache_size = 0 @@cached_files = {} of Path => CachedFile # Returns metadata for the requested file @@ -71,9 +72,8 @@ module Invidious::HttpServer # Writes file data to the cache private def flush_io_to_cache(io, file_path, file_info) - if @@cached_files.sum(&.[1].size) + file_info.size < CACHE_LIMIT - data_slice = io.to_slice - @@cached_files[file_path] = CachedFile.new(data_slice, file_info.size, file_info.modification_time) + if (@@current_cache_size += file_info.size) <= CACHE_LIMIT + @@cached_files[file_path] = CachedFile.new(io.to_slice, file_info.size, file_info.modification_time) end end @@ -112,6 +112,7 @@ module Invidious::HttpServer # # This is only used in the specs to clear the cache before each handler test def self.clear_cache + @@current_cache_size = 0 return @@cached_files.clear end end From 1f5685ef92ef020f60e69e4f2a966dca15368e7b Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 23 Aug 2025 20:51:30 -0700 Subject: [PATCH 14/17] Reduce indent in StaticAssetsHandler#serve_file --- .../http_server/static_assets_handler.cr | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr index e086ac3b..7902c95b 100644 --- a/src/invidious/http_server/static_assets_handler.cr +++ b/src/invidious/http_server/static_assets_handler.cr @@ -50,24 +50,25 @@ module Invidious::HttpServer range_header = context.request.headers["Range"]? - if !file_info.is_a? CachedFile - retrieve_bytes_from = IO::Memory.new - - File.open(file_path) do |file| - # We cannot cache partial data so we'll rewind and read from the start - if range_header - dispatch_serve(context, file, file_info, range_header) - IO.copy(file.rewind, retrieve_bytes_from) - else - context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true) - dispatch_serve(context, file, file_info, range_header) - end - end - - return flush_io_to_cache(retrieve_bytes_from, file_path, file_info) - else + # If the file is cached we can just directly serve it + if file_info.is_a? CachedFile return dispatch_serve(context, file_info.data, file_info, range_header) end + + # Otherwise we'll need to read from disk and cache it + retrieve_bytes_from = IO::Memory.new + File.open(file_path) do |file| + # We cannot cache partial data so we'll rewind and read from the start + if range_header + dispatch_serve(context, file, file_info, range_header) + IO.copy(file.rewind, retrieve_bytes_from) + else + context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true) + dispatch_serve(context, file, file_info, range_header) + end + end + + return flush_io_to_cache(retrieve_bytes_from, file_path, file_info) end # Writes file data to the cache From bf17d5306872f0f997900330117f7fd85d371d22 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Fri, 19 Dec 2025 10:59:42 -0300 Subject: [PATCH 15/17] Replace deprecated `blocking` property of `Socket` (#5538) * Replace deprecated `blocking` property of `Socket` This replaces the deprecated argument `blocking` and uses `Socket.set_blocking(fd, value)` instead. Fixes a warning in the compiler https://github.com/crystal-lang/crystal/pull/16033 * Upgrade to upstream * chore: only Socket.set_blocking for > 1.18 --------- Co-authored-by: Emilien <4016501+unixfox@users.noreply.github.com> --- .../helpers/crystal_class_overrides.cr | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index fec3f62c..6fa89395 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -3,15 +3,28 @@ # IPv6 addresses. # class TCPSocket - def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) - Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| - super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) - connect(addrinfo, timeout: connect_timeout) do |error| - close - error + {% if compare_versions(Crystal::VERSION, "1.18.0-dev") >= 0 %} + def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) + Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| + super(family: addrinfo.family, type: addrinfo.type, protocol: addrinfo.protocol) + Socket.set_blocking(self.fd, blocking) + connect(addrinfo, timeout: connect_timeout) do |error| + close + error + end end end - end + {% else %} + def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) + Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| + super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) + connect(addrinfo, timeout: connect_timeout) do |error| + close + error + end + end + end + {% end %} end # :ditto: From 7a4b9018463ba48c1e59bca7d11c498f14cf0f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Fri, 19 Dec 2025 15:08:07 +0100 Subject: [PATCH 16/17] chore: update crystal 1.18.2 + alpine 3.23 (#5574) --- .github/workflows/ci.yml | 4 ++-- docker/Dockerfile | 4 ++-- docker/Dockerfile.arm64 | 11 +++++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b28873d1..847342f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,11 +38,11 @@ jobs: matrix: stable: [true] crystal: - - 1.12.2 - - 1.13.3 - 1.14.1 - 1.15.1 - 1.16.3 + - 1.17.1 + - 1.18.2 include: - crystal: nightly stable: false diff --git a/docker/Dockerfile b/docker/Dockerfile index 3e0d2f7f..383a60ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ ARG OPENSSL_VERSION='3.5.2' ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec' -FROM crystallang/crystal:1.16.3-alpine AS dependabot-crystal +FROM crystallang/crystal:1.18.2-alpine AS dependabot-crystal # We compile openssl ourselves due to a memory leak in how crystal interacts # with openssl @@ -61,7 +61,7 @@ RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.21 +FROM alpine:3.23 RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index b02cc8ce..8508d4fa 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -2,7 +2,7 @@ ARG OPENSSL_VERSION='3.5.2' ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec' -FROM alpine:3.21 AS dependabot-alpine +FROM alpine:3.23 AS dependabot-alpine # We compile openssl ourselves due to a memory leak in how crystal interacts # with openssl @@ -21,8 +21,11 @@ RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc) FROM dependabot-alpine AS builder -RUN apk add --no-cache 'crystal=1.14.0-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \ - zlib-static musl-dev xz-static +RUN apk add --no-cache 'crystal=1.18.2-r0' shards \ + sqlite-static yaml-static yaml-dev \ + pcre2-static gc-static \ + libxml2-static zlib-static \ + openssl-libs-static openssl-dev musl-dev xz-static ARG release @@ -60,7 +63,7 @@ RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.21 +FROM alpine:3.23 RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ From dbbaf51f1f4e80c7db14e669aadac7fb87f6267d Mon Sep 17 00:00:00 2001 From: Jeroen Boersma Date: Fri, 19 Dec 2025 15:09:22 +0100 Subject: [PATCH 17/17] Allow downloading via companion (#5561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow downloading via companion * post request where not proxied for the download companion which made it impossible to download with the companion enabled * Re-apply Channel to Channels rename which was pulled in * Update src/invidious/routes/companion.cr * doc: better comments for each route --------- Co-authored-by: Fijxu Co-authored-by: Émilien (perso) <4016501+unixfox@users.noreply.github.com> --- src/invidious/routes/companion.cr | 20 +++++++++++++++++++- src/invidious/routing.cr | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/invidious/routes/companion.cr b/src/invidious/routes/companion.cr index 11c2e3f5..949b213f 100644 --- a/src/invidious/routes/companion.cr +++ b/src/invidious/routes/companion.cr @@ -1,5 +1,5 @@ module Invidious::Routes::Companion - # /companion + # GET /companion def self.get_companion(env) url = env.request.path if env.request.query @@ -16,6 +16,24 @@ module Invidious::Routes::Companion end end + # POST /companion + def self.post_companion(env) + url = env.request.path + if env.request.query + url += "?#{env.request.query}" + end + + begin + COMPANION_POOL.client do |wrapper| + wrapper.client.post(url, env.request.headers, env.request.body) do |resp| + return self.proxy_companion(env, resp) + end + end + rescue ex + end + end + + def self.options_companion(env) url = env.request.path if env.request.query diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index a51bb4b6..32e8554c 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -227,6 +227,7 @@ module Invidious::Routing def register_companion_routes if CONFIG.invidious_companion.present? get "/companion/*", Routes::Companion, :get_companion + post "/companion/*", Routes::Companion, :post_companion options "/companion/*", Routes::Companion, :options_companion end end