mirror of
https://git.nadeko.net/Fijxu/invidious.git
synced 2025-12-25 20:38:51 +00:00
Merge remote-tracking branch 'upstream'
This commit is contained in:
8
.github/workflows/ci.yml
vendored
8
.github/workflows/ci.yml
vendored
@@ -38,11 +38,11 @@ jobs:
|
|||||||
matrix:
|
matrix:
|
||||||
stable: [true]
|
stable: [true]
|
||||||
crystal:
|
crystal:
|
||||||
- 1.12.2
|
|
||||||
- 1.13.3
|
|
||||||
- 1.14.1
|
- 1.14.1
|
||||||
- 1.15.1
|
- 1.15.1
|
||||||
- 1.16.3
|
- 1.16.3
|
||||||
|
- 1.17.1
|
||||||
|
- 1.18.2
|
||||||
include:
|
include:
|
||||||
- crystal: nightly
|
- crystal: nightly
|
||||||
stable: false
|
stable: false
|
||||||
@@ -63,7 +63,7 @@ jobs:
|
|||||||
crystal: ${{ matrix.crystal }}
|
crystal: ${{ matrix.crystal }}
|
||||||
|
|
||||||
- name: Cache Shards
|
- name: Cache Shards
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v5
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./lib
|
./lib
|
||||||
@@ -139,7 +139,7 @@ jobs:
|
|||||||
crystal: latest
|
crystal: latest
|
||||||
|
|
||||||
- name: Cache Shards
|
- name: Cache Shards
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v5
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./lib
|
./lib
|
||||||
|
|||||||
@@ -1,20 +1,34 @@
|
|||||||
# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2
|
# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2
|
||||||
ARG OPENSSL_VERSION='3.5.2'
|
ARG OPENSSL_VERSION='3.5.2'
|
||||||
|
ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec'
|
||||||
|
|
||||||
FROM mirror.gcr.io/84codes/crystal:1.16.3-alpine AS builder
|
FROM mirror.gcr.io/84codes/crystal:1.18.2-alpine AS dependabot-crystal
|
||||||
|
|
||||||
|
# We compile openssl ourselves due to a memory leak in how crystal interacts
|
||||||
|
# with openssl
|
||||||
|
# Reference: https://github.com/iv-org/invidious/issues/1438#issuecomment-3087636228
|
||||||
|
FROM dependabot-crystal AS openssl-builder
|
||||||
|
RUN apk add --no-cache curl perl linux-headers
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
|
||||||
|
ARG OPENSSL_VERSION
|
||||||
|
ARG OPENSSL_SHA256
|
||||||
|
RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" --output openssl-${OPENSSL_VERSION}.tar.gz
|
||||||
|
RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c
|
||||||
|
RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz
|
||||||
|
|
||||||
|
RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc)
|
||||||
|
|
||||||
|
FROM dependabot-crystal AS builder
|
||||||
|
|
||||||
RUN apk add --no-cache sqlite-static yaml-static
|
RUN apk add --no-cache sqlite-static yaml-static
|
||||||
RUN apk del openssl-dev openssl-libs-static
|
RUN apk del openssl-dev openssl-libs-static
|
||||||
RUN apk add curl perl linux-headers
|
|
||||||
|
|
||||||
ARG release
|
ARG release
|
||||||
|
|
||||||
WORKDIR /invidious
|
WORKDIR /invidious
|
||||||
|
|
||||||
ARG OPENSSL_VERSION
|
|
||||||
RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" | tar xz
|
|
||||||
RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j
|
|
||||||
|
|
||||||
COPY ./shard.yml ./shard.yml
|
COPY ./shard.yml ./shard.yml
|
||||||
COPY ./shard.lock ./shard.lock
|
COPY ./shard.lock ./shard.lock
|
||||||
|
|
||||||
@@ -30,14 +44,26 @@ COPY ./scripts/ ./scripts/
|
|||||||
COPY ./assets/ ./assets/
|
COPY ./assets/ ./assets/
|
||||||
COPY ./videojs-dependencies.yml ./videojs-dependencies.yml
|
COPY ./videojs-dependencies.yml ./videojs-dependencies.yml
|
||||||
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/crystal \
|
RUN crystal spec --warnings all \
|
||||||
PKG_CONFIG_PATH=$PWD/openssl-${OPENSSL_VERSION} \
|
--link-flags "-lxml2 -llzma"
|
||||||
|
|
||||||
|
ARG OPENSSL_VERSION
|
||||||
|
COPY --from=openssl-builder /openssl-${OPENSSL_VERSION} /openssl-${OPENSSL_VERSION}
|
||||||
|
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \
|
||||||
|
PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \
|
||||||
crystal build ./src/invidious.cr \
|
crystal build ./src/invidious.cr \
|
||||||
--release -s -p -t --mcpu=x86-64-v2 \
|
--release -s -p -t --mcpu=x86-64-v2 \
|
||||||
--static --warnings all \
|
--static --warnings all \
|
||||||
--link-flags "-lxml2 -llzma";
|
--link-flags "-lxml2 -llzma"; \
|
||||||
|
else \
|
||||||
|
PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \
|
||||||
|
crystal build ./src/invidious.cr \
|
||||||
|
--static --warnings all \
|
||||||
|
--link-flags "-lxml2 -llzma"; \
|
||||||
|
fi
|
||||||
|
|
||||||
FROM mirror.gcr.io/alpine:3.22
|
FROM mirror.gcr.io/alpine:3.23
|
||||||
RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
|
RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
|
||||||
WORKDIR /invidious
|
WORKDIR /invidious
|
||||||
RUN addgroup -g 1000 -S invidious && \
|
RUN addgroup -g 1000 -S invidious && \
|
||||||
|
|||||||
@@ -1,6 +1,31 @@
|
|||||||
FROM alpine:3.23 AS builder
|
# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2
|
||||||
RUN apk add --no-cache 'crystal=1.14.0-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \
|
ARG OPENSSL_VERSION='3.5.2'
|
||||||
zlib-static openssl-libs-static openssl-dev musl-dev xz-static
|
ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec'
|
||||||
|
|
||||||
|
FROM alpine:3.23 AS dependabot-alpine
|
||||||
|
|
||||||
|
# We compile openssl ourselves due to a memory leak in how crystal interacts
|
||||||
|
# with openssl
|
||||||
|
# Reference: https://github.com/iv-org/invidious/issues/1438#issuecomment-3087636228
|
||||||
|
FROM dependabot-alpine AS openssl-builder
|
||||||
|
RUN apk add --no-cache curl perl linux-headers build-base
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
|
||||||
|
ARG OPENSSL_VERSION
|
||||||
|
ARG OPENSSL_SHA256
|
||||||
|
RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" --output openssl-${OPENSSL_VERSION}.tar.gz
|
||||||
|
RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c
|
||||||
|
RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz
|
||||||
|
|
||||||
|
RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc)
|
||||||
|
|
||||||
|
FROM dependabot-alpine AS builder
|
||||||
|
RUN apk add --no-cache 'crystal=1.18.2-r0' shards \
|
||||||
|
sqlite-static yaml-static yaml-dev \
|
||||||
|
pcre2-static gc-static \
|
||||||
|
libxml2-static zlib-static \
|
||||||
|
openssl-libs-static openssl-dev musl-dev xz-static
|
||||||
|
|
||||||
ARG release
|
ARG release
|
||||||
|
|
||||||
@@ -22,12 +47,17 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml
|
|||||||
RUN crystal spec --warnings all \
|
RUN crystal spec --warnings all \
|
||||||
--link-flags "-lxml2 -llzma"
|
--link-flags "-lxml2 -llzma"
|
||||||
|
|
||||||
|
ARG OPENSSL_VERSION
|
||||||
|
COPY --from=openssl-builder /openssl-${OPENSSL_VERSION} /openssl-${OPENSSL_VERSION}
|
||||||
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \
|
RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \
|
||||||
|
PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \
|
||||||
crystal build ./src/invidious.cr \
|
crystal build ./src/invidious.cr \
|
||||||
--release \
|
--release \
|
||||||
--static --warnings all \
|
--static --warnings all \
|
||||||
--link-flags "-lxml2 -llzma"; \
|
--link-flags "-lxml2 -llzma"; \
|
||||||
else \
|
else \
|
||||||
|
PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \
|
||||||
crystal build ./src/invidious.cr \
|
crystal build ./src/invidious.cr \
|
||||||
--static --warnings all \
|
--static --warnings all \
|
||||||
--link-flags "-lxml2 -llzma"; \
|
--link-flags "-lxml2 -llzma"; \
|
||||||
|
|||||||
1
spec/http_server/handlers/static_assets_handler/test.txt
Normal file
1
spec/http_server/handlers/static_assets_handler/test.txt
Normal file
@@ -0,0 +1 @@
|
|||||||
|
Hello world
|
||||||
233
spec/http_server/handlers/static_assets_handler_spec.cr
Normal file
233
spec/http_server/handlers/static_assets_handler_spec.cr
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
# Due to the way that specs are handled this file cannot be run together with
|
||||||
|
# everything else without causing a compile time error that'll be incredibly
|
||||||
|
# annoying to resolve.
|
||||||
|
#
|
||||||
|
# TODO: Create different spec categories that can then be ran through make.
|
||||||
|
# An implementation of this can be seen with the tests for the Crystal compiler itself.
|
||||||
|
#
|
||||||
|
# For now run this with `crystal spec spec/http_server/handlers/static_assets_handler_spec.cr -Drunning_by_self`
|
||||||
|
|
||||||
|
{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 || !flag?(:running_by_self) %}
|
||||||
|
|
||||||
|
require "http"
|
||||||
|
require "spectator"
|
||||||
|
require "../../../src/invidious/http_server/static_assets_handler.cr"
|
||||||
|
|
||||||
|
private def get_static_assets_handler
|
||||||
|
return Invidious::HttpServer::StaticAssetsHandler.new "spec/http_server/handlers/static_assets_handler", directory_listing: false
|
||||||
|
end
|
||||||
|
|
||||||
|
# Slightly modified version of `handle` function from
|
||||||
|
#
|
||||||
|
# https://github.com/crystal-lang/crystal/blob/3f369d2c721e9462d9f6126cb0bcd4c6992f0225/spec/std/http/server/handlers/static_file_handler_spec.cr#L5
|
||||||
|
|
||||||
|
private def handle(request, handler : HTTP::Handler? = nil, decompress : Bool = false)
|
||||||
|
io = IO::Memory.new
|
||||||
|
response = HTTP::Server::Response.new(io)
|
||||||
|
context = HTTP::Server::Context.new(request, response)
|
||||||
|
|
||||||
|
if !handler
|
||||||
|
handler = get_static_assets_handler
|
||||||
|
get_static_assets_handler.call context
|
||||||
|
else
|
||||||
|
handler.call(context)
|
||||||
|
end
|
||||||
|
|
||||||
|
response.close
|
||||||
|
io.rewind
|
||||||
|
|
||||||
|
HTTP::Client::Response.from_io(io, decompress: decompress)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Makes and yields a temporary file with the given prefix
|
||||||
|
private def make_temporary_file(prefix, contents = nil, &)
|
||||||
|
tempfile = File.tempfile(prefix, "static_assets_handler_spec", dir: "spec/http_server/handlers/static_assets_handler")
|
||||||
|
file_link = "/#{File.basename(tempfile.path)}"
|
||||||
|
yield tempfile, file_link
|
||||||
|
ensure
|
||||||
|
tempfile.try &.delete
|
||||||
|
end
|
||||||
|
|
||||||
|
# Changes the contents of the temporary file after yield
|
||||||
|
private def cycle_temporary_file_contents(temporary_file, initial, &)
|
||||||
|
temporary_file.rewind << initial
|
||||||
|
temporary_file.rewind.flush
|
||||||
|
yield
|
||||||
|
temporary_file.rewind << "something else"
|
||||||
|
temporary_file.rewind.flush
|
||||||
|
end
|
||||||
|
|
||||||
|
# Get relative file path to a file within the static_assets_handler folder
|
||||||
|
macro get_file_path(basename)
|
||||||
|
"spec/http_server/handlers/static_assets_handler/#{ {{basename}} }"
|
||||||
|
end
|
||||||
|
|
||||||
|
Spectator.describe StaticAssetsHandler do
|
||||||
|
it "Can serve a file" do
|
||||||
|
response = handle HTTP::Request.new("GET", "/test.txt")
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to eq(File.read(get_file_path("test.txt")))
|
||||||
|
end
|
||||||
|
|
||||||
|
it "Can serve cached file" do
|
||||||
|
make_temporary_file("cache_test") do |temporary_file, file_link|
|
||||||
|
cycle_temporary_file_contents(temporary_file, "foo") do
|
||||||
|
expect(temporary_file.rewind.gets_to_end).to eq("foo")
|
||||||
|
|
||||||
|
# Should get cached by the first run
|
||||||
|
response = handle HTTP::Request.new("GET", file_link)
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to eq("foo")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Temporary file is updated after `cycle_temporary_file_contents` is called
|
||||||
|
# but if the file is successfully cached then we'll only get the original
|
||||||
|
# contents.
|
||||||
|
response = handle HTTP::Request.new("GET", file_link)
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to eq("foo")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it "Adds cache headers" do
|
||||||
|
response = handle HTTP::Request.new("GET", "/test.txt")
|
||||||
|
expect(response.headers["cache_control"]).to eq("max-age=2629800")
|
||||||
|
end
|
||||||
|
|
||||||
|
context "Can handle range requests" do
|
||||||
|
it "Can serve range request" do
|
||||||
|
headers = HTTP::Headers{"Range" => "bytes=0-2"}
|
||||||
|
response = handle HTTP::Request.new("GET", "/test.txt", headers)
|
||||||
|
|
||||||
|
expect(response.status_code).to eq(206)
|
||||||
|
expect(response.headers["Content-Range"]?).to eq "bytes 0-2/11"
|
||||||
|
expect(response.body).to eq "Hel"
|
||||||
|
end
|
||||||
|
|
||||||
|
it "Will cache entire file even if doing partial requests" do
|
||||||
|
make_temporary_file("range_cache") do |temporary_file, file_link|
|
||||||
|
cycle_temporary_file_contents(temporary_file, "Hello world") do
|
||||||
|
handle HTTP::Request.new("GET", file_link, HTTP::Headers{"Range" => "bytes=0-2"})
|
||||||
|
end
|
||||||
|
|
||||||
|
# Second request shouldn't have changed
|
||||||
|
headers = HTTP::Headers{"Range" => "bytes=3-8"}
|
||||||
|
response = handle HTTP::Request.new("GET", file_link, headers)
|
||||||
|
expect(response.status_code).to eq(206)
|
||||||
|
expect(response.body).to eq "lo wor"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context "Is able to support compression" do
|
||||||
|
def decompressed(string : String)
|
||||||
|
decompressed = Compress::Gzip::Reader.open(IO::Memory.new(string)) do |gzip|
|
||||||
|
gzip.gets_to_end
|
||||||
|
end
|
||||||
|
|
||||||
|
return expect(decompressed)
|
||||||
|
end
|
||||||
|
|
||||||
|
it "For full file requests" do
|
||||||
|
handler = HTTP::CompressHandler.new
|
||||||
|
handler.next = get_static_assets_handler()
|
||||||
|
|
||||||
|
make_temporary_file("check decompression handler") do |temporary_file, file_link|
|
||||||
|
cycle_temporary_file_contents(temporary_file, "Hello world") do
|
||||||
|
response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler
|
||||||
|
expect(response.headers["Content-Encoding"]).to eq("gzip")
|
||||||
|
decompressed(response.body).to eq("Hello world")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Are cached requests working?
|
||||||
|
response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler
|
||||||
|
expect(response.headers["Content-Encoding"]).to eq("gzip")
|
||||||
|
decompressed(response.body).to eq("Hello world")
|
||||||
|
|
||||||
|
# Able to retrieve non gzipped file?
|
||||||
|
response = handle HTTP::Request.new("GET", file_link), handler: handler
|
||||||
|
expect(response.body).to eq("Hello world")
|
||||||
|
expect(response.headers).to_not have_key("Content-Encoding")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Inspired by the equivalent tests from upstream
|
||||||
|
it "For partial file requests" do
|
||||||
|
handler = HTTP::CompressHandler.new
|
||||||
|
handler.next = get_static_assets_handler()
|
||||||
|
|
||||||
|
make_temporary_file("check_decompression_handler_on_partial_requests") do |temporary_file, file_link|
|
||||||
|
cycle_temporary_file_contents(temporary_file, "Hello world this is a very long string") do
|
||||||
|
range_response_results = {
|
||||||
|
"10-20/38" => "d this is a",
|
||||||
|
"0-0/38" => "H",
|
||||||
|
"5-9/38" => " worl",
|
||||||
|
}
|
||||||
|
|
||||||
|
range_request_header_value = {"10-20", "5-9", "0-0"}.join(',')
|
||||||
|
range_response_header_value = range_response_results.keys
|
||||||
|
|
||||||
|
response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Range" => "bytes=#{range_request_header_value}", "Accept-Encoding" => "gzip"}), handler: handler
|
||||||
|
expect(response.headers["Content-Encoding"]).to eq("gzip")
|
||||||
|
|
||||||
|
# Decompress response
|
||||||
|
response = HTTP::Client::Response.new(
|
||||||
|
status: response.status,
|
||||||
|
headers: response.headers,
|
||||||
|
body_io: Compress::Gzip::Reader.new(IO::Memory.new(response.body)),
|
||||||
|
)
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
MIME::Multipart.parse(response) do |headers, part|
|
||||||
|
part_range = headers["Content-Range"][6..]
|
||||||
|
expect(part_range).to be_within(range_response_header_value)
|
||||||
|
expect(part.gets_to_end).to eq(range_response_results[part_range])
|
||||||
|
count += 1
|
||||||
|
end
|
||||||
|
|
||||||
|
expect(count).to eq(3)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Is the file cached?
|
||||||
|
temporary_file << "Something else"
|
||||||
|
temporary_file.flush.rewind
|
||||||
|
|
||||||
|
response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler
|
||||||
|
decompressed(response.body).to eq("Hello world this is a very long string")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it "Will not cache additional files if the cache limit is reached" do
|
||||||
|
5.times do |times|
|
||||||
|
data = "a" * 1_000_000
|
||||||
|
|
||||||
|
make_temporary_file("test cache size limit #{times}") do |temporary_file, file_link|
|
||||||
|
cycle_temporary_file_contents(temporary_file, data) do
|
||||||
|
response = handle HTTP::Request.new("GET", file_link)
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to eq(data)
|
||||||
|
end
|
||||||
|
|
||||||
|
response = handle HTTP::Request.new("GET", file_link)
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to eq(data)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Cache should be 5 mb so no more files will be cached.
|
||||||
|
make_temporary_file("test cache size limit uncached") do |temporary_file, file_link|
|
||||||
|
cycle_temporary_file_contents(temporary_file, "a") do
|
||||||
|
response = handle HTTP::Request.new("GET", file_link)
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to eq("a")
|
||||||
|
end
|
||||||
|
|
||||||
|
response = handle HTTP::Request.new("GET", file_link)
|
||||||
|
expect(response.status_code).to eq(200)
|
||||||
|
expect(response.body).to_not eq("a")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
after_each { Invidious::HttpServer::StaticAssetsHandler.clear_cache }
|
||||||
|
end
|
||||||
@@ -1,3 +1,24 @@
|
|||||||
|
{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
|
||||||
|
# Strip StaticFileHandler from the binary
|
||||||
|
#
|
||||||
|
# This allows us to compile on 1.17.0 as the compiler won't try to
|
||||||
|
# semantically check the outdated upstream code.
|
||||||
|
class Kemal::Config
|
||||||
|
private def setup_static_file_handler
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Nullify `Kemal::StaticFileHandler`
|
||||||
|
#
|
||||||
|
# Needed until the next release of Kemal after 1.7
|
||||||
|
class Kemal::StaticFileHandler < HTTP::StaticFileHandler
|
||||||
|
def call(context : HTTP::Server::Context)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
{% skip_file %}
|
||||||
|
{% end %}
|
||||||
|
|
||||||
# Since systems have a limit on number of open files (`ulimit -a`),
|
# Since systems have a limit on number of open files (`ulimit -a`),
|
||||||
# we serve them from memory to avoid 'Too many open files' without needing
|
# we serve them from memory to avoid 'Too many open files' without needing
|
||||||
# to modify ulimit.
|
# to modify ulimit.
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ HTTP_CHUNK_SIZE = 10485760 # ~10MB
|
|||||||
CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
|
CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
|
||||||
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
|
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
|
||||||
CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
|
CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
|
||||||
|
CURRENT_TAG = {{ "#{`git tag --points-at HEAD`.strip}" }}
|
||||||
|
|
||||||
# This is used to determine the `?v=` on the end of file URLs (for cache busting). We
|
# This is used to determine the `?v=` on the end of file URLs (for cache busting). We
|
||||||
# only need to expire modified assets, so we can use this to find the last commit that changes
|
# only need to expire modified assets, so we can use this to find the last commit that changes
|
||||||
@@ -229,19 +230,25 @@ error 500 do |env, exception|
|
|||||||
error_template(500, exception)
|
error_template(500, exception)
|
||||||
end
|
end
|
||||||
|
|
||||||
static_headers do |env|
|
|
||||||
env.response.headers.add("Cache-Control", "max-age=2629800")
|
|
||||||
end
|
|
||||||
|
|
||||||
# Init Kemal
|
# Init Kemal
|
||||||
|
|
||||||
public_folder "assets"
|
|
||||||
|
|
||||||
Kemal.config.powered_by_header = false
|
Kemal.config.powered_by_header = false
|
||||||
add_handler FilteredCompressHandler.new
|
add_handler FilteredCompressHandler.new
|
||||||
add_handler APIHandler.new
|
add_handler APIHandler.new
|
||||||
add_handler AuthHandler.new
|
add_handler AuthHandler.new
|
||||||
add_handler DenyFrame.new
|
add_handler DenyFrame.new
|
||||||
|
|
||||||
|
{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
|
||||||
|
Kemal.config.serve_static = false
|
||||||
|
add_handler Invidious::HttpServer::StaticAssetsHandler.new("assets", directory_listing: false)
|
||||||
|
{% else %}
|
||||||
|
public_folder "assets"
|
||||||
|
|
||||||
|
static_headers do |env|
|
||||||
|
env.response.headers.add("Cache-Control", "max-age=2629800")
|
||||||
|
end
|
||||||
|
{% end %}
|
||||||
|
|
||||||
add_context_storage_type(Array(String))
|
add_context_storage_type(Array(String))
|
||||||
add_context_storage_type(Preferences)
|
add_context_storage_type(Preferences)
|
||||||
add_context_storage_type(Invidious::User)
|
add_context_storage_type(Invidious::User)
|
||||||
@@ -256,11 +263,8 @@ Kemal.config.app_name = "Invidious"
|
|||||||
{% end %}
|
{% end %}
|
||||||
|
|
||||||
Kemal.run do |config|
|
Kemal.run do |config|
|
||||||
# Set max request line size if configured
|
config.server.not_nil!.max_request_line_size = 16384
|
||||||
if max_size = CONFIG.max_request_line_size
|
|
||||||
config.server.not_nil!.max_request_line_size = max_size
|
|
||||||
end
|
|
||||||
|
|
||||||
if socket_binding = CONFIG.socket_binding
|
if socket_binding = CONFIG.socket_binding
|
||||||
File.delete?(socket_binding.path)
|
File.delete?(socket_binding.path)
|
||||||
# Create a socket and set its desired permissions
|
# Create a socket and set its desired permissions
|
||||||
|
|||||||
@@ -3,15 +3,28 @@
|
|||||||
# IPv6 addresses.
|
# IPv6 addresses.
|
||||||
#
|
#
|
||||||
class TCPSocket
|
class TCPSocket
|
||||||
def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
|
{% if compare_versions(Crystal::VERSION, "1.18.0-dev") >= 0 %}
|
||||||
Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
|
def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
|
||||||
super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
|
Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
|
||||||
connect(addrinfo, timeout: connect_timeout) do |error|
|
super(family: addrinfo.family, type: addrinfo.type, protocol: addrinfo.protocol)
|
||||||
close
|
Socket.set_blocking(self.fd, blocking)
|
||||||
error
|
connect(addrinfo, timeout: connect_timeout) do |error|
|
||||||
|
close
|
||||||
|
error
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
{% else %}
|
||||||
|
def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
|
||||||
|
Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
|
||||||
|
super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
|
||||||
|
connect(addrinfo, timeout: connect_timeout) do |error|
|
||||||
|
close
|
||||||
|
error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
{% end %}
|
||||||
end
|
end
|
||||||
|
|
||||||
# :ditto:
|
# :ditto:
|
||||||
|
|||||||
120
src/invidious/http_server/static_assets_handler.cr
Normal file
120
src/invidious/http_server/static_assets_handler.cr
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %}
|
||||||
|
|
||||||
|
module Invidious::HttpServer
|
||||||
|
class StaticAssetsHandler < HTTP::StaticFileHandler
|
||||||
|
# In addition to storing the actual data of a file, it also implements the required
|
||||||
|
# getters needed for the object to imitate a `File::Stat` within `StaticFileHandler`.
|
||||||
|
#
|
||||||
|
# Since the `File::Stat` is created once in `#call` and then passed around to the
|
||||||
|
# rest of the class's methods, imitating the object allows us to only lookup
|
||||||
|
# the cache hash once for every request.
|
||||||
|
#
|
||||||
|
private record CachedFile, data : Bytes, size : Int64, modification_time : Time do
|
||||||
|
def directory?
|
||||||
|
false
|
||||||
|
end
|
||||||
|
|
||||||
|
def file?
|
||||||
|
true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
CACHE_LIMIT = 5_000_000 # 5MB
|
||||||
|
@@current_cache_size = 0
|
||||||
|
@@cached_files = {} of Path => CachedFile
|
||||||
|
|
||||||
|
# Returns metadata for the requested file
|
||||||
|
#
|
||||||
|
# If the requested file is cached, a `CachedFile` is returned instead of a `File::Stat`.
|
||||||
|
# This represents the metadata info of a cached file and implements all the methods of `File::Stat` that
|
||||||
|
# is used by the `StaticAssetsHandler`.
|
||||||
|
#
|
||||||
|
# The `CachedFile` also stores the raw bytes of the cached file, and this method serves as the place where
|
||||||
|
# the cached file is retrieved if it exists. Though the data will only be read in `#serve_file`
|
||||||
|
private def file_info(expanded_path : Path)
|
||||||
|
file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native))
|
||||||
|
{@@cached_files[file_path]? || File.info?(file_path), file_path}
|
||||||
|
end
|
||||||
|
|
||||||
|
# Add "Cache-Control" header to the response
|
||||||
|
private def add_cache_headers(response_headers : HTTP::Headers, last_modified : Time) : Nil
|
||||||
|
super; response_headers["Cache-Control"] = "max-age=2629800"
|
||||||
|
end
|
||||||
|
|
||||||
|
# Serves and caches the file at the given path.
|
||||||
|
#
|
||||||
|
# This is an override of `serve_file` to allow serving a file from memory, and to cache it
|
||||||
|
# it as needed.
|
||||||
|
private def serve_file(context : HTTP::Server::Context, file_info, file_path : Path, original_file_path : Path, last_modified : Time)
|
||||||
|
context.response.content_type = MIME.from_filename(original_file_path.to_s, "application/octet-stream")
|
||||||
|
|
||||||
|
range_header = context.request.headers["Range"]?
|
||||||
|
|
||||||
|
# If the file is cached we can just directly serve it
|
||||||
|
if file_info.is_a? CachedFile
|
||||||
|
return dispatch_serve(context, file_info.data, file_info, range_header)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Otherwise we'll need to read from disk and cache it
|
||||||
|
retrieve_bytes_from = IO::Memory.new
|
||||||
|
File.open(file_path) do |file|
|
||||||
|
# We cannot cache partial data so we'll rewind and read from the start
|
||||||
|
if range_header
|
||||||
|
dispatch_serve(context, file, file_info, range_header)
|
||||||
|
IO.copy(file.rewind, retrieve_bytes_from)
|
||||||
|
else
|
||||||
|
context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true)
|
||||||
|
dispatch_serve(context, file, file_info, range_header)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
return flush_io_to_cache(retrieve_bytes_from, file_path, file_info)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Writes file data to the cache
|
||||||
|
private def flush_io_to_cache(io, file_path, file_info)
|
||||||
|
if (@@current_cache_size += file_info.size) <= CACHE_LIMIT
|
||||||
|
@@cached_files[file_path] = CachedFile.new(io.to_slice, file_info.size, file_info.modification_time)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Either send the file in full, or just fragments of it depending on the request
|
||||||
|
private def dispatch_serve(context, file, file_info, range_header)
|
||||||
|
if range_header
|
||||||
|
# an IO is needed for `serve_file_range`
|
||||||
|
file = file.is_a?(Bytes) ? IO::Memory.new(file, writeable: false) : file
|
||||||
|
serve_file_range(context, file, range_header, file_info)
|
||||||
|
else
|
||||||
|
context.response.headers["Accept-Ranges"] = "bytes"
|
||||||
|
serve_file_full(context, file, file_info)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# If we're serving the full file right away then there's no need for an IO at all.
|
||||||
|
private def serve_file_full(context : HTTP::Server::Context, file : Bytes, file_info)
|
||||||
|
context.response.status = :ok
|
||||||
|
context.response.content_length = file_info.size
|
||||||
|
context.response.write file
|
||||||
|
end
|
||||||
|
|
||||||
|
# Serves segments of a file based on the `Range header`
|
||||||
|
#
|
||||||
|
# An override of `serve_file_range` to allow using a generic IO rather than a `File`.
|
||||||
|
# Literally the same code as what we inherited but just with the `file` argument's type
|
||||||
|
# being set to `IO` rather than `File`
|
||||||
|
#
|
||||||
|
# Can be removed once https://github.com/crystal-lang/crystal/issues/15817 is fixed.
|
||||||
|
private def serve_file_range(context : HTTP::Server::Context, file : IO, range_header : String, file_info)
|
||||||
|
# Paste in the body of inherited serve_file_range
|
||||||
|
{{@type.superclass.methods.select(&.name.==("serve_file_range"))[0].body}}
|
||||||
|
end
|
||||||
|
|
||||||
|
# Clear cached files.
|
||||||
|
#
|
||||||
|
# This is only used in the specs to clear the cache before each handler test
|
||||||
|
def self.clear_cache
|
||||||
|
@@current_cache_size = 0
|
||||||
|
return @@cached_files.clear
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
module Invidious::Routes::Companion
|
module Invidious::Routes::Companion
|
||||||
# /companion
|
# GET /companion
|
||||||
def self.get_companion(env)
|
def self.get_companion(env)
|
||||||
current_companion = env.get("current_companion").as(Int32)
|
current_companion = env.get("current_companion").as(Int32)
|
||||||
|
|
||||||
@@ -18,6 +18,24 @@ module Invidious::Routes::Companion
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# POST /companion
|
||||||
|
def self.post_companion(env)
|
||||||
|
url = env.request.path
|
||||||
|
if env.request.query
|
||||||
|
url += "?#{env.request.query}"
|
||||||
|
end
|
||||||
|
|
||||||
|
begin
|
||||||
|
COMPANION_POOL.client do |wrapper|
|
||||||
|
wrapper.client.post(url, env.request.headers, env.request.body) do |resp|
|
||||||
|
return self.proxy_companion(env, resp)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
rescue ex
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
def self.options_companion(env)
|
def self.options_companion(env)
|
||||||
current_companion = env.get("current_companion").as(Int32)
|
current_companion = env.get("current_companion").as(Int32)
|
||||||
|
|
||||||
|
|||||||
@@ -98,6 +98,8 @@ module Invidious::Routes::Login
|
|||||||
|
|
||||||
begin
|
begin
|
||||||
validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
|
validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
|
||||||
|
rescue ex : InfoException
|
||||||
|
return error_template(400, InfoException.new("Erroneous CAPTCHA"))
|
||||||
rescue ex
|
rescue ex
|
||||||
return error_template(400, ex)
|
return error_template(400, ex)
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -230,6 +230,7 @@ module Invidious::Routing
|
|||||||
def register_companion_routes
|
def register_companion_routes
|
||||||
if CONFIG.invidious_companion.present?
|
if CONFIG.invidious_companion.present?
|
||||||
get "/companion/*", Routes::Companion, :get_companion
|
get "/companion/*", Routes::Companion, :get_companion
|
||||||
|
post "/companion/*", Routes::Companion, :post_companion
|
||||||
options "/companion/*", Routes::Companion, :options_companion
|
options "/companion/*", Routes::Companion, :options_companion
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -344,8 +344,21 @@
|
|||||||
<% else %>
|
<% else %>
|
||||||
<%= translate(locale, "Current version: ") %>
|
<%= translate(locale, "Current version: ") %>
|
||||||
<% end %>
|
<% end %>
|
||||||
|
<% if CONFIG.modified_source_code_url %>
|
||||||
<%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>
|
<a href="<%= CONFIG.modified_source_code_url %>/commit/<%= CURRENT_COMMIT %>"><%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %></a>
|
||||||
|
<% else %>
|
||||||
|
<a href="https://github.com/iv-org/invidious/commit/<%= CURRENT_COMMIT %>"><%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %></a>
|
||||||
|
<% end %>
|
||||||
|
@ <%= CURRENT_BRANCH %>
|
||||||
|
<% if CURRENT_TAG != "" %>
|
||||||
|
(
|
||||||
|
<% if CONFIG.modified_source_code_url %>
|
||||||
|
<a href="<%= CONFIG.modified_source_code_url %>/releases/tag/<%= CURRENT_TAG %>"><%= CURRENT_TAG %></a>
|
||||||
|
<% else %>
|
||||||
|
<a href="https://github.com/iv-org/invidious/releases/tag/<%= CURRENT_TAG %>"><%= CURRENT_TAG %></a>
|
||||||
|
<% end %>
|
||||||
|
)
|
||||||
|
<% end %>
|
||||||
</span>
|
</span>
|
||||||
<div class="right">
|
<div class="right">
|
||||||
<a href="/privacy" title="<%= translate(locale, "footer_privacy_policy_link")%>"><%= translate(locale, "footer_privacy_policy_link") %></a>
|
<a href="/privacy" title="<%= translate(locale, "footer_privacy_policy_link")%>"><%= translate(locale, "footer_privacy_policy_link") %></a>
|
||||||
|
|||||||
Reference in New Issue
Block a user