diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index 4149bd0bc..ba005d9ad 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -17,16 +17,26 @@ on: jobs: release: - runs-on: ubuntu-latest + strategy: + matrix: + include: + - os: ubuntu-latest + platform: linux/amd64 + name: "AMD64" + dockerfile: "docker/Dockerfile" + tag_suffix: "" + # GitHub doesn't have a ubuntu-latest-arm runner + - os: ubuntu-24.04-arm + platform: linux/arm64/v8 + name: "ARM64" + dockerfile: "docker/Dockerfile.arm64" + tag_suffix: "-arm64" + + runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 + uses: actions/checkout@v5 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -43,45 +53,22 @@ jobs: uses: docker/metadata-action@v5 with: images: quay.io/invidious/invidious + flavor: | + suffix=${{ matrix.tag_suffix }} tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w - - name: Build and push Docker AMD64 image for Push Event + - name: Build and push Docker ${{ matrix.name }} image for Push Event uses: docker/build-push-action@v6 with: context: . - file: docker/Dockerfile - platforms: linux/amd64 + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} push: true tags: ${{ steps.meta.outputs.tags }} build-args: | "release=1" - - - name: Docker meta - id: meta-arm64 - uses: docker/metadata-action@v5 - with: - images: quay.io/invidious/invidious - flavor: | - suffix=-arm64 - tags: | - type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - labels: | - quay.expires-after=12w - - - name: Build and push Docker ARM64 image for Push Event - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - labels: ${{ steps.meta-arm64.outputs.labels }} - push: true - tags: ${{ steps.meta-arm64.outputs.tags }} - build-args: | - "release=1" diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 1a23e68ca..1423bb695 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -8,16 +8,26 @@ on: jobs: release: - runs-on: ubuntu-latest + strategy: + matrix: + include: + - os: ubuntu-latest + platform: linux/amd64 + name: "AMD64" + dockerfile: "docker/Dockerfile" + tag_suffix: "" + # GitHub doesn't have a ubuntu-latest-arm runner + - os: ubuntu-24.04-arm + platform: linux/arm64/v8 + name: "ARM64" + dockerfile: "docker/Dockerfile.arm64" + tag_suffix: "-arm64" + + runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 + uses: actions/checkout@v5 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -36,46 +46,21 @@ jobs: images: quay.io/invidious/invidious flavor: | latest=false + suffix=${{ matrix.tag_suffix }} tags: | type=semver,pattern={{version}} type=raw,value=latest labels: | quay.expires-after=12w - - name: Build and push Docker AMD64 image for Push Event + - name: Build and push Docker ${{ matrix.name }} image for Push Event uses: docker/build-push-action@v6 with: context: . - file: docker/Dockerfile - platforms: linux/amd64 + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} push: true tags: ${{ steps.meta.outputs.tags }} build-args: | "release=1" - - - name: Docker meta - id: meta-arm64 - uses: docker/metadata-action@v5 - with: - images: quay.io/invidious/invidious - flavor: | - latest=false - suffix=-arm64 - tags: | - type=semver,pattern={{version}} - type=raw,value=latest - labels: | - quay.expires-after=12w - - - name: Build and push Docker ARM64 image for Push Event - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - labels: ${{ steps.meta-arm64.outputs.labels }} - push: true - tags: ${{ steps.meta-arm64.outputs.tags }} - build-args: | - "release=1" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d6a930ad..ce166b7b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: stable: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: true @@ -83,46 +83,43 @@ jobs: run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr build-docker: + strategy: + matrix: + include: + - os: ubuntu-latest + name: "AMD64" + # GitHub doesn't have a ubuntu-latest-arm runner + - os: ubuntu-24.04-arm + name: "ARM64" - runs-on: ubuntu-latest + name: Test ${{ matrix.name }} Docker build + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + + - name: Use ARM64 Dockerfile if ARM64 + if: ${{ matrix.name == 'ARM64' }} + run: sed -i 's/Dockerfile/Dockerfile.arm64/' docker-compose.yml - name: Build Docker - run: docker compose build --build-arg release=0 + run: docker compose build + + - name: Change hmac_key on docker-compose.yml + run: sed -i '/hmac_key/s/CHANGE_ME!!/docker-build-hmac-key/' docker-compose.yml - name: Run Docker run: docker compose up -d - name: Test Docker - run: while curl -Isf http://localhost:3000; do sleep 1; done + id: test + run: curl -If http://localhost:3000 --retry 5 --retry-delay 1 --retry-all-errors - build-docker-arm64: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker ARM64 image - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - build-args: release=0 - - - name: Test Docker - run: while curl -Isf http://localhost:3000; do sleep 1; done + - name: Print Invidious container logs + # Tells Github Actions to always run this step regardless of whether the previous step has failed + # Without this expression this step would simply be skipped when the previous step fails. + if: success() || steps.test.conclusion == 'failure' + run: docker compose logs lint: @@ -131,7 +128,7 @@ jobs: continue-on-error: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: true diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 65340d14b..ab45ce120 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 730 diff --git a/assets/css/player.css b/assets/css/player.css index 9cb400ad9..d95549ac7 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -86,6 +86,7 @@ ul.vjs-menu-content::-webkit-scrollbar { background-color: rgba(0, 0, 0, 0.75) !important; border-radius: 9px !important; padding: 5px !important; + line-height: 1.5 !important; } .vjs-play-control, diff --git a/assets/js/notifications.js b/assets/js/notifications.js index 55b7a15c6..16d9866d9 100644 --- a/assets/js/notifications.js +++ b/assets/js/notifications.js @@ -77,7 +77,7 @@ function create_notification_stream(subscriptions) { function update_ticker_count() { var notification_ticker = document.getElementById('notification_ticker'); - const notification_count = helpers.storage.get(STORAGE_KEY_STREAM); + const notification_count = helpers.storage.get(STORAGE_KEY_NOTIF_COUNT) || 0; if (notification_count > 0) { notification_ticker.innerHTML = '' + notification_count + ' '; diff --git a/assets/js/player.js b/assets/js/player.js index f32c9b561..108709159 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -5,6 +5,10 @@ var video_data = JSON.parse(document.getElementById('video_data').textContent); var options = { liveui: true, playbackRates: [0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + fontPercent: [0.5, 0.75, 1.25, 1.5, 1.75, 2, 3, 4], + windowOpacity: ['0', '0.5', '1'], + textOpacity: ['0.5', '1'], + persistTextTrackSettings: true, controlBar: { children: [ 'playToggle', @@ -180,7 +184,7 @@ var shareOptions = { }; if (location.pathname.startsWith('/embed/')) { - var overlay_content = '

' + player_data.title + '

'; + var overlay_content = '

' + player_data.title + '

'; player.overlay({ overlays: [ { start: 'loadstart', content: overlay_content, end: 'playing', align: 'top'}, @@ -450,7 +454,7 @@ if (!video_data.params.listen && video_data.params.annotations) { if (target === 'current') { location.href = path; } else if (target === 'new') { - open(path, '_blank'); + open(path, '_blank', 'noopener,noreferrer'); } }); @@ -585,6 +589,13 @@ const toggle_captions = (function () { }; })(); +// For real-time updates to captions (if currently showing) +function update_captions() { + if (document.body.querySelector('.vjs-text-track-cue')) { + toggle_captions(); toggle_captions(); + } +} + function toggle_fullscreen() { player.isFullscreen() ? player.exitFullscreen() : player.requestFullscreen(); } @@ -597,6 +608,34 @@ function increase_playback_rate(steps) { player.playbackRate(options.playbackRates[newIndex]); } +function increase_caption_size(steps) { + const maxIndex = options.fontPercent.length - 1; + const fontPercent = player.textTrackSettings.getValues().fontPercent || 1.25; + const curIndex = options.fontPercent.indexOf(fontPercent); + let newIndex = curIndex + steps; + newIndex = helpers.clamp(newIndex, 0, maxIndex); + player.textTrackSettings.setValues({ fontPercent: options.fontPercent[newIndex] }); + update_captions(); +} + +function toggle_caption_window() { + const numOptions = options.windowOpacity.length; + const windowOpacity = player.textTrackSettings.getValues().windowOpacity || '0'; + const curIndex = options.windowOpacity.indexOf(windowOpacity); + const newIndex = (curIndex + 1) % numOptions; + player.textTrackSettings.setValues({ windowOpacity: options.windowOpacity[newIndex] }); + update_captions(); +} + +function toggle_caption_opacity() { + const numOptions = options.textOpacity.length; + const textOpacity = player.textTrackSettings.getValues().textOpacity || '1'; + const curIndex = options.textOpacity.indexOf(textOpacity); + const newIndex = (curIndex + 1) % numOptions; + player.textTrackSettings.setValues({ textOpacity: options.textOpacity[newIndex] }); + update_captions(); +} + addEventListener('keydown', function (e) { if (e.target.tagName.toLowerCase() === 'input') { // Ignore input when focus is on certain elements, e.g. form fields. @@ -692,6 +731,12 @@ addEventListener('keydown', function (e) { case '>': action = increase_playback_rate.bind(this, 1); break; case '<': action = increase_playback_rate.bind(this, -1); break; + + case '=': action = increase_caption_size.bind(this, 1); break; + case '-': action = increase_caption_size.bind(this, -1); break; + + case 'w': action = toggle_caption_window; break; + case 'o': action = toggle_caption_opacity; break; default: console.info('Unhandled key down event: %s:', decoratedKey, e); diff --git a/assets/js/watch.js b/assets/js/watch.js index d869d40d1..ee9c29e89 100644 --- a/assets/js/watch.js +++ b/assets/js/watch.js @@ -141,7 +141,7 @@ function get_reddit_comments() { \

\ \ - {redditPermalinkText} \ + {redditPermalinkText} \ \ \
{contentHtml}
\ diff --git a/config/config.example.yml b/config/config.example.yml index 8d3e62120..cabbecfd7 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -75,17 +75,25 @@ db: ## If you are using a reverse proxy then you will probably need to ## configure the public_url to be the same as the domain used for Invidious. ## Also apply when used from an external IP address (without a domain). -## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282 +## Examples: https://MYINVIDIOUSDOMAIN/companion or http://192.168.1.100:8282/companion ## ## Both parameter can have identical URL when Invidious is hosted in ## an internal network or at home or locally (localhost). ## +## NOTE: If public_url is omitted, Invidious will use its built-in proxy +## to route companion requests through /companion, which is useful for +## simple setups where companion runs on the same network. When using +## the built-in proxy, CSP headers are not modified since requests +## stay within the same domain. +## ## Accepted values: "http(s)://:" ## Default: ## #invidious_companion: -# - private_url: "http://localhost:8282" -# public_url: "http://localhost:8282" +# - private_url: "http://localhost:8282/companion" +# public_url: "http://localhost:8282/companion" +# # Example with built-in proxy (omit public_url): +# # - private_url: "http://localhost:8282/companion" ## ## API key for Invidious companion, used for securing the communication @@ -865,7 +873,7 @@ default_user_preferences: ## ## Default dash video quality. ## - ## Note: this setting only takes effet if the + ## Note: this setting only takes effect if the ## 'quality' parameter is set to "dash". ## ## Accepted values: diff --git a/docker-compose.yml b/docker-compose.yml index afda87266..0de51feb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,10 @@ services: restart: unless-stopped ports: - "127.0.0.1:3000:3000" + depends_on: + invidious-db: + condition: service_healthy + restart: true environment: # Please read the following file for a comprehensive list of all available # configuration options and their associated syntax: diff --git a/src/invidious.cr b/src/invidious.cr index 69f8a26cb..197b150ca 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -60,7 +60,13 @@ alias IV = Invidious CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key -PG_DB = DB.open CONFIG.database_url +PG_DB = begin + DB.open CONFIG.database_url +rescue ex + puts "Failed to connect to PostgreSQL database: #{ex.cause.try &.message}" + puts "Check your 'config.yml' database settings or PostgreSQL settings." + exit(1) +end ARCHIVE_URL = URI.parse("https://archive.org") PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") REDDIT_URL = URI.parse("https://www.reddit.com") @@ -221,8 +227,8 @@ error 404 do |env| Invidious::Routes::ErrorRoutes.error_404(env) end -error 500 do |env, ex| - error_template(500, ex) +error 500 do |env, exception| + error_template(500, exception) end static_headers do |env| diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 49ffd9902..43843b119 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -3,8 +3,8 @@ private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} # TODO: Add "sort_by" def fetch_channel_community(ucid, cursor, locale, format, thin_mode) if cursor.nil? - # Egljb21tdW5pdHk%3D is the protobuf object to load "community" - initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D") + # EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts" + initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D") items = [] of JSON::Any extract_items(initial_data) do |item| @@ -24,15 +24,21 @@ def fetch_channel_community(ucid, cursor, locale, format, thin_mode) return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) end +def decode_ucid_from_post_protobuf(params) + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s) +end + def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) object = { - "2:string" => "community", - "25:embedded" => { - "22:string" => post_id.to_s, - }, - "45:embedded" => { - "2:varint" => 1_i64, - "3:varint" => 1_i64, + "56:embedded" => { + "2:string" => ucid, + "3:string" => post_id.to_s, + "11:string" => ucid, }, } params = object.try { |i| Protodec::Any.cast_json(i) } @@ -40,7 +46,7 @@ def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - initial_data = YoutubeAPI.browse(ucid, params: params) + initial_data = YoutubeAPI.browse("FEpost_detail", params: params) items = [] of JSON::Any extract_items(initial_data) do |item| diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index 9b45d0c86..cba1abd9c 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -6,19 +6,19 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) case sort_by when "last", "last_added" # Equivalent to "&sort=lad" - # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1} - "EglwbGF5bGlzdHMYBCABMAE%3D" + # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} + "EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA" when "oldest", "oldest_created" # formerly "&sort=da" # Not available anymore :c or maybe ?? - # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1} - "EglwbGF5bGlzdHMYAiABMAE%3D" + # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} + "EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA" # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1} # "EglwbGF5bGlzdHMYASABMAE%3D" when "newest", "newest_created" # Formerly "&sort=dd" - # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1} - "EglwbGF5bGlzdHMYAyABMAE%3D" + # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} + "EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA" end initial_data = YoutubeAPI.browse(ucid, params: params || "") diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 0716fcde6..e923b2f8d 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -16,23 +16,27 @@ module Invidious::Comments return parse_youtube(id, response, format, locale, thin_mode, sort_by) end - def fetch_community_post_comments(ucid, post_id) + def fetch_community_post_comments(ucid, post_id, sort_by = "top") + case sort_by + when "top" + sort_by_val = 0_i64 + when "new", "newest" + sort_by_val = 1_i64 + else # top + sort_by_val = 0_i64 + end + object = { - "2:string" => "community", - "25:embedded" => { - "22:string" => post_id, - }, - "45:embedded" => { - "2:varint" => 1_i64, - "3:varint" => 1_i64, - }, + "2:string" => "posts", "53:embedded" => { "4:embedded" => { - "6:varint" => 0_i64, - "27:varint" => 1_i64, + "6:varint" => sort_by_val, + "15:varint" => 2_i64, + "25:varint" => 0_i64, "29:string" => post_id, "30:string" => ucid, }, + "7:varint" => 0_i64, "8:string" => "comments-section", }, } @@ -43,7 +47,7 @@ module Invidious::Comments object2 = { "80226972:embedded" => { - "2:string" => ucid, + "2:string" => "FEcomment_post_detail_page_web_top_level", "3:string" => object_parsed, }, } @@ -320,6 +324,15 @@ module Invidious::Comments end def produce_continuation(video_id, cursor = "", sort_by = "top") + case sort_by + when "top" + sort_by_val = 0_i64 + when "new", "newest" + sort_by_val = 1_i64 + else # top + sort_by_val = 0_i64 + end + object = { "2:embedded" => { "2:string" => video_id, @@ -340,21 +353,12 @@ module Invidious::Comments "1:string" => cursor, "4:embedded" => { "4:string" => video_id, - "6:varint" => 0_i64, + "6:varint" => sort_by_val, }, "5:varint" => 20_i64, }, } - case sort_by - when "top" - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 - when "new", "newest" - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64 - else # top - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 - end - continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i) } diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 4d69854c4..e47e405ce 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -82,6 +82,9 @@ class Config @[YAML::Field(converter: Preferences::URIConverter)] property public_url : URI = URI.parse("") + + # Indicates if this companion instance uses the built-in proxy + property builtin_proxy : Bool = false end # Number of threads to use for crawling videos from channels (for updating subscriptions) @@ -271,6 +274,14 @@ class Config puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 characters." exit(1) end + + # Set public_url to built-in proxy path when omitted + config.invidious_companion.each do |companion| + if companion.public_url.to_s.empty? + companion.public_url = URI.parse("/companion") + companion.builtin_proxy = true + end + end elsif config.signature_server puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/companion-installation/") else diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index 15d925e34..c0926164e 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -34,7 +34,7 @@ module Invidious::Frontend::WatchPage str << " class=\"pure-form pure-form-stacked\"" str << " action='#{url}'" str << " method='post'" - str << " rel='noopener'" + str << " rel='noopener noreferrer'" str << " target='_blank'>" str << '\n' diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 13ea9fe95..7c5ef1185 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -61,28 +61,13 @@ class Kemal::ExceptionHandler end end -class FilteredCompressHandler < Kemal::Handler +class FilteredCompressHandler < HTTP::CompressHandler exclude ["/videoplayback", "/videoplayback/*", "/vi/*", "/sb/*", "/ggpht/*", "/api/v1/auth/notifications"] exclude ["/api/v1/auth/notifications", "/data_control"], "POST" - def call(env) - return call_next env if exclude_match? env - - {% if flag?(:without_zlib) %} - call_next env - {% else %} - request_headers = env.request.headers - - if request_headers.includes_word?("Accept-Encoding", "gzip") - env.response.headers["Content-Encoding"] = "gzip" - env.response.output = Compress::Gzip::Writer.new(env.response.output, sync_close: true) - elsif request_headers.includes_word?("Accept-Encoding", "deflate") - env.response.headers["Content-Encoding"] = "deflate" - env.response.output = Compress::Deflate::Writer.new(env.response.output, sync_close: true) - end - - call_next env - {% end %} + def call(context) + return call_next context if exclude_match? context + super end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index a940ee682..503b8c051 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -436,7 +436,7 @@ module Invidious::Routes::API::V1::Channels if ucid.nil? response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") return error_json(400, "Invalid post ID") if response["error"]? - ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) else ucid = ucid.to_s end @@ -460,13 +460,15 @@ module Invidious::Routes::API::V1::Channels format = env.params.query["format"]? format ||= "json" + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by ||= "top" continuation = env.params.query["continuation"]? case continuation when nil, "" ucid = env.params.query["ucid"] - comments = Comments.fetch_community_post_comments(ucid, id) + comments = Comments.fetch_community_post_comments(ucid, id, sort_by: sort_by) else comments = YoutubeAPI.browse(continuation: continuation) end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 4f5b58da2..4ae877a8b 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -190,15 +190,30 @@ module Invidious::Routes::API::V1::Misc sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint params = sub_endpoint.try &.dig?("params") + + if sub_endpoint["browseId"]?.try &.as_s == "FEpost_detail" + decoded_protobuf = params.try &.as_s.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + ucid = decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s) + post_id = decoded_protobuf.try(&.["56:0:embedded"]["3:1:string"].as_s) + else + ucid = sub_endpoint["browseId"]? if sub_endpoint["browseId"]? && sub_endpoint["browseId"]?.try &.as_s.starts_with? "UC" + post_id = nil + end rescue ex return error_json(500, ex) end JSON.build do |json| json.object do - json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? + json.field "browseId", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? + json.field "ucid", ucid if ucid != nil json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? + json.field "postId", post_id if post_id != nil json.field "params", params.try &.as_s json.field "pageType", page_type end diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index b52696687..63b935ec6 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -63,6 +63,7 @@ module Invidious::Routes::BeforeAll "/videoplayback", "/latest_version", "/download", + "/companion/", }.any? { |r| env.request.resource.starts_with? r } if env.request.cookies.has_key? "SID" diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 508aa3e41..6d2b4465c 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -284,7 +284,7 @@ module Invidious::Routes::Channels response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") return error_template(400, "Invalid post ID") if response["error"]? - ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s + ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) end diff --git a/src/invidious/routes/companion.cr b/src/invidious/routes/companion.cr new file mode 100644 index 000000000..11c2e3f59 --- /dev/null +++ b/src/invidious/routes/companion.cr @@ -0,0 +1,43 @@ +module Invidious::Routes::Companion + # /companion + def self.get_companion(env) + url = env.request.path + if env.request.query + url += "?#{env.request.query}" + end + + begin + COMPANION_POOL.client do |wrapper| + wrapper.client.get(url, env.request.headers) do |resp| + return self.proxy_companion(env, resp) + end + end + rescue ex + end + end + + def self.options_companion(env) + url = env.request.path + if env.request.query + url += "?#{env.request.query}" + end + + begin + COMPANION_POOL.client do |wrapper| + wrapper.client.options(url, env.request.headers) do |resp| + return self.proxy_companion(env, resp) + end + end + rescue ex + end + end + + private def self.proxy_companion(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + env.response.headers[key] = value + end + + return IO.copy response.body_io, env.response + end +end diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 930e4915a..6b0887d52 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -20,7 +20,7 @@ module Invidious::Routes::Embed return error_template(500, ex) end - url = "/embed/#{first_playlist_video}?#{env.params.query}" + url = "/embed/#{first_playlist_video.id}?#{env.params.query}" if env.params.query.size > 0 url += "?#{env.params.query}" @@ -209,10 +209,17 @@ module Invidious::Routes::Embed if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion.public_url}") - .gsub("connect-src", "connect-src #{invidious_companion.public_url}") + invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| + uri = + "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" + end.join(" ") + + if !invidious_companion_urls.empty? + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{invidious_companion_urls}") + .gsub("connect-src", "connect-src #{invidious_companion_urls}") + end end rendered "embed" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index e777b3f18..8a4fa2468 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -194,10 +194,17 @@ module Invidious::Routes::Watch if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion.public_url}") - .gsub("connect-src", "connect-src #{invidious_companion.public_url}") + invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| + uri = + "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" + end.join(" ") + + if !invidious_companion_urls.empty? + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{invidious_companion_urls}") + .gsub("connect-src", "connect-src #{invidious_companion_urls}") + end end templated "watch" diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 46b71f1f9..a51bb4b67 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -46,6 +46,7 @@ module Invidious::Routing self.register_api_v1_routes self.register_api_manifest_routes self.register_video_playback_routes + self.register_companion_routes end # ------------------- @@ -188,7 +189,7 @@ module Invidious::Routing end # ------------------- - # Media proxy routes + # Proxy routes # ------------------- def register_api_manifest_routes @@ -223,6 +224,13 @@ module Invidious::Routing get "/vi/:id/:name", Routes::Images, :thumbnails end + def register_companion_routes + if CONFIG.invidious_companion.present? + get "/companion/*", Routes::Companion, :get_companion + options "/companion/*", Routes::Companion, :options_companion + end + end + # ------------------- # API routes # ------------------- diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index feb584405..6b1dedd69 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -102,6 +102,9 @@ def extract_video_info(video_id : String) # Don't fetch the next endpoint if the video is unavailable. if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) + # Remove the microformat returned by the /next endpoint on some videos + # to prevent player_response microformat from being overwritten. + next_response.delete("microformat") player_response = player_response.merge(next_response) end @@ -111,16 +114,17 @@ def extract_video_info(video_id : String) if !CONFIG.invidious_companion.present? if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil? LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.") - players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile} + players_fallback = {YoutubeAPI::ClientType::TvSimply, YoutubeAPI::ClientType::WebMobile} players_fallback.each do |player_fallback| client_config.client_type = player_fallback next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config)) - if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url") + adaptive_formats = player_fallback_response.dig?("streamingData", "adaptiveFormats") + if adaptive_formats && (adaptive_formats.dig?(0, "url") || adaptive_formats.dig?(0, "signatureCipher")) streaming_data = player_response["streamingData"].as_h - streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"] + streaming_data["adaptiveFormats"] = adaptive_formats player_response["streamingData"] = JSON::Any.new(streaming_data) break end @@ -146,7 +150,11 @@ def extract_video_info(video_id : String) if streaming_data = player_response["streamingData"]? %w[formats adaptiveFormats].each do |key| streaming_data.as_h[key]?.try &.as_a.each do |format| - format.as_h["url"] = JSON::Any.new(convert_url(format)) + format = format.as_h + if format["url"]?.nil? + format["url"] = format["signatureCipher"] + end + format["url"] = JSON::Any.new(convert_url(format)) end end diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index af3521025..85fa4373f 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -65,12 +65,18 @@ <% end %> <% end %> - <% preferred_captions.each do |caption| %> - + <% preferred_captions.each do |caption| + api_captions_url = "/api/v1/captions/" + api_captions_url = invidious_companion.public_url.to_s + api_captions_url if (invidious_companion) + %> + <% end %> - <% captions.each do |caption| %> - + <% captions.each do |caption| + api_captions_url = "/api/v1/captions/" + api_captions_url = invidious_companion.public_url.to_s + api_captions_url if (invidious_companion) + %> + <% end %> <% end %> diff --git a/src/invidious/views/user/data_control.ecr b/src/invidious/views/user/data_control.ecr index 9ce42c994..e57926f50 100644 --- a/src/invidious/views/user/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr @@ -14,7 +14,7 @@
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0daed46c5..42241d159 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -46,8 +46,27 @@ struct YoutubeConnectionPool end end +# Packages a `HTTP::Client` to an Invidious companion instance alongside the configuration for that instance. +# +# This is used as the resource for the `CompanionPool` as to allow the ability to +# proxy the requests to Invidious companion from Invidious directly. +# Instead of setting up routes in a reverse proxy. +struct CompanionWrapper + property client : HTTP::Client + property companion : Config::CompanionConfig + + def initialize(companion : Config::CompanionConfig) + @companion = companion + @client = make_client(companion.private_url, use_http_proxy: false) + end + + def close + @client.close + end +end + struct CompanionConnectionPool - property pool : DB::Pool(HTTP::Client) + property pool : DB::Pool(CompanionWrapper) def initialize(capacity = 5, timeout = 5.0) options = DB::Pool::Options.new( @@ -57,26 +76,28 @@ struct CompanionConnectionPool checkout_timeout: timeout ) - @pool = DB::Pool(HTTP::Client).new(options) do + @pool = DB::Pool(CompanionWrapper).new(options) do companion = CONFIG.invidious_companion.sample - next make_client(companion.private_url, use_http_proxy: false) + make_client(companion.private_url, use_http_proxy: false) + CompanionWrapper.new(companion: companion) end end def client(&) - conn = pool.checkout + wrapper = pool.checkout begin - response = yield conn + response = yield wrapper rescue ex - conn.close + wrapper.close companion = CONFIG.invidious_companion.sample - conn = make_client(companion.private_url, use_http_proxy: false) + make_client(companion.private_url, use_http_proxy: false) + wrapper = CompanionWrapper.new(companion: companion) - response = yield conn + response = yield wrapper ensure - pool.release(conn) + pool.release(wrapper) end response diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b40092a1b..6fa8ae0ec 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -6,10 +6,10 @@ module YoutubeAPI extend self # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history - private ANDROID_APP_VERSION = "19.32.34" - private ANDROID_VERSION = "12" - private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip" - private ANDROID_SDK_VERSION = 31_i64 + private ANDROID_APP_VERSION = "19.35.36" + private ANDROID_VERSION = "13" + private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; en_US; SM-S908E Build/TP1A.220624.014) gzip" + private ANDROID_SDK_VERSION = 33_i64 private ANDROID_TS_APP_VERSION = "1.9" private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" @@ -17,9 +17,9 @@ module YoutubeAPI # For Apple device names, see https://gist.github.com/adamawolf/3048717 # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, # then go to the dedicated article of the major version you want. - private IOS_APP_VERSION = "19.32.8" - private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)" - private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build + private IOS_APP_VERSION = "20.11.6" + private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 18_5 like Mac OS X;)" + private IOS_VERSION = "18.5.0.22F76" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -42,6 +42,7 @@ module YoutubeAPI TvHtml5 TvHtml5ScreenEmbed + TvSimply end # List of hard-coded values used by the different clients @@ -49,7 +50,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20240814.00.00", + version: "2.20250222.10.00", screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -58,7 +59,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20240812.01.00", + version: "1.20250219.01.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -67,7 +68,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20240813.02.00", + version: "2.20250224.01.00", os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -75,7 +76,7 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20240814.00.00", + version: "2.20250222.10.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -84,7 +85,7 @@ module YoutubeAPI ClientType::WebCreator => { name: "WEB_CREATOR", name_proto: "62", - version: "1.20240918.03.00", + version: "1.20241203.01.00", os_name: "Windows", os_version: WINDOWS_VERSION, platform: "DESKTOP", @@ -170,7 +171,7 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20240813.07.00", + version: "7.20250219.14.00", }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", @@ -178,6 +179,11 @@ module YoutubeAPI version: "2.0", screen: "EMBED", }, + ClientType::TvSimply => { + name: "TVHTML5_SIMPLY", + name_proto: "74", + version: "1.0", + }, } #################################################################### @@ -695,22 +701,20 @@ module YoutubeAPI # Send the POST request begin - response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json) - body = response.body - if (response.status_code != 200) - raise Exception.new( - "Error while communicating with Invidious companion: \ - status code: #{response.status_code} and body: #{body.dump}" - ) + response_body = Hash(String, JSON::Any).new + + COMPANION_POOL.client do |wrapper| + companion_base_url = wrapper.companion.private_url.path + + wrapper.client.post("#{companion_base_url}#{endpoint}", headers: headers, body: data.to_json) do |response| + response_body = JSON.parse(response.body_io).as_h + end end + + return response_body rescue ex raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end - - # Convert result to Hash - initial_data = JSON.parse(body).as_h - - return initial_data end ####################################################################