From c582e72ebccee995a2e6d20d272c472215445f71 Mon Sep 17 00:00:00 2001 From: Ramon Date: Sat, 3 May 2025 11:43:36 +0200 Subject: [PATCH 1/9] feat: Instead of music, fetch subscription related videoids --- src/invidious/routes/api/v1/feeds.cr | 2 +- src/invidious/routes/feeds.cr | 2 +- src/invidious/trending.cr | 35 ++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index fea2993c..ef297996 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Feeds trending_type = env.params.query["type"]? begin - trending, plid = fetch_trending(trending_type, region, locale) + trending, plid = fetch_trending(trending_type, region, locale, env) rescue ex return error_json(500, ex) end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 7f9a0edb..961353ab 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -52,7 +52,7 @@ module Invidious::Routes::Feeds region ||= env.get("preferences").as(Preferences).region begin - trending, plid = fetch_trending(trending_type, region, locale) + trending, plid = fetch_trending(trending_type, region, locale, env) rescue ex return error_template(500, ex) end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 107d148d..688d3bb3 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -1,4 +1,4 @@ -def fetch_trending(trending_type, region, locale) +def fetch_trending(trending_type, region, locale, env) region ||= "US" region = region.upcase @@ -6,7 +6,8 @@ def fetch_trending(trending_type, region, locale) case trending_type.try &.downcase when "music" - params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" + # params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" + return fetch_subscription_related_videoids(env, region, locale) when "gaming" params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D" when "movies" @@ -40,3 +41,33 @@ def fetch_trending(trending_type, region, locale) # Deduplicate items before returning results return extracted.select(SearchVideo).uniq!(&.id), plid end + +def fetch_subscription_related_videoids(env, region, locale) + user = env.get("user").as(Invidious::User) + channel_videos, notifications = get_subscription_feed(user, 5, 1) + + videos = [] of SearchVideo + channel_videos.each do |video| + video = get_video(video.id) + related = video.related_videos + related.each do |related_video| + related_id = related_video["id"] + videos << SearchVideo.new({ + title: related_video["title"], + id: related_video["id"], + author: related_video["author"], + ucid: related_video["ucid"]? || "", + published: related_video["published"]?.try { |p| Time.parse_rfc3339(p) } || Time.utc, + views: related_video["view_count"]?.try &.to_i64 || 0_i64, + description_html: "", # not available + length_seconds: related_video["length_seconds"]?.try &.to_i || 0, + premiere_timestamp: nil, + author_verified: related_video["author_verified"]? == "true", + author_thumbnail: related_video["author_thumbnail"]?, + badges: VideoBadges::None, + }) + end + end + + return videos.uniq!(&.id), nil +end From cd1108039de075800a5d1b913b1db24cccaa17d9 Mon Sep 17 00:00:00 2001 From: Ramon Date: Fri, 9 May 2025 15:33:14 +0200 Subject: [PATCH 2/9] feat: Randomly fetch a few channels, then get their recommendations --- Dockerfile | 52 +++++++++++++++++++++++++++++++++++++++ deploy.sh | 4 +++ src/invidious/trending.cr | 22 +++++++++++------ 3 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 Dockerfile create mode 100755 deploy.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a07bef28 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +FROM crystallang/crystal:1.12.2-alpine AS builder + +RUN apk add --no-cache sqlite-static yaml-static + +ARG release + +WORKDIR /invidious +COPY ./shard.yml ./shard.yml +COPY ./shard.lock ./shard.lock +RUN shards install --production + +COPY ./src/ ./src/ +# TODO: .git folder is required for building – this is destructive. +# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. +COPY ./.git/ ./.git/ + +# Required for fetching player dependencies +COPY ./scripts/ ./scripts/ +COPY ./assets/ ./assets/ +COPY ./videojs-dependencies.yml ./videojs-dependencies.yml + +RUN crystal spec --warnings all \ + --link-flags "-lxml2 -llzma" +RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \ + crystal build ./src/invidious.cr \ + --release \ + --static --warnings all \ + --link-flags "-lxml2 -llzma"; \ + else \ + crystal build ./src/invidious.cr \ + --static --warnings all \ + --link-flags "-lxml2 -llzma"; \ + fi + +FROM alpine:3.20 +RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata +WORKDIR /invidious +RUN addgroup -g 1000 -S invidious && \ + adduser -u 1000 -S invidious -G invidious +COPY --chown=invidious ./config/config.* ./config/ +RUN mv -n config/config.example.yml config/config.yml +RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: invidious-db/' config/config.yml +COPY ./config/sql/ ./config/sql/ +COPY ./locales/ ./locales/ +COPY --from=builder /invidious/assets ./assets/ +COPY --from=builder /invidious/invidious . +RUN chmod o+rX -R ./assets ./config ./locales + +EXPOSE 3000 +USER invidious +ENTRYPOINT ["/sbin/tini", "--"] +CMD [ "/invidious/invidious" ] diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 00000000..0b40a8c5 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,4 @@ +docker buildx build --platform=linux/x86_64 --tag=invidious . && docker tag invidious main.local:20000/invidious/invidious +docker push main.local:20000/invidious/invidious +docker rmi $(docker images main.local:20000/invidious/invidious -q) --force +echo "Finished." diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 688d3bb3..4bb4fee1 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -44,20 +44,28 @@ end def fetch_subscription_related_videoids(env, region, locale) user = env.get("user").as(Invidious::User) - channel_videos, notifications = get_subscription_feed(user, 5, 1) + channel_videos, notifications = get_subscription_feed(user, 10, 1) videos = [] of SearchVideo - channel_videos.each do |video| - video = get_video(video.id) - related = video.related_videos + channel_videos.sample((channel_videos.size / 3).to_i).each do |channel_video| + next if channel_video.live_now || channel_video.premiere_timestamp || channel_video.length_seconds == 0 || channel_video.views == 0 + + video = get_video(channel_video.id) + next unless video.video_type == VideoType::Video + + related = video.related_videos.sample(10) # pick random related videos related.each do |related_video| - related_id = related_video["id"] + next unless id = related_video["id"]? + next unless related_video["view_count"]? && related_video["view_count"]? != 0 + next unless related_video["published"]? + next unless related_video["length_seconds"]? && related_video["length_seconds"]? != 0 + videos << SearchVideo.new({ title: related_video["title"], - id: related_video["id"], + id: id, author: related_video["author"], ucid: related_video["ucid"]? || "", - published: related_video["published"]?.try { |p| Time.parse_rfc3339(p) } || Time.utc, + published: (Time.parse_rfc3339(related_video["published"].to_s) rescue Time.utc), views: related_video["view_count"]?.try &.to_i64 || 0_i64, description_html: "", # not available length_seconds: related_video["length_seconds"]?.try &.to_i || 0, From 1478c768c16dc7eb8403e62f1be47a82a28d5ac5 Mon Sep 17 00:00:00 2001 From: Ramon Date: Fri, 16 May 2025 16:54:47 +0200 Subject: [PATCH 3/9] feat: Read watch history and subscriptions, merge and mix, get related videos --- src/invidious/trending.cr | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 4bb4fee1..8b3e42b4 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -44,13 +44,25 @@ end def fetch_subscription_related_videoids(env, region, locale) user = env.get("user").as(Invidious::User) - channel_videos, notifications = get_subscription_feed(user, 10, 1) + + # Filter valid channel videos + channel_videos, _ = get_subscription_feed(user, 10, 1) + valid_channel_videoids = channel_videos.select do |v| + !v.live_now && v.premiere_timestamp.nil? && (v.length_seconds || 0) > 0 && (v.views || 0) > 0 + end.map(&.id) + + # Sample more from watched, fewer from channels + watched_video_ids = user.watched.sample(10) + + video_ids = watched_video_ids + valid_channel_videoids + video_ids = video_ids.uniq + video_ids = video_ids.reject(&.nil?) + video_ids = video_ids.reject(&.empty?) + video_ids = video_ids.sample(10) if video_ids.size > 10 videos = [] of SearchVideo - channel_videos.sample((channel_videos.size / 3).to_i).each do |channel_video| - next if channel_video.live_now || channel_video.premiere_timestamp || channel_video.length_seconds == 0 || channel_video.views == 0 - - video = get_video(channel_video.id) + video_ids.each do |video_id| + video = get_video(video_id) next unless video.video_type == VideoType::Video related = video.related_videos.sample(10) # pick random related videos From 80e2e22804a01629a98846dabcebe623e9fa9d76 Mon Sep 17 00:00:00 2001 From: Ramon Date: Fri, 23 May 2025 15:01:58 +0200 Subject: [PATCH 4/9] feat: Track watch history when requesting through API --- src/invidious/routes/api/v1/videos.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 6a3eb8ae..9ff210b0 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -12,6 +12,10 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) + user = env.get("user").as(User) + if user.preferences.watch_history + Invidious::Database::Users.mark_watched(user, id) + end rescue ex : NotFoundException return error_json(404, ex) rescue ex From f444efeba87bcd0b90c2b72e779e67e276e64bee Mon Sep 17 00:00:00 2001 From: Ramon Date: Fri, 23 May 2025 15:12:22 +0200 Subject: [PATCH 5/9] feat: Replace the default suggestions when personalized suggestions are available --- src/invidious/trending.cr | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index afcdd29a..e853d51c 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -6,14 +6,17 @@ def fetch_trending(trending_type, region, locale, env) case trending_type.try &.downcase when "music" - # params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" - return fetch_subscription_related_videoids(env, region, locale) + params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" when "gaming" params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D" when "movies" params = "4gIKGgh0cmFpbGVycw%3D%3D" else # Default params = "" + suggested_videos = fetch_suggested_video_ids(env, region, locale) + if suggested_videos.size > 0 + return suggested_videos + end end client_config = YoutubeAPI::ClientConfig.new(region: region) @@ -42,30 +45,39 @@ def fetch_trending(trending_type, region, locale, env) return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid end -def fetch_subscription_related_videoids(env, region, locale) +def fetch_suggested_video_ids(env, region, locale) user = env.get("user").as(Invidious::User) - # Filter valid channel videos + if user.nil? + return [] of SearchVideo, nil + end + + # Get some new videos from the subscription feed channel_videos, _ = get_subscription_feed(user, 10, 1) valid_channel_videoids = channel_videos.select do |v| + # Make sure the video is not live, not a premiere, has a length and has views !v.live_now && v.premiere_timestamp.nil? && (v.length_seconds || 0) > 0 && (v.views || 0) > 0 end.map(&.id) - # Sample more from watched, fewer from channels - watched_video_ids = user.watched.sample(10) + # Get the last 10 watched videos + watched_video_ids = user.watched.last(10) - video_ids = watched_video_ids + valid_channel_videoids + video_ids = watched_video_ids + valid_channel_videoids.sample(4) video_ids = video_ids.uniq video_ids = video_ids.reject(&.nil?) video_ids = video_ids.reject(&.empty?) - video_ids = video_ids.sample(10) if video_ids.size > 10 + if video_ids.empty? + return [] of SearchVideo, nil + end + + # Fetch related videos for each video and return 10 random ones videos = [] of SearchVideo video_ids.each do |video_id| video = get_video(video_id) next unless video.video_type == VideoType::Video - related = video.related_videos.sample(10) # pick random related videos + related = video.related_videos.sample(10) related.each do |related_video| next unless id = related_video["id"]? next unless related_video["view_count"]? && related_video["view_count"]? != 0 From 289929f339cf65301be1bc57fb3eee6738a619af Mon Sep 17 00:00:00 2001 From: Ramon Date: Wed, 28 May 2025 17:06:01 +0200 Subject: [PATCH 6/9] feat: Add videos from subscriptions that have not been seen yet --- src/invidious/trending.cr | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index e853d51c..aac5d2e1 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -77,12 +77,32 @@ def fetch_suggested_video_ids(env, region, locale) video = get_video(video_id) next unless video.video_type == VideoType::Video + if !user.preferences.unseen_only || !user.watched.includes?(video.id) + videos << SearchVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid || "", + published: video.published || Time.utc, + views: video.views || 0_i64, + description_html: "", # not available + length_seconds: video.length_seconds || 0, + premiere_timestamp: video.premiere_timestamp, + author_verified: video.author_verified, + author_thumbnail: video.author_thumbnail, + badges: VideoBadges::None, + }) + end + + next unless video.related_videos + related = video.related_videos.sample(10) related.each do |related_video| next unless id = related_video["id"]? next unless related_video["view_count"]? && related_video["view_count"]? != 0 next unless related_video["published"]? next unless related_video["length_seconds"]? && related_video["length_seconds"]? != 0 + next if user.preferences.unseen_only && user.watched.includes?(related_video["id"]?) videos << SearchVideo.new({ title: related_video["title"], @@ -101,5 +121,5 @@ def fetch_suggested_video_ids(env, region, locale) end end - return videos.uniq!(&.id), nil + return videos.shuffle.uniq(&.id), nil end From 9c0757a500831df4f0e810f98c4f00523a5ed01b Mon Sep 17 00:00:00 2001 From: Ramon Date: Wed, 28 May 2025 17:11:05 +0200 Subject: [PATCH 7/9] chore: Remove helper files --- .DS_Store | Bin 0 -> 6148 bytes Dockerfile | 52 ---------------------------------------------------- deploy.sh | 4 ---- 3 files changed, 56 deletions(-) create mode 100644 .DS_Store delete mode 100644 Dockerfile delete mode 100755 deploy.sh diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Thu, 29 May 2025 15:12:01 +0200 Subject: [PATCH 8/9] chore: Delete .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Sun, 1 Jun 2025 16:34:35 +0200 Subject: [PATCH 9/9] fix: Crashfix for unauthenticated user --- src/invidious/routes/api/v1/videos.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 9ff210b0..6359d1e1 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -12,8 +12,9 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) - user = env.get("user").as(User) - if user.preferences.watch_history + + user = env.get?("user").try &.as?(User) + if user && user.preferences.watch_history Invidious::Database::Users.mark_watched(user, id) end rescue ex : NotFoundException