\n)
- str << %(
\n)
+ str << %(
)
+
+ if !first_page
+ self.first_page(str, locale, base_url.to_s)
+ end
+
+ str << %(
\n)
str << %(
)
if !ctoken.nil?
- params_next = URI::Params{"continuation" => ctoken}
- url_next = HttpServer::Utils.add_params_to_url(base_url, params_next)
+ params["continuation"] = ctoken
+ url_next = HttpServer::Utils.add_params_to_url(base_url, params)
self.next_page(str, locale, url_next.to_s)
end
diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr
index 1ba3ea61..bca2edda 100644
--- a/src/invidious/helpers/i18n.cr
+++ b/src/invidious/helpers/i18n.cr
@@ -54,6 +54,7 @@ LOCALES_LIST = {
"sr" => "Srpski (latinica)", # Serbian (Latin)
"sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic)
"sv-SE" => "Svenska", # Swedish
+ "ta" => "தமிழ்", # Tamil
"tr" => "Türkçe", # Turkish
"uk" => "Українська", # Ukrainian
"vi" => "Tiếng Việt", # Vietnamese
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
index b445107b..968ee47f 100644
--- a/src/invidious/jobs/notification_job.cr
+++ b/src/invidious/jobs/notification_job.cr
@@ -1,8 +1,32 @@
+struct VideoNotification
+ getter video_id : String
+ getter channel_id : String
+ getter published : Time
+
+ def_hash @channel_id, @video_id
+
+ def ==(other)
+ video_id == other.video_id
+ end
+
+ def self.from_video(video : ChannelVideo) : self
+ VideoNotification.new(video.id, video.ucid, video.published)
+ end
+
+ def initialize(@video_id, @channel_id, @published)
+ end
+
+ def clone : VideoNotification
+ VideoNotification.new(video_id.clone, channel_id.clone, published.clone)
+ end
+end
+
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
+ private getter notification_channel : ::Channel(VideoNotification)
private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
- def initialize(@connection_channel, @pg_url)
+ def initialize(@notification_channel, @connection_channel, @pg_url)
end
def begin
@@ -10,6 +34,70 @@ class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
+ # hash of channels to their videos (id+published) that need notifying
+ to_notify = Hash(String, Set(VideoNotification)).new(
+ ->(hash : Hash(String, Set(VideoNotification)), key : String) {
+ hash[key] = Set(VideoNotification).new
+ }
+ )
+ notify_mutex = Mutex.new
+
+ # fiber to locally cache all incoming notifications (from pubsub webhooks and refresh channels job)
+ spawn do
+ begin
+ loop do
+ notification = notification_channel.receive
+ notify_mutex.synchronize do
+ to_notify[notification.channel_id] << notification
+ end
+ end
+ end
+ end
+ # fiber to regularly persist all cached notifications
+ spawn do
+ loop do
+ begin
+ LOGGER.debug("NotificationJob: waking up")
+ cloned = {} of String => Set(VideoNotification)
+ notify_mutex.synchronize do
+ cloned = to_notify.clone
+ to_notify.clear
+ end
+
+ cloned.each do |channel_id, notifications|
+ if notifications.empty?
+ next
+ end
+
+ LOGGER.info("NotificationJob: updating channel #{channel_id} with #{notifications.size} notifications")
+ if CONFIG.enable_user_notifications
+ video_ids = notifications.map(&.video_id)
+ Invidious::Database::Users.add_multiple_notifications(channel_id, video_ids)
+ PG_DB.using_connection do |conn|
+ notifications.each do |n|
+ # Deliver notifications to `/api/v1/auth/notifications`
+ payload = {
+ "topic" => n.channel_id,
+ "videoId" => n.video_id,
+ "published" => n.published.to_unix,
+ }.to_json
+ conn.exec("NOTIFY notifications, E'#{payload}'")
+ end
+ end
+ else
+ Invidious::Database::Users.feed_needs_update(channel_id)
+ end
+ end
+
+ LOGGER.trace("NotificationJob: Done, sleeping")
+ rescue ex
+ LOGGER.error("NotificationJob: #{ex.message}")
+ end
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+
loop do
action, connection = connection_channel.receive
diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr
index 3439ae60..58805af2 100644
--- a/src/invidious/jsonify/api_v1/video_json.cr
+++ b/src/invidious/jsonify/api_v1/video_json.cr
@@ -268,7 +268,7 @@ module Invidious::JSONify::APIv1
json.field "viewCountText", rv["short_view_count"]?
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
json.field "published", rv["published"]?
- if !rv["published"]?.nil?
+ if rv["published"]?.try &.presence
json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale))
else
json.field "publishedText", ""
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 588bbc2a..a940ee68 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -368,6 +368,35 @@ module Invidious::Routes::API::V1::Channels
end
end
+ def self.courses(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
def self.community(env)
locale = env.get("preferences").as(Preferences).locale
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 368304ac..6a3eb8ae 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -429,4 +429,90 @@ module Invidious::Routes::API::V1::Videos
end
end
end
+
+ # Fetches transcripts from YouTube
+ #
+ # Use the `lang` and `autogen` query parameter to select which transcript to fetch
+ # Request without any URL parameters to see all the available transcripts.
+ def self.transcripts(env)
+ env.response.content_type = "application/json"
+
+ id = env.params.url["id"]
+ lang = env.params.query["lang"]?
+ label = env.params.query["label"]?
+ auto_generated = env.params.query["autogen"]? ? true : false
+
+ # Return all available transcript options when none is given
+ if !label && !lang
+ begin
+ video = get_video(id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ response = JSON.build do |json|
+ # The amount of transcripts available to fetch is the
+ # same as the amount of captions available.
+ available_transcripts = video.captions
+
+ json.object do
+ json.field "transcripts" do
+ json.array do
+ available_transcripts.each do |transcript|
+ json.object do
+ json.field "label", transcript.name
+ json.field "languageCode", transcript.language_code
+ json.field "autoGenerated", transcript.auto_generated
+
+ if transcript.auto_generated
+ json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}&autogen"
+ else
+ json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}"
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ return response
+ end
+
+ # If lang is not given then we attempt to fetch
+ # the transcript through the given label
+ if lang.nil?
+ begin
+ video = get_video(id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ target_transcript = video.captions.select(&.name.== label)
+ if target_transcript.empty?
+ return error_json(404, NotFoundException.new("Requested transcript does not exist"))
+ else
+ target_transcript = target_transcript[0]
+ lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
+ end
+ end
+
+ params = Invidious::Videos::Transcript.generate_param(id, lang, auto_generated)
+
+ begin
+ transcript = Invidious::Videos::Transcript.from_raw(
+ YoutubeAPI.get_transcript(params), lang, auto_generated
+ )
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return transcript.to_json
+ end
end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index e4a7e8a1..e662c931 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -197,6 +197,26 @@ module Invidious::Routes::Channels
templated "channel"
end
+ def self.courses(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_courses(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Courses
+ templated "channel"
+ end
+
def self.community(env)
return env.redirect env.request.path.sub("posts", "community") if env.request.path.split("/").last == "posts"
@@ -309,7 +329,7 @@ module Invidious::Routes::Channels
private KNOWN_TABS = {
"home", "videos", "shorts", "streams", "podcasts",
- "releases", "playlists", "community", "channels", "about",
+ "releases", "courses", "playlists", "community", "channels", "about",
"posts",
}
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 24619b61..48ce6768 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -143,32 +143,25 @@ module Invidious::Routes::Feeds
# RSS feeds
def self.rss_channel(env)
- locale = env.get("preferences").as(Preferences).locale
-
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- ucid = env.params.url["ucid"]
+ if env.params.url["ucid"].matches?(/^[\w-]+$/)
+ ucid = env.params.url["ucid"]
+ else
+ return error_atom(400, InfoException.new("Invalid channel ucid provided."))
+ end
params = HTTP::Params.parse(env.params.query["params"]? || "")
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
- rescue ex : NotFoundException
- return error_atom(404, ex)
- rescue ex
- return error_atom(500, ex)
- end
-
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
+ return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@@ -179,7 +172,7 @@ module Invidious::Routes::Feeds
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+ video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
@@ -187,7 +180,7 @@ module Invidious::Routes::Feeds
title: title,
id: video_id,
author: author,
- ucid: ucid,
+ ucid: video_ucid,
published: published,
views: views,
description_html: description_html,
@@ -199,30 +192,32 @@ module Invidious::Routes::Feeds
})
end
+ author = ""
+ author = videos[0].author if videos.size > 0
+
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
- xml.element("yt:channelId") { xml.text channel.ucid }
- xml.element("icon") { xml.text channel.author_thumbnail }
- xml.element("title") { xml.text channel.author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
+ xml.element("id") { xml.text "yt:channel:#{ucid}" }
+ xml.element("yt:channelId") { xml.text ucid }
+ xml.element("title") { author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
xml.element("author") do
- xml.element("name") { xml.text channel.author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
+ xml.element("name") { xml.text author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
xml.element("image") do
- xml.element("url") { xml.text channel.author_thumbnail }
- xml.element("title") { xml.text channel.author }
+ xml.element("url") { xml.text "" }
+ xml.element("title") { xml.text author }
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(channel.auto_generated, params, xml)
+ video.to_xml(false, params, xml)
end
end
end
@@ -310,8 +305,9 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- document = XML.parse(response.body)
+ return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
+ document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -428,16 +424,6 @@ module Invidious::Routes::Feeds
end
end
- if CONFIG.enable_user_notifications
- # Deliver notifications to `/api/v1/auth/notifications`
- payload = {
- "topic" => ucid,
- "videoId" => id,
- "published" => published.to_unix,
- }.to_json
- PG_DB.exec("NOTIFY notifications, E'#{payload}'")
- end
-
video = ChannelVideo.new({
id: id,
title: title,
@@ -453,11 +439,7 @@ module Invidious::Routes::Feeds
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
if was_insert
- if CONFIG.enable_user_notifications
- Invidious::Database::Users.add_notification(video)
- # else
- # Invidious::Database::Users.feed_needs_update(video)
- end
+ NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
end
end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index 2754cdb0..dfdd5a1b 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -48,12 +48,17 @@ module Invidious::Routes::Misc
referer = get_referer(env)
instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
- if instance_list.empty?
+ # Filter out the current instance
+ other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
+
+ if other_available_instances.empty?
+ # If the current instance is the only one, use the redirect URL as fallback
instance_url = "redirect.invidious.io"
else
+ # Select other random instance
# Sample returns an array
# Instances are packaged as {region, domain} in the instance list
- instance_url = instance_list.sample(1)[0][1]
+ instance_url = other_available_instances.sample(1)[0][1]
end
env.redirect "https://#{instance_url}#{referer}"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 975659eb..46b71f1f 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -120,6 +120,7 @@ module Invidious::Routing
get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
get "/channel/:ucid/releases", Routes::Channels, :releases
+ get "/channel/:ucid/courses", Routes::Channels, :courses
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
get "/channel/:ucid/posts", Routes::Channels, :community
@@ -237,6 +238,7 @@ module Invidious::Routing
get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
+ get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
# Feeds
get "/api/v1/trending", {{namespace}}::Feeds, :trending
@@ -250,6 +252,7 @@ module Invidious::Routing
get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
+ get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
index 4bd9f820..ee1272d1 100644
--- a/src/invidious/videos/transcript.cr
+++ b/src/invidious/videos/transcript.cr
@@ -122,5 +122,40 @@ module Invidious::Videos
return vtt
end
+
+ def to_json(json : JSON::Builder)
+ json.field "languageCode", @language_code
+ json.field "autoGenerated", @auto_generated
+ json.field "label", @label
+ json.field "body" do
+ json.array do
+ @lines.each do |line|
+ json.object do
+ if line.is_a? HeadingLine
+ json.field "type", "heading"
+ else
+ json.field "type", "regular"
+ end
+
+ json.field "startMs", line.start_ms.total_milliseconds
+ json.field "endMs", line.end_ms.total_milliseconds
+ json.field "line", line.line
+ end
+ end
+ end
+ end
+ end
+
+ def to_json
+ JSON.build do |json|
+ json.object do
+ json.field "transcript" do
+ json.object do
+ to_json(json)
+ end
+ end
+ end
+ end
+ end
end
end
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 4400b8ea..a64088a2 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -12,6 +12,7 @@
when .channels? then "/channel/#{ucid}/channels"
when .podcasts? then "/channel/#{ucid}/podcasts"
when .releases? then "/channel/#{ucid}/releases"
+ when .courses? then "/channel/#{ucid}/courses"
else
"/channel/#{ucid}"
end
@@ -21,7 +22,9 @@
page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
base_url: relative_url,
- ctoken: next_continuation
+ ctoken: next_continuation,
+ first_page: continuation.nil?,
+ params: env.params.query,
)
%>
@@ -41,6 +44,8 @@
<%- end -%>
+
+
<%= author %> - Invidious
<% end %>
diff --git a/src/invidious/views/components/items_paginated.ecr b/src/invidious/views/components/items_paginated.ecr
index 4534a0a3..f69df3fe 100644
--- a/src/invidious/views/components/items_paginated.ecr
+++ b/src/invidious/views/components/items_paginated.ecr
@@ -8,4 +8,14 @@
<%= page_nav_html %>
+
+
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr
index d7589381..32badc10 100644
--- a/src/invidious/views/components/player.ecr
+++ b/src/invidious/views/components/player.ecr
@@ -4,7 +4,7 @@
<% if params.autoplay %>autoplay<% end %>
<% if params.video_loop %>loop<% end %>
<% if params.controls %>controls<% end %>>
- <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
+ <% if (hlsvp = video.hls_manifest_url) && video.live_now && !CONFIG.disabled?("livestreams") %>
<% else %>
<% if params.listen %>