mirrors/nadeko_invidious
1
0
Fork 0
mirror of https://git.nadeko.net/Fijxu/invidious.git synced 2025-12-15 01:25:08 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'master' into unix-sockets

Browse Source
This commit is contained in:
Caian Benedicto
2024-12-13 18:29:28 -03:00
parent 78773d7326 9892604758
commit d7f5cdc2f9
351 changed files with 40409 additions and 21596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/ext/kemal_content_for.cr Normal file
View File

@@ -0,0 +1,16 @@
# Overrides for Kemal's `content_for` macro in order to keep using
# kilt as it was before Kemal v1.1.1 (Kemal PR #618).
require "kemal"
require "kilt"
macro content_for(key, file = __FILE__)
%proc = ->() {
__kilt_io__ = IO::Memory.new
{{ yield }}
__kilt_io__.to_s
}
CONTENT_FOR_BLOCKS[{{key}}] = Tuple.new {{file}}, %proc
nil
end

33
src/invidious/helpers/static_file_handler.cr → src/ext/kemal_static_file_handler.cr
View File

@@ -111,7 +111,7 @@ module Kemal
if @fallthrough
call_next(context)
else
context.response.status_code = 405
context.response.status = HTTP::Status::METHOD_NOT_ALLOWED
context.response.headers.add("Allow", "GET, HEAD")
end
return
@@ -124,7 +124,7 @@ module Kemal
# File path cannot contains '\0' (NUL) because all filesystem I know
# don't accept '\0' character as file name.
if request_path.includes? '\0'
context.response.status_code = 400
context.response.status = HTTP::Status::BAD_REQUEST
return
end
@@ -143,13 +143,15 @@ module Kemal
add_cache_headers(context.response.headers, last_modified)
if cache_request?(context, last_modified)
context.response.status_code = 304
context.response.status = HTTP::Status::NOT_MODIFIED
return
end
send_file(context, file_path, file[:data], file[:filestat])
else
is_dir = Dir.exists? file_path
file_info = File.info?(file_path)
is_dir = file_info.try &.directory? || false
is_file = file_info.try &.file? || false
if request_path != expanded_path
redirect_to context, expanded_path
@@ -157,35 +159,34 @@ module Kemal
redirect_to context, expanded_path + '/'
end
if Dir.exists?(file_path)
return call_next(context) if file_info.nil?
if is_dir
if config.is_a?(Hash) && config["dir_listing"] == true
context.response.content_type = "text/html"
directory_listing(context.response, request_path, file_path)
else
call_next(context)
end
elsif File.exists?(file_path)
last_modified = modification_time(file_path)
elsif is_file
last_modified = file_info.modification_time
add_cache_headers(context.response.headers, last_modified)
if cache_request?(context, last_modified)
context.response.status_code = 304
context.response.status = HTTP::Status::NOT_MODIFIED
return
end
if @cached_files.sum { |element| element[1][:data].bytesize } + (size = File.size(file_path)) < CACHE_LIMIT
if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT
data = Bytes.new(size)
File.open(file_path) do |file|
file.read(data)
end
filestat = File.info(file_path)
File.open(file_path, &.read(data))
@cached_files[file_path] = {data: data, filestat: filestat}
send_file(context, file_path, data, filestat)
@cached_files[file_path] = {data: data, filestat: file_info}
send_file(context, file_path, data, file_info)
else
send_file(context, file_path)
end
else
else # Not a normal file (FIFO/device/socket)
call_next(context)
end
end

3861
src/invidious.cr
View File

File diff suppressed because it is too large Load Diff

984
src/invidious/channels.cr
View File

@@ -1,984 +0,0 @@
struct InvidiousChannel
include DB::Serializable
property id : String
property author : String
property updated : Time
property deleted : Bool
property subscribed : Time?
end
struct ChannelVideo
include DB::Serializable
property id : String
property title : String
property published : Time
property updated : Time
property ucid : String
property author : String
property length_seconds : Int32 = 0
property live_now : Bool = false
property premiere_timestamp : Time? = nil
property views : Int64? = nil
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "viewCount", self.views
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(locale, xml : XML::Builder | Nil = nil)
if xml
to_xml(locale, xml)
else
XML.build do |xml|
to_xml(locale, xml)
end
end
end
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| var.name }}}
}
{% end %}
end
end
struct AboutRelatedChannel
include DB::Serializable
property ucid : String
property author : String
property author_url : String
property author_thumbnail : String
end
# TODO: Refactor into either SearchChannel or InvidiousChannel
struct AboutChannel
include DB::Serializable
property ucid : String
property author : String
property auto_generated : Bool
property author_url : String
property author_thumbnail : String
property banner : String?
property description_html : String
property paid : Bool
property total_views : Int64
property sub_count : Int32
property joined : Time
property is_family_friendly : Bool
property allowed_regions : Array(String)
property related_channels : Array(AboutRelatedChannel)
property tabs : Array(String)
end
class ChannelRedirect < Exception
property channel_id : String
def initialize(@channel_id)
end
end
def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
finished_channel = Channel(String | Nil).new
spawn do
active_threads = 0
active_channel = Channel(Nil).new
channels.each do |ucid|
if active_threads >= max_threads
active_channel.receive
active_threads -= 1
end
active_threads += 1
spawn do
begin
get_channel(ucid, db, refresh, pull_all_videos)
finished_channel.send(ucid)
rescue ex
finished_channel.send(nil)
ensure
active_channel.send(nil)
end
end
end
end
final = [] of String
channels.size.times do
if ucid = finished_channel.receive
final << ucid
end
end
return final
end
def get_channel(id, db, refresh = true, pull_all_videos = true)
if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
if refresh && Time.utc - channel.updated > 10.minutes
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
args = arg_array(channel_array)
db.exec("INSERT INTO channels VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array)
end
else
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
args = arg_array(channel_array)
db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array)
end
return channel
end
def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}")
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse_html(rss)
author = rss.xpath_node(%q(//feed/title))
if !author
raise InfoException.new("Deleted or invalid channel")
end
author = author.content
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
page = 1
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = [] of SearchVideo
begin
initial_data = JSON.parse(response_body)
raise InfoException.new("Could not extract channel JSON") if !initial_data
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data")
videos = extract_videos(initial_data.as_h, author, ucid)
rescue ex
if response_body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
response_body.includes?("https://www.google.com/sorry/index")
raise InfoException.new("Could not extract channel info. Instance is likely blocked.")
end
raise ex
end
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//feed/entry").each do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
author = entry.xpath_node("author/name").not_nil!.content
ucid = entry.xpath_node("channelid").not_nil!.content
views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
views ||= 0_i64
channel_video = videos.select { |video| video.id == video_id }[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.live_now
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: Time.utc,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
})
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid)
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
end
if pull_all_videos
page += 1
ids = [] of String
loop do
response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
initial_data = JSON.parse(response_body)
raise InfoException.new("Could not extract channel JSON") if !initial_data
videos = extract_videos(initial_data.as_h, author, ucid)
count = videos.size
videos = videos.map { |video| ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
}) }
videos.each do |video|
ids << video.id
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
end
end
break if count < 25
page += 1
end
end
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
return channel
end
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
response_json = request_youtube_api_browse(continuation)
result = JSON.parse(response_json)
continuationItems = result["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem, nil if !continuationItems
items = [] of SearchItem
continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
extract_item(item, author, ucid).try { |t| items << t }
}
continuation = continuationItems.as_a.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
else
url = "/channel/#{ucid}/playlists?flow=list&view=1"
case sort_by
when "last", "last_added"
#
when "oldest", "oldest_created"
url += "&sort=da"
when "newest", "newest_created"
url += "&sort=dd"
else nil # Ignore
end
response = YT_POOL.client &.get(url)
initial_data = extract_initial_data(response.body)
return [] of SearchItem, nil if !initial_data
items = extract_items(initial_data, author, ucid)
continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?
end
return items, continuation
end
def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "videos",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
if !v2
if auto_generated
seed = Time.unix(1525757349)
until seed >= Time.utc
seed += 1.month
end
timestamp = seed - (page - 1).months
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}"
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}"
end
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64
object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({
"1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({
"1:varint" => 30_i64 * (page - 1),
}))),
})))
end
case sort_by
when "newest"
when "popular"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64
when "oldest"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64
else nil # Ignore
end
object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
object["80226972:embedded"].delete("3:base64")
continuation = object.try { |i| Protodec::Any.cast_json(object) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
# Used in bypass_captcha_job.cr
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
# ## NOTE: DEPRECATED
# Reason -> Unstable
# The Protobuf object must be provided with an id of the last playlist from the current "page"
# in order to fetch the next one accurately
# (if the id isn't included, entries shift around erratically between pages,
# leading to repetitions and skip overs)
#
# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user,
# it's better to stick to continuation tokens provided by the first request and onward
def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "playlists",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
if cursor
cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor
end
if auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64
case sort
when "oldest", "oldest_created"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64
when "newest", "newest_created"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
when "last", "last_added"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
else nil # Ignore
end
end
object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
object["80226972:embedded"].delete("3:base64")
continuation = object.try { |i| Protodec::Any.cast_json(object) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
# TODO: Add "sort_by"
def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
if response.status_code != 200
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en")
end
if response.status_code != 200
raise InfoException.new("This channel does not exist.")
end
ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"]
if !continuation || continuation.empty?
initial_data = extract_initial_data(response.body)
body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]?
if !body
raise InfoException.new("Could not extract community tab.")
end
body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]
else
continuation = produce_channel_community_continuation(ucid, continuation)
headers = HTTP::Headers.new
headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || ""
post_req = {
session_token: session_token,
}
response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req)
body = JSON.parse(response.body)
body = body["response"]["continuationContents"]["itemSectionContinuation"]? ||
body["response"]["continuationContents"]["backstageCommentsContinuation"]?
if !body
raise InfoException.new("Could not extract continuation.")
end
end
continuation = body["continuations"]?.try &.[0]["nextContinuationData"]["continuation"].as_s
posts = body["contents"].as_a
if message = posts[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || ""
raise InfoException.new(error_message)
end
response = JSON.build do |json|
json.object do
json.field "authorId", ucid
json.field "comments" do
json.array do
posts.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
next if !post
content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]?.try &.["simpleText"]? || ""
json.object do
json.field "author", author
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s
qualities.each do |quality|
json.object do
json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
if post["authorEndpoint"]?
json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
else
json.field "authorId", ""
json.field "authorUrl", ""
end
published_text = post["publishedTimeText"]["runs"][0]["text"].as_s
published = decode_date(published_text.rchop(" (edited)"))
if published_text.includes?(" (edited)")
json.field "isEdited", true
else
json.field "isEdited", false
end
like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"]
.try &.as_s.gsub(/\D/, "").to_i? || 0
json.field "content", html_to_content(content_html)
json.field "contentHtml", content_html
json.field "published", published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
json.field "likeCount", like_count
json.field "commentId", post["postId"]? || post["commentId"]? || ""
json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid
if attachment = post["backstageAttachment"]?
json.field "attachment" do
json.object do
case attachment.as_h
when .has_key?("videoRenderer")
attachment = attachment["videoRenderer"]
json.field "type", "video"
if !attachment["videoId"]?
error_message = (attachment["title"]["simpleText"]? ||
attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?)
json.field "error", error_message
else
video_id = attachment["videoId"].as_s
video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?
json.field "title", video_title
json.field "videoId", video_id
json.field "videoThumbnails" do
generate_thumbnails(json, video_id)
end
json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s)
author_info = attachment["ownerText"]["runs"][0].as_h
json.field "author", author_info["text"].as_s
json.field "authorId", author_info["navigationEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", author_info["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
# TODO: json.field "authorThumbnails", "channelThumbnailSupportedRenderers"
# TODO: json.field "authorVerified", "ownerBadges"
published = decode_date(attachment["publishedTimeText"]["simpleText"].as_s)
json.field "published", published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64
json.field "viewCount", view_count
json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count))
end
when .has_key?("backstageImageRenderer")
attachment = attachment["backstageImageRenderer"]
json.field "type", "image"
json.field "imageThumbnails" do
json.array do
thumbnail = attachment["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
qualities = {320, 560, 640, 1280, 2000}
qualities.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
# TODO
# when .has_key?("pollRenderer")
# attachment = attachment["pollRenderer"]
# json.field "type", "poll"
else
json.field "type", "unknown"
json.field "error", "Unrecognized attachment type."
end
end
end
end
if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? ||
comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i?)
continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
continuation ||= ""
json.field "replies" do
json.object do
json.field "replyCount", reply_count
json.field "continuation", extract_channel_community_cursor(continuation)
end
end
end
end
end
end
end
if body["continuations"]?
continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s
json.field "continuation", extract_channel_community_cursor(continuation)
end
end
end
if format == "html"
response = JSON.parse(response)
content_html = template_youtube_comments(response, locale, thin_mode)
response = JSON.build do |json|
json.object do
json.field "contentHtml", content_html
end
end
end
return response
end
def produce_channel_community_continuation(ucid, cursor)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => cursor || "",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(object) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def extract_channel_community_cursor(continuation)
object = URI.decode_www_form(continuation)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h }
if object["53:2:embedded"]?.try &.["3:0:embedded"]?
object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"]
.try { |i| i["2:0:base64"].as_h }
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64")
end
cursor = Protodec::Any.cast_json(object)
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
cursor
end
def get_about_info(ucid, locale)
result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en")
if result.status_code != 200
result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en")
end
if md = result.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/)
raise ChannelRedirect.new(channel_id: md["ucid"])
end
if result.status_code != 200
raise InfoException.new("This channel does not exist.")
end
about = XML.parse_html(result.body)
if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")]))
raise InfoException.new("This channel does not exist.")
end
initdata = extract_initial_data(result.body)
if initdata.empty?
error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip
error_message ||= translate(locale, "Could not get channel info.")
raise InfoException.new(error_message)
end
if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]?
raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s)
end
auto_generated = false
# Check for special auto generated gaming channels
if !initdata.has_key?("metadata")
auto_generated = true
end
if auto_generated
author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
# Raises a KeyError on failure.
banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banner = banners.try &.[-1]?.try &.["url"].as_s?
description = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]["simpleText"].as_s
description_html = HTML.escape(description).gsub("\n", "<br>")
paid = false
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s }
related_channels = [] of AboutRelatedChannel
else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s
ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
# Raises a KeyError on failure.
banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banner = banners.try &.[-1]?.try &.["url"].as_s?
# if banner.includes? "channels/c4/default_banner"
# banner = nil
# end
description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || ""
description_html = HTML.escape(description).gsub("\n", "<br>")
paid = about.xpath_node(%q(//meta[@itemprop="paid"])).not_nil!["content"] == "True"
is_family_friendly = about.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True"
allowed_regions = about.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",")
related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"]
.["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]?
.try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node|
renderer = node["miniChannelRenderer"]?
related_id = renderer.try &.["channelId"]?.try &.as_s?
related_id ||= ""
related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s?
related_title ||= ""
related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]?
.try &.["url"]?.try &.as_s?
related_author_url ||= ""
related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a?
related_author_thumbnails ||= [] of JSON::Any
related_author_thumbnail = ""
if related_author_thumbnails.size > 0
related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s?
related_author_thumbnail ||= ""
end
AboutRelatedChannel.new({
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
})
end
related_channels ||= [] of AboutRelatedChannel
end
total_views = 0_i64
joined = Time.unix(0)
tabs = [] of String
tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
if !tabs_json.nil?
# Retrieve information from the tabs array. The index we are looking for varies between channels.
tabs_json.each do |node|
# Try to find the about section which is located in only one of the tabs.
channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
.try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
.try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
if !channel_about_meta.nil?
total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s }
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
(channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
auto_generated = true
end
end
end
tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase }
end
sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]) } || 0
AboutChannel.new({
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description_html: description_html,
paid: paid,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
related_channels: related_channels,
tabs: tabs,
})
end
def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
continuation = produce_channel_videos_continuation(ucid, page,
auto_generated: auto_generated, sort_by: sort_by, v2: true)
return request_youtube_api_browse(continuation)
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
videos = [] of SearchVideo
2.times do |i|
response_json = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
initial_data = JSON.parse(response_json)
break if !initial_data
videos.concat extract_videos(initial_data.as_h, author, ucid)
end
return videos.size, videos
end
def get_latest_videos(ucid)
response_json = get_channel_videos_response(ucid)
initial_data = JSON.parse(response_json)
return [] of SearchVideo if !initial_data
author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
items = extract_videos(initial_data.as_h, author, ucid)
return items
end

206
src/invidious/channels/about.cr Normal file
View File

@@ -0,0 +1,206 @@
# TODO: Refactor into either SearchChannel or InvidiousChannel
record AboutChannel,
ucid : String,
author : String,
auto_generated : Bool,
author_url : String,
author_thumbnail : String,
banner : String?,
description : String,
description_html : String,
total_views : Int64,
sub_count : Int32,
joined : Time,
is_family_friendly : Bool,
allowed_regions : Array(String),
tabs : Array(String),
tags : Array(String),
verified : Bool,
is_age_gated : Bool
def get_about_info(ucid, locale) : AboutChannel
begin
# Fetch channel information from channel home page
initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end
if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR"
error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
end
if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]?
raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s)
end
auto_generated = false
# Check for special auto generated gaming channels
if !initdata.has_key?("metadata")
auto_generated = true
end
tags = [] of String
tab_names = [] of String
total_views = 0_i64
joined = Time.unix(0)
if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer")
description_node = nil
author = age_gate_renderer["channelTitle"].as_s
ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s
author_url = "https://www.youtube.com/channel/#{ucid}"
author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s
banner = nil
is_family_friendly = false
is_age_gated = true
tab_names = ["videos", "shorts", "streams"]
auto_generated = false
else
if auto_generated
author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
# Raises a KeyError on failure.
banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banner = banners.try &.[-1]?.try &.["url"].as_s?
description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]
# some channels have the description in a simpleText
# ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
description_node = description_base_node.dig?("simpleText") || description_base_node
tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges")
.try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s
author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges"))
ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
# Raises a KeyError on failure.
banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
banner = banners.try &.[-1]?.try &.["url"].as_s?
# if banner.includes? "channels/c4/default_banner"
# banner = nil
# end
description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String
end
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
tab_names = tabs_json.as_a.compact_map do |entry|
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
# This is a small fix to not add extra code on the HTML side
# I.e, the URL for the "live" tab is .../streams, so use "streams"
# everywhere for the sake of simplicity
(name == "live") ? "streams" : name
end
# Get the currently active tab ("About")
about_tab = extract_selected_tab(tabs_json)
# Try to find the about metadata section
channel_about_meta = about_tab.dig?(
"content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"channelAboutFullMetadataRenderer"
)
if !channel_about_meta.nil?
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = extract_text(channel_about_meta["joinedDateText"]?)
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" ||
channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube"
)
end
end
end
allowed_regions = initdata
.dig?("microformat", "microformatDataRenderer", "availableCountries")
.try &.as_a.map(&.as_s) || [] of String
description = !description_node.nil? ? description_node.as_s : ""
description_html = HTML.escape(description)
if !description_node.nil?
if description_node.as_h?.nil?
description_node = text_to_parsed_content(description_node.as_s)
end
description_html = parse_content(description_node)
if description_html == "" && description != ""
description_html = HTML.escape(description)
end
end
sub_count = 0
if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
metadata_rows.each do |row|
metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
if !metadata_part.nil?
sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
end
break if sub_count != 0
end
end
AboutChannel.new(
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description: description,
description_html: description_html,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
tabs: tab_names,
tags: tags,
verified: author_verified || false,
is_age_gated: is_age_gated || false,
)
end
def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
if continuation.nil?
# params is {"2:string":"channels"} encoded
initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation)
end
items, continuation = extract_items(initial_data)
return items.select(SearchChannel), continuation
end

304
src/invidious/channels/channels.cr Normal file
View File

@@ -0,0 +1,304 @@
struct InvidiousChannel
include DB::Serializable
property id : String
property author : String
property updated : Time
property deleted : Bool
property subscribed : Time?
end
struct ChannelVideo
include DB::Serializable
property id : String
property title : String
property published : Time
property updated : Time
property ucid : String
property author : String
property length_seconds : Int32 = 0
property live_now : Bool = false
property premiere_timestamp : Time? = nil
property views : Int64? = nil
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "viewCount", self.views
end
end
def to_json(locale, _json : Nil = nil)
JSON.build do |json|
to_json(locale, json)
end
end
def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(locale, _xml : Nil = nil)
XML.build do |xml|
to_xml(locale, xml)
end
end
def to_tuple
{% begin %}
{
{{@type.instance_vars.map(&.name).splat}}
}
{% end %}
end
end
class ChannelRedirect < Exception
property channel_id : String
def initialize(@channel_id)
end
end
def get_batch_channels(channels)
finished_channel = Channel(String | Nil).new
max_threads = 10
spawn do
active_threads = 0
active_channel = Channel(Nil).new
channels.each do |ucid|
if active_threads >= max_threads
active_channel.receive
active_threads -= 1
end
active_threads += 1
spawn do
begin
get_channel(ucid)
finished_channel.send(ucid)
rescue ex
finished_channel.send(nil)
ensure
active_channel.send(nil)
end
end
end
end
final = [] of String
channels.size.times do
if ucid = finished_channel.receive
final << ucid
end
end
return final
end
def get_channel(id) : InvidiousChannel
channel = Invidious::Database::Channels.select(id)
if channel.nil? || (Time.utc - channel.updated) > 2.days
channel = fetch_channel(id, pull_all_videos: false)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
end
return channel
end
def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse(rss)
author = rss.xpath_node("//default:feed/default:title", namespaces)
if !author
raise InfoException.new("Deleted or invalid channel")
end
author = author.content
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
title = entry.xpath_node("default:title", namespaces).not_nil!.content
published = Time.parse_rfc3339(
entry.xpath_node("default:published", namespaces).not_nil!.content
)
updated = Time.parse_rfc3339(
entry.xpath_node("default:updated", namespaces).not_nil!.content
)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
views = entry
.xpath_node("media:group/media:community/media:statistics", namespaces)
.try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.badges.live_now?
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: updated,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
})
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
if CONFIG.enable_user_notifications
Invidious::Database::Users.add_notification(video)
else
Invidious::Database::Users.feed_needs_update(video)
end
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
end
if pull_all_videos
loop do
# Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
count = 0
videos.select(SearchVideo).each do |video|
count += 1
video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
if CONFIG.enable_user_notifications
Invidious::Database::Users.add_notification(video)
else
Invidious::Database::Users.feed_needs_update(video)
end
end
end
end
break if count < 25
sleep 500.milliseconds
end
end
channel.updated = Time.utc
return channel
end

332
src/invidious/channels/community.cr Normal file
View File

@@ -0,0 +1,332 @@
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
# TODO: Add "sort_by"
def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
if cursor.nil?
# Egljb21tdW5pdHk%3D is the protobuf object to load "community"
initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D")
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
else
continuation = produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
raise InfoException.new("Can't extract community data") if container.nil?
items = container.as_a
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
end
def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
object = {
"2:string" => "community",
"25:embedded" => {
"22:string" => post_id.to_s,
},
"45:embedded" => {
"2:varint" => 1_i64,
"3:varint" => 1_i64,
},
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
initial_data = YoutubeAPI.browse(ucid, params: params)
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true)
end
def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false)
if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || ""
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
end
response = JSON.build do |json|
json.object do
json.field "authorId", ucid
if is_single_post
json.field "singlePost", true
end
json.field "comments" do
json.array do
items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
next if !post
content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || ""
json.object do
json.field "author", author
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s
qualities.each do |quality|
json.object do
json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
if post["authorEndpoint"]?
json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
else
json.field "authorId", ""
json.field "authorUrl", ""
end
published_text = post["publishedTimeText"]["runs"][0]["text"].as_s
published = decode_date(published_text.rchop(" (edited)"))
if published_text.includes?(" (edited)")
json.field "isEdited", true
else
json.field "isEdited", false
end
like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"]
.try &.as_s.gsub(/\D/, "").to_i? || 0
reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")
json.field "content", html_to_content(content_html)
json.field "contentHtml", content_html
json.field "published", published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
json.field "likeCount", like_count
json.field "replyCount", reply_count
json.field "commentId", post["postId"]? || post["commentId"]? || ""
json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid
if attachment = post["backstageAttachment"]?
json.field "attachment" do
case attachment.as_h
when .has_key?("videoRenderer")
parse_item(attachment)
.as(SearchVideo)
.to_json(locale, json)
when .has_key?("backstageImageRenderer")
json.object do
attachment = attachment["backstageImageRenderer"]
json.field "type", "image"
json.field "imageThumbnails" do
json.array do
thumbnail = attachment["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
when .has_key?("pollRenderer")
json.object do
attachment = attachment["pollRenderer"]
json.field "type", "poll"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
# A choice can have an image associated with it.
# Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re
if choice["image"]?
thumbnail = choice["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
json.field "image" do
json.array do
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
end
end
end
end
end
when .has_key?("postMultiImageRenderer")
json.object do
attachment = attachment["postMultiImageRenderer"]
json.field "type", "multiImage"
json.field "images" do
json.array do
attachment["images"].as_a.each do |image|
json.array do
thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
end
end
when .has_key?("playlistRenderer")
parse_item(attachment)
.as(SearchPlaylist)
.to_json(locale, json)
when .has_key?("quizRenderer")
json.object do
attachment = attachment["quizRenderer"]
json.field "type", "quiz"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
json.field "isCorrect", choice["isCorrect"].as_bool
end
end
end
end
end
else
json.object do
json.field "type", "unknown"
json.field "error", "Unrecognized attachment type."
end
end
end
end
if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? ||
comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i?)
continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
continuation ||= ""
json.field "replies" do
json.object do
json.field "replyCount", reply_count
json.field "continuation", extract_channel_community_cursor(continuation)
end
end
end
end
end
end
end
if !is_single_post
if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
json.field "continuation", extract_channel_community_cursor(cont.as_s)
end
end
end
end
if format == "html"
response = JSON.parse(response)
content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode)
response = JSON.build do |json|
json.object do
json.field "contentHtml", content_html
end
end
end
return response
end
def produce_channel_community_continuation(ucid, cursor)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => cursor || "",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def extract_channel_community_cursor(continuation)
object = URI.decode_www_form(continuation)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try(&.["80226972:0:embedded"]["3:1:base64"].as_h)
if object["53:2:embedded"]?.try &.["3:0:embedded"]?
object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"]
.try(&.["2:0:base64"].as_h)
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64")
end
cursor = Protodec::Any.cast_json(object)
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
cursor
end

46
src/invidious/channels/playlists.cr Normal file
View File

@@ -0,0 +1,46 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
params =
case sort_by
when "last", "last_added"
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYBCABMAE%3D"
when "oldest", "oldest_created"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYAiABMAE%3D"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYAyABMAE%3D"
end
initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_podcasts(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA")
end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_releases(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA")
end
return extract_items(initial_data, author, ucid)
end

192
src/invidious/channels/videos.cr Normal file
View File

@@ -0,0 +1,192 @@
module Invidious::Channel::Tabs
extend self
# -------------------
# Regular videos
# -------------------
# Wrapper for AboutChannel, as we still need to call get_videos with
# an author name and ucid directly (e.g in RSS feeds).
# TODO: figure out how to get rid of that
def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
return get_videos(
channel.author, channel.ucid,
continuation: continuation, sort_by: sort_by
)
end
# Wrapper for InvidiousChannel, as we still need to call get_videos with
# an author name and ucid directly (e.g in RSS feeds).
# TODO: figure out how to get rid of that
def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest")
return get_videos(
channel.author, channel.id,
continuation: continuation, sort_by: sort_by
)
end
def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_videos_ctoken(ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, author, ucid)
end
def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
if continuation.nil?
# Fetch the first "page" of video
items, next_continuation = get_videos(channel, sort_by: sort_by)
else
# Fetch a "page" of videos using the given continuation token
items, next_continuation = get_videos(channel, continuation: continuation)
end
# If there is more to load, then load a second "page"
# and replace the previous continuation token
if !next_continuation.nil?
items_2, next_continuation = get_videos(channel, continuation: next_continuation)
items.concat items_2
end
return items, next_continuation
end
# -------------------
# Shorts
# -------------------
def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, channel.author, channel.ucid)
end
# -------------------
# Livestreams
# -------------------
def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, channel.author, channel.ucid)
end
def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
if continuation.nil?
# Fetch the first "page" of stream
items, next_continuation = get_livestreams(channel, sort_by: sort_by)
else
# Fetch a "page" of streams using the given continuation token
items, next_continuation = get_livestreams(channel, continuation: continuation)
end
# If there is more to load, then load a second "page"
# and replace the previous continuation token
if !next_continuation.nil?
items_2, next_continuation = get_livestreams(channel, continuation: next_continuation)
items.concat items_2
end
return items, next_continuation
end
# -------------------
# C-tokens
# -------------------
private def sort_options_videos_short(sort_by : String)
case sort_by
when "newest" then return 4_i64
when "popular" then return 2_i64
when "oldest" then return 5_i64
else return 4_i64 # Fallback to "newest"
end
end
# Generate the initial "continuation token" to get the first page of the
# "videos" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
object = {
"15:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"4:varint" => sort_options_videos_short(sort_by),
},
}
return channel_ctoken_wrap(ucid, object)
end
# Generate the initial "continuation token" to get the first page of the
# "shorts" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
object = {
"10:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"4:varint" => sort_options_videos_short(sort_by),
},
}
return channel_ctoken_wrap(ucid, object)
end
# Generate the initial "continuation token" to get the first page of the
# "livestreams" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
sort_by_numerical =
case sort_by
when "newest" then 12_i64
when "popular" then 14_i64
when "oldest" then 13_i64
else 12_i64 # Fallback to "newest"
end
object = {
"14:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"5:varint" => sort_by_numerical,
},
}
return channel_ctoken_wrap(ucid, object)
end
# The protobuf structure common between videos/shorts/livestreams
private def channel_ctoken_wrap(ucid : String, object)
object_inner = {
"110:embedded" => {
"3:embedded" => object,
},
}
object_inner_encoded = object_inner
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => object_inner_encoded,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
end

662
src/invidious/comments.cr
View File

@@ -1,662 +0,0 @@
class RedditThing
include JSON::Serializable
property kind : String
property data : RedditComment | RedditLink | RedditMore | RedditListing
end
class RedditComment
include JSON::Serializable
property author : String
property body_html : String
property replies : RedditThing | String
property score : Int32
property depth : Int32
property permalink : String
@[JSON::Field(converter: RedditComment::TimeConverter)]
property created_utc : Time
module TimeConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix(value.read_float.to_i)
end
def self.to_json(value : Time, json : JSON::Builder)
json.number(value.to_unix)
end
end
end
struct RedditLink
include JSON::Serializable
property author : String
property score : Int32
property subreddit : String
property num_comments : Int32
property id : String
property permalink : String
property title : String
end
struct RedditMore
include JSON::Serializable
property children : Array(String)
property count : Int32
property depth : Int32
end
class RedditListing
include JSON::Serializable
property children : Array(RedditThing)
property modhash : String
end
def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top", action = "action_get_comments")
video = get_video(id, db, region: region)
session_token = video.session_token
case cursor
when nil, ""
ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by)
# when .starts_with? "Ug"
# ctoken = produce_comment_reply_continuation(id, video.ucid, cursor)
when .starts_with? "ADSJ"
ctoken = produce_comment_continuation(id, cursor: cursor, sort_by: sort_by)
else
ctoken = cursor
end
if !session_token
if format == "json"
return {"comments" => [] of String}.to_json
else
return {"contentHtml" => "", "commentCount" => 0}.to_json
end
end
post_req = {
page_token: ctoken,
session_token: session_token,
}
headers = HTTP::Headers{
"cookie" => video.cookie,
}
response = YT_POOL.client(region, &.post("/comment_service_ajax?#{action}=1&hl=en&gl=US&pbj=1", headers, form: post_req))
response = JSON.parse(response.body)
# For some reason youtube puts it in an array for comment_replies but otherwise it's the same
if action == "action_get_comment_replies"
response = response[1]
end
if !response["response"]["continuationContents"]?
raise InfoException.new("Could not fetch comments")
end
response = response["response"]["continuationContents"]
if response["commentRepliesContinuation"]?
body = response["commentRepliesContinuation"]
else
body = response["itemSectionContinuation"]
end
contents = body["contents"]?
if !contents
if format == "json"
return {"comments" => [] of String}.to_json
else
return {"contentHtml" => "", "commentCount" => 0}.to_json
end
end
response = JSON.build do |json|
json.object do
if body["header"]?
count_text = body["header"]["commentsHeaderRenderer"]["countText"]
comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i? || 0
json.field "commentCount", comment_count
end
json.field "videoId", id
json.field "comments" do
json.array do
contents.as_a.each do |node|
json.object do
if !response["commentRepliesContinuation"]?
node = node["commentThreadRenderer"]
end
if node["replies"]?
node_replies = node["replies"]["commentRepliesRenderer"]
end
if !response["commentRepliesContinuation"]?
node_comment = node["comment"]["commentRenderer"]
else
node_comment = node["commentRenderer"]
end
content_html = node_comment["contentText"]?.try { |t| parse_content(t) } || ""
author = node_comment["authorText"]?.try &.["simpleText"]? || ""
json.field "author", author
json.field "authorThumbnails" do
json.array do
node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail|
json.object do
json.field "url", thumbnail["url"]
json.field "width", thumbnail["width"]
json.field "height", thumbnail["height"]
end
end
end
end
if node_comment["authorEndpoint"]?
json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"]
else
json.field "authorId", ""
json.field "authorUrl", ""
end
published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s
published = decode_date(published_text.rchop(" (edited)"))
if published_text.includes?(" (edited)")
json.field "isEdited", true
else
json.field "isEdited", false
end
json.field "content", html_to_content(content_html)
json.field "contentHtml", content_html
json.field "published", published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
comment_action_buttons_renderer = node_comment["actionButtons"]["commentActionButtonsRenderer"]
json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i
json.field "commentId", node_comment["commentId"]
json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"]
if comment_action_buttons_renderer["creatorHeart"]?
hearth_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"]
json.field "creatorHeart" do
json.object do
json.field "creatorThumbnail", hearth_data["thumbnails"][-1]["url"]
json.field "creatorName", hearth_data["accessibility"]["accessibilityData"]["label"]
end
end
end
if node_replies && !response["commentRepliesContinuation"]?
if node_replies["moreText"]?
reply_count = (node_replies["moreText"]["simpleText"]? || node_replies["moreText"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i? || 1
elsif node_replies["viewReplies"]?
reply_count = node_replies["viewReplies"]["buttonRenderer"]["text"]?.try &.["runs"][1]?.try &.["text"]?.try &.as_s.to_i? || 1
else
reply_count = 1
end
continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
continuation ||= ""
json.field "replies" do
json.object do
json.field "replyCount", reply_count
json.field "continuation", continuation
end
end
end
end
end
end
end
if body["continuations"]?
continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s
json.field "continuation", continuation
end
end
end
if format == "html"
response = JSON.parse(response)
content_html = template_youtube_comments(response, locale, thin_mode, action == "action_get_comment_replies")
response = JSON.build do |json|
json.object do
json.field "contentHtml", content_html
if response["commentCount"]?
json.field "commentCount", response["commentCount"]
else
json.field "commentCount", 0
end
end
end
end
return response
end
def fetch_reddit_comments(id, sort_by = "confidence")
client = make_client(REDDIT_URL)
headers = HTTP::Headers{"User-Agent" => "web:invidious:v#{CURRENT_VERSION} (by github.com/iv-org/invidious)"}
# TODO: Use something like #479 for a static list of instances to use here
query = "(url:3D#{id}%20OR%20url:#{id})%20(site:invidio.us%20OR%20site:youtube.com%20OR%20site:youtu.be)"
search_results = client.get("/search.json?q=#{query}", headers)
if search_results.status_code == 200
search_results = RedditThing.from_json(search_results.body)
# For videos that have more than one thread, choose the one with the highest score
thread = search_results.data.as(RedditListing).children.sort_by { |child| child.data.as(RedditLink).score }[-1]
thread = thread.data.as(RedditLink)
result = client.get("/r/#{thread.subreddit}/comments/#{thread.id}.json?limit=100&sort=#{sort_by}", headers).body
result = Array(RedditThing).from_json(result)
elsif search_results.status_code == 302
# Previously, if there was only one result then the API would redirect to that result.
# Now, it appears it will still return a listing so this section is likely unnecessary.
result = client.get(search_results.headers["Location"], headers).body
result = Array(RedditThing).from_json(result)
thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink)
else
raise InfoException.new("Could not fetch comments")
end
client.close
comments = result[1].data.as(RedditListing).children
return comments, thread
end
def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
String.build do |html|
root = comments["comments"].as_a
root.each do |child|
if child["replies"]?
replies_html = <<-END_HTML
<div id="replies" class="pure-g">
<div class="pure-u-1-24"></div>
<div class="pure-u-23-24">
<p>
<a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}"
data-onclick="get_youtube_replies" data-load-replies>#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a>
</p>
</div>
</div>
END_HTML
end
if !thin_mode
author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}"
else
author_thumbnail = ""
end
html << <<-END_HTML
<div class="pure-g" style="width:100%">
<div class="channel-profile pure-u-4-24 pure-u-md-2-24">
<img style="padding-right:1em;padding-top:1em;width:90%" src="#{author_thumbnail}">
</div>
<div class="pure-u-20-24 pure-u-md-22-24">
<p>
<b>
<a class="#{child["authorIsChannelOwner"] == true ? "channel-owner" : ""}" href="#{child["authorUrl"]}">#{child["author"]}</a>
</b>
<p style="white-space:pre-wrap">#{child["contentHtml"]}</p>
END_HTML
if child["attachment"]?
attachment = child["attachment"]
case attachment["type"]
when "image"
attachment = attachment["imageThumbnails"][1]
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-2">
<img style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}">
</div>
</div>
END_HTML
when "video"
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-2">
<div style="position:relative;width:100%;height:0;padding-bottom:56.25%;margin-bottom:5px">
END_HTML
if attachment["error"]?
html << <<-END_HTML
<p>#{attachment["error"]}</p>
END_HTML
else
html << <<-END_HTML
<iframe id='ivplayer' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' style='border:none;'></iframe>
END_HTML
end
html << <<-END_HTML
</div>
</div>
</div>
END_HTML
else nil # Ignore
end
end
html << <<-END_HTML
<span title="#{Time.unix(child["published"].as_i64).to_s(translate(locale, "%A %B %-d, %Y"))}">#{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}</span>
|
END_HTML
if comments["videoId"]?
html << <<-END_HTML
<a href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a>
|
END_HTML
elsif comments["authorId"]?
html << <<-END_HTML
<a href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a>
|
END_HTML
end
html << <<-END_HTML
<i class="icon ion-ios-thumbs-up"></i> #{number_with_separator(child["likeCount"])}
END_HTML
if child["creatorHeart"]?
if !thin_mode
creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}"
else
creator_thumbnail = ""
end
html << <<-END_HTML
<span class="creator-heart-container" title="#{translate(locale, "`x` marked it with a ❤", child["creatorHeart"]["creatorName"].as_s)}">
<div class="creator-heart">
<img class="creator-heart-background-hearted" src="#{creator_thumbnail}"></img>
<div class="creator-heart-small-hearted">
<div class="icon ion-ios-heart creator-heart-small-container"></div>
</div>
</div>
</span>
END_HTML
end
html << <<-END_HTML
</p>
#{replies_html}
</div>
</div>
END_HTML
end
if comments["continuation"]?
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1">
<p>
<a href="javascript:void(0)" data-continuation="#{comments["continuation"]}"
data-onclick="get_youtube_replies" data-load-more #{"data-load-replies" if is_replies}>#{translate(locale, "Load more")}</a>
</p>
</div>
</div>
END_HTML
end
end
end
def template_reddit_comments(root, locale)
String.build do |html|
root.each do |child|
if child.data.is_a?(RedditComment)
child = child.data.as(RedditComment)
body_html = HTML.unescape(child.body_html)
replies_html = ""
if child.replies.is_a?(RedditThing)
replies = child.replies.as(RedditThing)
replies_html = template_reddit_comments(replies.data.as(RedditListing).children, locale)
end
if child.depth > 0
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1-24">
</div>
<div class="pure-u-23-24">
END_HTML
else
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1">
END_HTML
end
html << <<-END_HTML
<p>
<a href="javascript:void(0)" data-onclick="toggle_parent">[ - ]</a>
<b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b>
#{translate(locale, "`x` points", number_with_separator(child.score))}
<span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span>
<a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a>
</p>
<div>
#{body_html}
#{replies_html}
</div>
</div>
</div>
END_HTML
end
end
end
end
def replace_links(html)
html = XML.parse_html(html)
html.xpath_nodes(%q(//a)).each do |anchor|
url = URI.parse(anchor["href"])
if {"www.youtube.com", "m.youtube.com", "youtu.be"}.includes?(url.host)
if url.path == "/redirect"
params = HTTP::Params.parse(url.query.not_nil!)
anchor["href"] = params["q"]?
else
anchor["href"] = url.request_target
end
elsif url.to_s == "#"
begin
length_seconds = decode_length_seconds(anchor.content)
rescue ex
length_seconds = decode_time(anchor.content)
end
if length_seconds > 0
anchor["href"] = "javascript:void(0)"
anchor["onclick"] = "player.currentTime(#{length_seconds})"
else
anchor["href"] = url.request_target
end
end
end
html = html.xpath_node(%q(//body)).not_nil!
if node = html.xpath_node(%q(./p))
html = node
end
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
def fill_links(html, scheme, host)
html = XML.parse_html(html)
html.xpath_nodes("//a").each do |match|
url = URI.parse(match["href"])
# Reddit links don't have host
if !url.host && !match["href"].starts_with?("javascript") && !url.to_s.ends_with? "#"
url.scheme = scheme
url.host = host
match["href"] = url
end
end
if host == "www.youtube.com"
html = html.xpath_node(%q(//body/p)).not_nil!
end
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
def parse_content(content : JSON::Any) : String
content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || ""
end
def content_to_comment_html(content)
comment_html = content.map do |run|
text = HTML.escape(run["text"].as_s).gsub("\n", "<br>")
if run["bold"]?
text = "<b>#{text}</b>"
end
if run["italics"]?
text = "<i>#{text}</i>"
end
if run["navigationEndpoint"]?
if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s
url = URI.parse(url)
if !url.host || {"m.youtube.com", "www.youtube.com", "youtu.be"}.includes? url.host
if url.path == "/redirect"
url = HTTP::Params.parse(url.query.not_nil!)["q"]
else
url = url.request_target
end
end
text = %(<a href="#{url}">#{text}</a>)
elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]?
length_seconds = watch_endpoint["startTimeSeconds"]?
video_id = watch_endpoint["videoId"].as_s
if length_seconds && length_seconds.as_i > 0
text = %(<a href="javascript:void(0)" data-onclick="jump_to_time" data-jump-time="#{length_seconds}">#{text}</a>)
else
text = %(<a href="/watch?v=#{video_id}">#{text}</a>)
end
elsif url = run["navigationEndpoint"]["commandMetadata"]?.try &.["webCommandMetadata"]["url"].as_s
text = %(<a href="#{url}">#{text}</a>)
end
end
text
end.join("").delete('\ufeff')
return comment_html
end
def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
object = {
"2:embedded" => {
"2:string" => video_id,
"25:varint" => 0_i64,
"28:varint" => 1_i64,
"36:embedded" => {
"5:varint" => -1_i64,
"8:varint" => 0_i64,
},
"40:embedded" => {
"1:varint" => 4_i64,
"3:string" => "https://www.youtube.com",
"4:string" => "",
},
},
"3:varint" => 6_i64,
"6:embedded" => {
"1:string" => cursor,
"4:embedded" => {
"4:string" => video_id,
"6:varint" => 0_i64,
},
"5:varint" => 20_i64,
},
}
case sort_by
when "top"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
when "new", "newest"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
else # top
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
end
continuation = object.try { |i| Protodec::Any.cast_json(object) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def produce_comment_reply_continuation(video_id, ucid, comment_id)
object = {
"2:embedded" => {
"2:string" => video_id,
"24:varint" => 1_i64,
"25:varint" => 1_i64,
"28:varint" => 1_i64,
"36:embedded" => {
"5:varint" => -1_i64,
"8:varint" => 0_i64,
},
},
"3:varint" => 6_i64,
"6:embedded" => {
"3:embedded" => {
"2:string" => comment_id,
"4:embedded" => {
"1:varint" => 0_i64,
},
"5:string" => ucid,
"6:string" => video_id,
"8:varint" => 1_i64,
"9:varint" => 10_i64,
},
},
}
continuation = object.try { |i| Protodec::Any.cast_json(object) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end

89
src/invidious/comments/content.cr Normal file
View File

@@ -0,0 +1,89 @@
def text_to_parsed_content(text : String) : JSON::Any
nodes = [] of JSON::Any
# For each line convert line to array of nodes
text.split('\n').each do |line|
# In first case line is just a simple node before
# check patterns inside line
# { 'text': line }
current_nodes = [] of JSON::Any
initial_node = {"text" => line}
current_nodes << (JSON.parse(initial_node.to_json))
# For each match with url pattern, get last node and preserve
# last node before create new node with url information
# { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } }
line.scan(/https?:\/\/[^ ]*/).each do |url_match|
# Retrieve last node and update node without match
last_node = current_nodes[-1].as_h
splitted_last_node = last_node["text"].as_s.split(url_match[0])
last_node["text"] = JSON.parse(splitted_last_node[0].to_json)
current_nodes[-1] = JSON.parse(last_node.to_json)
# Create new node with match and navigation infos
current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}}
current_nodes << (JSON.parse(current_node.to_json))
# If text remain after match create new simple node with text after match
after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""}
current_nodes << (JSON.parse(after_node.to_json))
end
# After processing of matches inside line
# Add \n at end of last node for preserve carriage return
last_node = current_nodes[-1].as_h
last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json)
current_nodes[-1] = JSON.parse(last_node.to_json)
# Finally add final nodes to nodes returned
current_nodes.each do |node|
nodes << (node)
end
end
return JSON.parse({"runs" => nodes}.to_json)
end
def parse_content(content : JSON::Any, video_id : String? = "") : String
content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "<br>") } || ""
end
def content_to_comment_html(content, video_id : String? = "")
html_array = content.map do |run|
# Sometimes, there is an empty element.
# See: https://github.com/iv-org/invidious/issues/3096
next if run.as_h.empty?
text = HTML.escape(run["text"].as_s)
if navigation_endpoint = run.dig?("navigationEndpoint")
text = parse_link_endpoint(navigation_endpoint, text, video_id)
end
text = "<b>#{text}</b>" if run["bold"]?
text = "<s>#{text}</s>" if run["strikethrough"]?
text = "<i>#{text}</i>" if run["italics"]?
# check for custom emojis
if run["emoji"]?
if run["emoji"]["isCustomEmoji"]?.try &.as_bool
if emoji_image = run.dig?("emoji", "image")
emoji_alt = emoji_image.dig?("accessibility", "accessibilityData", "label").try &.as_s || text
emoji_thumb = emoji_image["thumbnails"][0]
text = String.build do |str|
str << %(<img alt=") << emoji_alt << "\" "
str << %(src="/ggpht) << URI.parse(emoji_thumb["url"].as_s).request_target << "\" "
str << %(title=") << emoji_alt << "\" "
str << %(width=") << emoji_thumb["width"] << "\" "
str << %(height=") << emoji_thumb["height"] << "\" "
str << %(class="channel-emoji" />)
end
else
# Hide deleted channel emoji
text = ""
end
end
end
text
end
return html_array.join("").delete('\ufeff')
end

76
src/invidious/comments/links_util.cr Normal file
View File

@@ -0,0 +1,76 @@
module Invidious::Comments
extend self
def replace_links(html)
# Check if the document is empty
# Prevents edge-case bug with Reddit comments, see issue #3115
if html.nil? || html.empty?
return html
end
html = XML.parse_html(html)
html.xpath_nodes(%q(//a)).each do |anchor|
url = URI.parse(anchor["href"])
if url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") || url.host.not_nil!.ends_with?("youtu.be")
if url.host.try &.ends_with? "youtu.be"
url = "/watch?v=#{url.path.lstrip('/')}#{url.query_params}"
else
if url.path == "/redirect"
params = HTTP::Params.parse(url.query.not_nil!)
anchor["href"] = params["q"]?
else
anchor["href"] = url.request_target
end
end
elsif url.to_s == "#"
begin
length_seconds = decode_length_seconds(anchor.content)
rescue ex
length_seconds = decode_time(anchor.content)
end
if length_seconds > 0
anchor["href"] = "javascript:void(0)"
anchor["onclick"] = "player.currentTime(#{length_seconds})"
else
anchor["href"] = url.request_target
end
end
end
html = html.xpath_node(%q(//body)).not_nil!
if node = html.xpath_node(%q(./p))
html = node
end
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
def fill_links(html, scheme, host)
# Check if the document is empty
# Prevents edge-case bug with Reddit comments, see issue #3115
if html.nil? || html.empty?
return html
end
html = XML.parse_html(html)
html.xpath_nodes("//a").each do |match|
url = URI.parse(match["href"])
# Reddit links don't have host
if !url.host && !match["href"].starts_with?("javascript") && !url.to_s.ends_with? "#"
url.scheme = scheme
url.host = host
match["href"] = url
end
end
if host == "www.youtube.com"
html = html.xpath_node(%q(//body/p)).not_nil!
end
return html.to_xml(options: XML::SaveOptions::NO_DECL)
end
end

41
src/invidious/comments/reddit.cr Normal file
View File

@@ -0,0 +1,41 @@
module Invidious::Comments
extend self
def fetch_reddit(id, sort_by = "confidence")
client = make_client(REDDIT_URL)
headers = HTTP::Headers{"User-Agent" => "web:invidious:v#{CURRENT_VERSION} (by github.com/iv-org/invidious)"}
# TODO: Use something like #479 for a static list of instances to use here
query = URI::Params.encode({q: "(url:3D#{id} OR url:#{id}) AND (site:invidio.us OR site:youtube.com OR site:youtu.be)"})
search_results = client.get("/search.json?#{query}", headers)
if search_results.status_code == 200
search_results = RedditThing.from_json(search_results.body)
# For videos that have more than one thread, choose the one with the highest score
threads = search_results.data.as(RedditListing).children
thread = threads.max_by?(&.data.as(RedditLink).score).try(&.data.as(RedditLink))
result = thread.try do |t|
body = client.get("/r/#{t.subreddit}/comments/#{t.id}.json?limit=100&sort=#{sort_by}", headers).body
Array(RedditThing).from_json(body)
end
result ||= [] of RedditThing
elsif search_results.status_code == 302
# Previously, if there was only one result then the API would redirect to that result.
# Now, it appears it will still return a listing so this section is likely unnecessary.
result = client.get(search_results.headers["Location"], headers).body
result = Array(RedditThing).from_json(result)
thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink)
else
raise NotFoundException.new("Comments not found.")
end
client.close
comments = result[1]?.try(&.data.as(RedditListing).children)
comments ||= [] of RedditThing
return comments, thread
end
end

57
src/invidious/comments/reddit_types.cr Normal file
View File

@@ -0,0 +1,57 @@
class RedditThing
include JSON::Serializable
property kind : String
property data : RedditComment | RedditLink | RedditMore | RedditListing
end
class RedditComment
include JSON::Serializable
property author : String
property body_html : String
property replies : RedditThing | String
property score : Int32
property depth : Int32
property permalink : String
@[JSON::Field(converter: RedditComment::TimeConverter)]
property created_utc : Time
module TimeConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix(value.read_float.to_i)
end
def self.to_json(value : Time, json : JSON::Builder)
json.number(value.to_unix)
end
end
end
struct RedditLink
include JSON::Serializable
property author : String
property score : Int32
property subreddit : String
property num_comments : Int32
property id : String
property permalink : String
property title : String
end
struct RedditMore
include JSON::Serializable
property children : Array(String)
property count : Int32
property depth : Int32
end
class RedditListing
include JSON::Serializable
property children : Array(RedditThing)
property modhash : String
end

365
src/invidious/comments/youtube.cr Normal file
View File

@@ -0,0 +1,365 @@
module Invidious::Comments
extend self
def fetch_youtube(id, cursor, format, locale, thin_mode, region, sort_by = "top")
case cursor
when nil, ""
ctoken = Comments.produce_continuation(id, cursor: "", sort_by: sort_by)
when .starts_with? "ADSJ"
ctoken = Comments.produce_continuation(id, cursor: cursor, sort_by: sort_by)
else
ctoken = cursor
end
client_config = YoutubeAPI::ClientConfig.new(region: region)
response = YoutubeAPI.next(continuation: ctoken, client_config: client_config)
return parse_youtube(id, response, format, locale, thin_mode, sort_by)
end
def fetch_community_post_comments(ucid, post_id)
object = {
"2:string" => "community",
"25:embedded" => {
"22:string" => post_id,
},
"45:embedded" => {
"2:varint" => 1_i64,
"3:varint" => 1_i64,
},
"53:embedded" => {
"4:embedded" => {
"6:varint" => 0_i64,
"27:varint" => 1_i64,
"29:string" => post_id,
"30:string" => ucid,
},
"8:string" => "comments-section",
},
}
object_parsed = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
object2 = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => object_parsed,
},
}
continuation = object2.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
initial_data = YoutubeAPI.browse(continuation: continuation)
return initial_data
end
def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false)
contents = nil
if on_response_received_endpoints = response["onResponseReceivedEndpoints"]?
header = nil
on_response_received_endpoints.as_a.each do |item|
if item["reloadContinuationItemsCommand"]?
case item["reloadContinuationItemsCommand"]["slot"]
when "RELOAD_CONTINUATION_SLOT_HEADER"
header = item["reloadContinuationItemsCommand"]["continuationItems"][0]
when "RELOAD_CONTINUATION_SLOT_BODY"
# continuationItems is nil when video has no comments
contents = item["reloadContinuationItemsCommand"]["continuationItems"]?
end
elsif item["appendContinuationItemsAction"]?
contents = item["appendContinuationItemsAction"]["continuationItems"]
end
end
elsif response["continuationContents"]?
response = response["continuationContents"]
if response["commentRepliesContinuation"]?
body = response["commentRepliesContinuation"]
else
body = response["itemSectionContinuation"]
end
contents = body["contents"]?
header = body["header"]?
else
raise NotFoundException.new("Comments not found.")
end
if !contents
if format == "json"
return {"comments" => [] of String}.to_json
else
return {"contentHtml" => "", "commentCount" => 0}.to_json
end
end
continuation_item_renderer = nil
contents.as_a.reject! do |item|
if item["continuationItemRenderer"]?
continuation_item_renderer = item["continuationItemRenderer"]
true
end
end
mutations = response.dig?("frameworkUpdates", "entityBatchUpdate", "mutations").try &.as_a || [] of JSON::Any
response = JSON.build do |json|
json.object do
if header
count_text = header["commentsHeaderRenderer"]["countText"]
comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i? || 0
json.field "commentCount", comment_count
end
if is_post
json.field "postId", id
else
json.field "videoId", id
end
json.field "comments" do
json.array do
contents.as_a.each do |node|
json.object do
if node["commentThreadRenderer"]?
node = node["commentThreadRenderer"]
end
if node["replies"]?
node_replies = node["replies"]["commentRepliesRenderer"]
end
if cvm = node["commentViewModel"]?
# two commentViewModels for inital request
# one commentViewModel when getting a replies to a comment
cvm = cvm["commentViewModel"] if cvm["commentViewModel"]?
comment_key = cvm["commentKey"]
toolbar_key = cvm["toolbarStateKey"]
comment_mutation = mutations.find { |i| i.dig?("payload", "commentEntityPayload", "key") == comment_key }
toolbar_mutation = mutations.find { |i| i.dig?("entityKey") == toolbar_key }
if !comment_mutation.nil? && !toolbar_mutation.nil?
# todo parse styleRuns, commandRuns and attachmentRuns for comments
html_content = parse_description(comment_mutation.dig("payload", "commentEntityPayload", "properties", "content"), id)
comment_author = comment_mutation.dig("payload", "commentEntityPayload", "author")
json.field "authorId", comment_author["channelId"].as_s
json.field "authorUrl", "/channel/#{comment_author["channelId"].as_s}"
json.field "author", comment_author["displayName"].as_s
json.field "verified", comment_author["isVerified"].as_bool
json.field "authorThumbnails" do
json.array do
comment_mutation.dig?("payload", "commentEntityPayload", "avatar", "image", "sources").try &.as_a.each do |thumbnail|
json.object do
json.field "url", thumbnail["url"]
json.field "width", thumbnail["width"]
json.field "height", thumbnail["height"]
end
end
end
end
json.field "authorIsChannelOwner", comment_author["isCreator"].as_bool
json.field "isSponsor", (comment_author["sponsorBadgeUrl"]? != nil)
if sponsor_badge_url = comment_author["sponsorBadgeUrl"]?
# Sponsor icon thumbnails always have one object and there's only ever the url property in it
json.field "sponsorIconUrl", sponsor_badge_url
end
comment_toolbar = comment_mutation.dig("payload", "commentEntityPayload", "toolbar")
json.field "likeCount", short_text_to_number(comment_toolbar["likeCountNotliked"].as_s)
reply_count = short_text_to_number(comment_toolbar["replyCount"]?.try &.as_s || "0")
if heart_state = toolbar_mutation.dig?("payload", "engagementToolbarStateEntityPayload", "heartState")
if heart_state.as_s == "TOOLBAR_HEART_STATE_HEARTED"
json.field "creatorHeart" do
json.object do
json.field "creatorThumbnail", comment_toolbar["creatorThumbnailUrl"].as_s
json.field "creatorName", comment_toolbar["heartActiveTooltip"].as_s.sub("❤ by ", "")
end
end
end
end
published_text = comment_mutation.dig?("payload", "commentEntityPayload", "properties", "publishedTime").try &.as_s
end
json.field "isPinned", (cvm.dig?("pinnedText") != nil)
json.field "commentId", cvm["commentId"]
else
if node["comment"]?
node_comment = node["comment"]["commentRenderer"]
else
node_comment = node["commentRenderer"]
end
json.field "commentId", node_comment["commentId"]
html_content = node_comment["contentText"]?.try { |t| parse_content(t, id) }
json.field "verified", (node_comment["authorCommentBadge"]? != nil)
json.field "author", node_comment["authorText"]?.try &.["simpleText"]? || ""
json.field "authorThumbnails" do
json.array do
node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail|
json.object do
json.field "url", thumbnail["url"]
json.field "width", thumbnail["width"]
json.field "height", thumbnail["height"]
end
end
end
end
if comment_action_buttons_renderer = node_comment.dig?("actionButtons", "commentActionButtonsRenderer")
json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i
if comment_action_buttons_renderer["creatorHeart"]?
heart_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"]
json.field "creatorHeart" do
json.object do
json.field "creatorThumbnail", heart_data["thumbnails"][-1]["url"]
json.field "creatorName", heart_data["accessibility"]["accessibilityData"]["label"]
end
end
end
end
if node_comment["authorEndpoint"]?
json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"]
else
json.field "authorId", ""
json.field "authorUrl", ""
end
json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"]
json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil)
published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s
json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil)
if node_comment["sponsorCommentBadge"]?
# Sponsor icon thumbnails always have one object and there's only ever the url property in it
json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s
end
reply_count = node_comment["replyCount"]?
end
content_html = html_content || ""
json.field "content", html_to_content(content_html)
json.field "contentHtml", content_html
if published_text != nil
published_text = published_text.to_s
if published_text.includes?(" (edited)")
json.field "isEdited", true
published = decode_date(published_text.rchop(" (edited)"))
else
json.field "isEdited", false
published = decode_date(published_text)
end
json.field "published", published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
end
if node_replies && !response["commentRepliesContinuation"]?
if node_replies["continuations"]?
continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
elsif node_replies["contents"]?
continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
end
continuation ||= ""
json.field "replies" do
json.object do
json.field "replyCount", reply_count || 1
json.field "continuation", continuation
end
end
end
end
end
end
end
if continuation_item_renderer
if continuation_item_renderer["continuationEndpoint"]?
continuation_endpoint = continuation_item_renderer["continuationEndpoint"]
elsif continuation_item_renderer["button"]?
continuation_endpoint = continuation_item_renderer["button"]["buttonRenderer"]["command"]
end
if continuation_endpoint
json.field "continuation", continuation_endpoint["continuationCommand"]["token"].as_s
end
end
end
end
if format == "html"
response = JSON.parse(response)
content_html = Frontend::Comments.template_youtube(response, locale, thin_mode)
response = JSON.build do |json|
json.object do
json.field "contentHtml", content_html
if response["commentCount"]?
json.field "commentCount", response["commentCount"]
else
json.field "commentCount", 0
end
end
end
end
return response
end
def produce_continuation(video_id, cursor = "", sort_by = "top")
object = {
"2:embedded" => {
"2:string" => video_id,
"25:varint" => 0_i64,
"28:varint" => 1_i64,
"36:embedded" => {
"5:varint" => -1_i64,
"8:varint" => 0_i64,
},
"40:embedded" => {
"1:varint" => 4_i64,
"3:string" => "https://www.youtube.com",
"4:string" => "",
},
},
"3:varint" => 6_i64,
"6:embedded" => {
"1:string" => cursor,
"4:embedded" => {
"4:string" => video_id,
"6:varint" => 0_i64,
},
"5:varint" => 20_i64,
},
}
case sort_by
when "top"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
when "new", "newest"
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
else # top
object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
end
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
end

256
src/invidious/config.cr Normal file
View File

@@ -0,0 +1,256 @@
struct DBConfig
include YAML::Serializable
property user : String
property password : String
property host : String
property port : Int32
property dbname : String
end
struct ConfigPreferences
include YAML::Serializable
property annotations : Bool = false
property annotations_subscribed : Bool = false
property preload : Bool = true
property autoplay : Bool = false
property captions : Array(String) = ["", "", ""]
property comments : Array(String) = ["youtube", ""]
property continue : Bool = false
property continue_autoplay : Bool = true
property dark_mode : String = ""
property latest_only : Bool = false
property listen : Bool = false
property local : Bool = false
property locale : String = "en-US"
property watch_history : Bool = true
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = "invidious"
property quality : String = "hd720"
property quality_dash : String = "auto"
property default_home : String? = "Popular"
property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
property automatic_instance_redirect : Bool = false
property region : String = "US"
property related_videos : Bool = true
property sort : String = "published"
property speed : Float32 = 1.0_f32
property thin_mode : Bool = false
property unseen_only : Bool = false
property video_loop : Bool = false
property extend_desc : Bool = false
property volume : Int32 = 100
property vr_mode : Bool = true
property show_nick : Bool = true
property save_player_pos : Bool = false
def to_tuple
{% begin %}
{
{{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}}
}
{% end %}
end
end
struct HTTPProxyConfig
include YAML::Serializable
property user : String
property password : String
property host : String
property port : Int32
end
class Config
include YAML::Serializable
# Number of threads to use for crawling videos from channels (for updating subscriptions)
property channel_threads : Int32 = 1
# Time interval between two executions of the job that crawls channel videos (subscriptions update).
@[YAML::Field(converter: Preferences::TimeSpanConverter)]
property channel_refresh_interval : Time::Span = 30.minutes
# Number of threads to use for updating feeds
property feed_threads : Int32 = 1
# Log file path or STDOUT
property output : String = "STDOUT"
# Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
property log_level : LogLevel = LogLevel::Info
# Enables colors in logs. Useful for debugging purposes
property colorize_logs : Bool = false
# Database configuration with separate parameters (username, hostname, etc)
property db : DBConfig? = nil
# Database configuration using 12-Factor "Database URL" syntax
@[YAML::Field(converter: Preferences::URIConverter)]
property database_url : URI = URI.parse("")
# Used for crawling channels: threads should check all videos uploaded by a channel
property full_refresh : Bool = false
# Jobs config structure. See jobs.cr and jobs/base_job.cr
property jobs = Invidious::Jobs::JobsConfig.new
# Used to tell Invidious it is behind a proxy, so links to resources should be https://
property https_only : Bool?
# HMAC signing key for CSRF tokens and verifying pubsub subscriptions
property hmac_key : String = ""
# Domain to be used for links to resources on the site where an absolute URL is required
property domain : String?
# Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
property use_pubsub_feeds : Bool | Int32 = false
property popular_enabled : Bool = true
property captcha_enabled : Bool = true
property login_enabled : Bool = true
property registration_enabled : Bool = true
property statistics_enabled : Bool = false
property admins : Array(String) = [] of String
property external_port : Int32? = nil
property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("")
# For compliance with DMCA, disables download widget using list of video IDs
property dmca_content : Array(String) = [] of String
# Check table integrity, automatically try to add any missing columns, create tables, etc.
property check_tables : Bool = false
# Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
property cache_annotations : Bool = false
# Optional banner to be displayed along top of page for announcements, etc.
property banner : String? = nil
# Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
property hsts : Bool? = true
# Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
property disable_proxy : Bool? | Array(String)? = false
# Enable the user notifications for all users
property enable_user_notifications : Bool = true
# URL to the modified source code to be easily AGPL compliant
# Will display in the footer, next to the main source code link
property modified_source_code_url : String? = nil
# Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC
# External signature solver server socket (either a path to a UNIX domain socket or "<IP>:<Port>")
property signature_server : String? = nil
# Port to listen for connections (overridden by command line argument)
property port : Int32 = 3000
# Host to bind (overridden by command line argument)
property host_binding : String = "0.0.0.0"
# Make Invidious listening on UNIX sockets - Example: /tmp/invidious.sock
property bind_unix : String? = nil
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100
# HTTP Proxy configuration
property http_proxy : HTTPProxyConfig? = nil
# Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false
# visitor data ID for Google session
property visitor_data : String? = nil
# poToken for passing bot attestation
property po_token : String? = nil
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new
# Playlist length limit
property playlist_length_limit : Int32 = 500
def disabled?(option)
case disabled = CONFIG.disable_proxy
when Bool
return disabled
when Array
if disabled.includes? option
return true
else
return false
end
else
return false
end
end
def self.load
# Load config from file or YAML string env var
env_config_file = "INVIDIOUS_CONFIG_FILE"
env_config_yaml = "INVIDIOUS_CONFIG"
config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
config = Config.from_yaml(config_yaml)
# Update config from env vars (upcased and prefixed with "INVIDIOUS_")
{% for ivar in Config.instance_vars %}
{% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
if ENV.has_key?({{env_id}})
env_value = ENV.fetch({{env_id}})
success = false
# Use YAML converter if specified
{% ann = ivar.annotation(::YAML::Field) %}
{% if ann && ann[:converter] %}
config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
success = true
# Use regular YAML parser otherwise
{% else %}
{% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %}
# Sort types to avoid parsing nulls and numbers as strings
{% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %}
{{ivar_types}}.each do |ivar_type|
if !success
begin
config.{{ivar.id}} = ivar_type.from_yaml(env_value)
success = true
rescue
# nop
end
end
end
{% end %}
# Exit on fail
if !success
puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}})
exit(1)
end
end
{% end %}
# HMAC_key is mandatory
# See: https://github.com/iv-org/invidious/issues/3854
if config.hmac_key.empty?
puts "Config: 'hmac_key' is required/can't be empty"
exit(1)
elsif config.hmac_key == "CHANGE_ME!!"
puts "Config: The value of 'hmac_key' needs to be changed!!"
exit(1)
end
# Build database_url from db.* if it's not set directly
if config.database_url.to_s.empty?
if db = config.db
config.database_url = URI.new(
scheme: "postgres",
user: db.user,
password: db.password,
host: db.host,
port: db.port,
path: db.dbname,
)
else
puts "Config: Either database_url or db.* is required"
exit(1)
end
end
return config
end
end

24
src/invidious/database/annotations.cr Normal file
View File

@@ -0,0 +1,24 @@
require "./base.cr"
module Invidious::Database::Annotations
extend self
def insert(id : String, annotations : String)
request = <<-SQL
INSERT INTO annotations
VALUES ($1, $2)
ON CONFLICT DO NOTHING
SQL
PG_DB.exec(request, id, annotations)
end
def select(id : String) : Annotation?
request = <<-SQL
SELECT * FROM annotations
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: Annotation)
end
end

136
src/invidious/database/base.cr Normal file
View File

@@ -0,0 +1,136 @@
require "pg"
module Invidious::Database
extend self
# Checks table integrity
#
# Note: config is passed as a parameter to avoid complex
# dependencies between different parts of the software.
def check_integrity(cfg)
return if !cfg.check_tables
Invidious::Database.check_enum("privacy", PlaylistPrivacy)
Invidious::Database.check_table("channels", InvidiousChannel)
Invidious::Database.check_table("channel_videos", ChannelVideo)
Invidious::Database.check_table("playlists", InvidiousPlaylist)
Invidious::Database.check_table("playlist_videos", PlaylistVideo)
Invidious::Database.check_table("nonces", Nonce)
Invidious::Database.check_table("session_ids", SessionId)
Invidious::Database.check_table("users", User)
Invidious::Database.check_table("videos", Video)
if cfg.cache_annotations
Invidious::Database.check_table("annotations", Annotation)
end
end
#
# Table/enum integrity checks
#
def check_enum(enum_name, struct_type = nil)
return # TODO
if !PG_DB.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
LOGGER.info("check_enum: CREATE TYPE #{enum_name}")
PG_DB.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql"))
end
end
end
def check_table(table_name, struct_type = nil)
# Create table if it doesn't exist
begin
PG_DB.exec("SELECT * FROM #{table_name} LIMIT 0")
rescue ex
LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}")
PG_DB.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql"))
end
end
return if !struct_type
struct_array = struct_type.type_array
column_array = get_column_array(PG_DB, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT")
return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
if !column_array[i]?
new_column = column_types.select(&.starts_with?(name))[0]
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
PG_DB.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
next
end
# Column doesn't exist
if !column_array.includes? name
new_column = column_types.select(&.starts_with?(name))[0]
PG_DB.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
end
# Column exists but in the wrong position, rotate
if struct_array.includes? column_array[i]
until name == column_array[i]
new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new")
# There's a column we didn't expect
if !new_column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}")
PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
column_array = get_column_array(PG_DB, table_name)
next
end
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
PG_DB.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
PG_DB.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
PG_DB.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
column_array = get_column_array(PG_DB, table_name)
end
else
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
end
end
end
return if column_array.size <= struct_array.size
column_array.each do |column|
if !struct_array.includes? column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
PG_DB.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
end
end
end
def get_column_array(db, table_name)
column_array = [] of String
PG_DB.query("SELECT * FROM #{table_name} LIMIT 0") do |rs|
rs.column_count.times do |i|
column = rs.as(PG::ResultSet).field(i)
column_array << column.name
end
end
return column_array
end
end

158
src/invidious/database/channels.cr Normal file
View File

@@ -0,0 +1,158 @@
require "./base.cr"
#
# This module contains functions related to the "channels" table.
#
module Invidious::Database::Channels
extend self
# -------------------
# Insert / delete
# -------------------
def insert(channel : InvidiousChannel, update_on_conflict : Bool = false)
channel_array = channel.to_a
request = <<-SQL
INSERT INTO channels
VALUES (#{arg_array(channel_array)})
SQL
if update_on_conflict
request += <<-SQL
ON CONFLICT (id) DO UPDATE
SET author = $2, updated = $3
SQL
end
PG_DB.exec(request, args: channel_array)
end
# -------------------
# Update
# -------------------
def update_author(id : String, author : String)
request = <<-SQL
UPDATE channels
SET updated = now(), author = $1, deleted = false
WHERE id = $2
SQL
PG_DB.exec(request, author, id)
end
def update_subscription_time(id : String)
request = <<-SQL
UPDATE channels
SET subscribed = now()
WHERE id = $1
SQL
PG_DB.exec(request, id)
end
def update_mark_deleted(id : String)
request = <<-SQL
UPDATE channels
SET updated = now(), deleted = true
WHERE id = $1
SQL
PG_DB.exec(request, id)
end
# -------------------
# Select
# -------------------
def select(id : String) : InvidiousChannel?
request = <<-SQL
SELECT * FROM channels
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: InvidiousChannel)
end
def select(ids : Array(String)) : Array(InvidiousChannel)?
return [] of InvidiousChannel if ids.empty?
request = <<-SQL
SELECT * FROM channels
WHERE id = ANY($1)
SQL
return PG_DB.query_all(request, ids, as: InvidiousChannel)
end
end
#
# This module contains functions related to the "channel_videos" table.
#
module Invidious::Database::ChannelVideos
extend self
# -------------------
# Insert
# -------------------
# This function returns the status of the query (i.e: success?)
def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool
if with_premiere_timestamp
last_items = "premiere_timestamp = $9, views = $10"
else
last_items = "views = $10"
end
request = <<-SQL
INSERT INTO channel_videos
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (id) DO UPDATE
SET title = $2, published = $3, updated = $4, ucid = $5,
author = $6, length_seconds = $7, live_now = $8, #{last_items}
RETURNING (xmax=0) AS was_insert
SQL
return PG_DB.query_one(request, *video.to_tuple, as: Bool)
end
# -------------------
# Select
# -------------------
def select(ids : Array(String)) : Array(ChannelVideo)
return [] of ChannelVideo if ids.empty?
request = <<-SQL
SELECT * FROM channel_videos
WHERE id = ANY($1)
ORDER BY published DESC
SQL
return PG_DB.query_all(request, ids, as: ChannelVideo)
end
def select_notfications(ucid : String, since : Time) : Array(ChannelVideo)
request = <<-SQL
SELECT * FROM channel_videos
WHERE ucid = $1 AND published > $2
ORDER BY published DESC
LIMIT 15
SQL
return PG_DB.query_all(request, ucid, since, as: ChannelVideo)
end
def select_popular_videos : Array(ChannelVideo)
request = <<-SQL
SELECT DISTINCT ON (ucid) *
FROM channel_videos
WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC
SQL
PG_DB.query_all(request, as: ChannelVideo)
end
end

38
src/invidious/database/migration.cr Normal file
View File

@@ -0,0 +1,38 @@
abstract class Invidious::Database::Migration
macro inherited
Migrator.migrations << self
end
@@version : Int64?
def self.version(version : Int32 | Int64)
@@version = version.to_i64
end
getter? completed = false
def initialize(@db : DB::Database)
end
abstract def up(conn : DB::Connection)
def migrate
# migrator already ignores completed migrations
# but this is an extra check to make sure a migration doesn't run twice
return if completed?
@db.transaction do |txn|
up(txn.connection)
track(txn.connection)
@completed = true
end
end
def version : Int64
@@version.not_nil!
end
private def track(conn : DB::Connection)
conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version)
end
end

30
src/invidious/database/migrations/0001_create_channels_table.cr Normal file
View File

@@ -0,0 +1,30 @@
module Invidious::Database::Migrations
class CreateChannelsTable < Migration
version 1
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.channels
(
id text NOT NULL,
author text,
updated timestamp with time zone,
deleted boolean,
subscribed timestamp with time zone,
CONSTRAINT channels_id_key UNIQUE (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.channels TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS channels_id_idx
ON public.channels
USING btree
(id COLLATE pg_catalog."default");
SQL
end
end
end

28
src/invidious/database/migrations/0002_create_videos_table.cr Normal file
View File

@@ -0,0 +1,28 @@
module Invidious::Database::Migrations
class CreateVideosTable < Migration
version 2
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE UNLOGGED TABLE IF NOT EXISTS public.videos
(
id text NOT NULL,
info text,
updated timestamp with time zone,
CONSTRAINT videos_pkey PRIMARY KEY (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.videos TO current_user;
SQL
conn.exec <<-SQL
CREATE UNIQUE INDEX IF NOT EXISTS id_idx
ON public.videos
USING btree
(id COLLATE pg_catalog."default");
SQL
end
end
end

35
src/invidious/database/migrations/0003_create_channel_videos_table.cr Normal file
View File

@@ -0,0 +1,35 @@
module Invidious::Database::Migrations
class CreateChannelVideosTable < Migration
version 3
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.channel_videos
(
id text NOT NULL,
title text,
published timestamp with time zone,
updated timestamp with time zone,
ucid text,
author text,
length_seconds integer,
live_now boolean,
premiere_timestamp timestamp with time zone,
views bigint,
CONSTRAINT channel_videos_id_key UNIQUE (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.channel_videos TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
ON public.channel_videos
USING btree
(ucid COLLATE pg_catalog."default");
SQL
end
end
end

34
src/invidious/database/migrations/0004_create_users_table.cr Normal file
View File

@@ -0,0 +1,34 @@
module Invidious::Database::Migrations
class CreateUsersTable < Migration
version 4
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.users
(
updated timestamp with time zone,
notifications text[],
subscriptions text[],
email text NOT NULL,
preferences text,
password text,
token text,
watched text[],
feed_needs_update boolean,
CONSTRAINT users_email_key UNIQUE (email)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.users TO current_user;
SQL
conn.exec <<-SQL
CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx
ON public.users
USING btree
(lower(email) COLLATE pg_catalog."default");
SQL
end
end
end

28
src/invidious/database/migrations/0005_create_session_ids_table.cr Normal file
View File

@@ -0,0 +1,28 @@
module Invidious::Database::Migrations
class CreateSessionIdsTable < Migration
version 5
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.session_ids
(
id text NOT NULL,
email text,
issued timestamp with time zone,
CONSTRAINT session_ids_pkey PRIMARY KEY (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.session_ids TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS session_ids_id_idx
ON public.session_ids
USING btree
(id COLLATE pg_catalog."default");
SQL
end
end
end

27
src/invidious/database/migrations/0006_create_nonces_table.cr Normal file
View File

@@ -0,0 +1,27 @@
module Invidious::Database::Migrations
class CreateNoncesTable < Migration
version 6
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.nonces
(
nonce text,
expire timestamp with time zone,
CONSTRAINT nonces_id_key UNIQUE (nonce)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.nonces TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS nonces_nonce_idx
ON public.nonces
USING btree
(nonce COLLATE pg_catalog."default");
SQL
end
end
end

20
src/invidious/database/migrations/0007_create_annotations_table.cr Normal file
View File

@@ -0,0 +1,20 @@
module Invidious::Database::Migrations
class CreateAnnotationsTable < Migration
version 7
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.annotations
(
id text NOT NULL,
annotations xml,
CONSTRAINT annotations_id_key UNIQUE (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.annotations TO current_user;
SQL
end
end
end

50
src/invidious/database/migrations/0008_create_playlists_table.cr Normal file
View File

@@ -0,0 +1,50 @@
module Invidious::Database::Migrations
class CreatePlaylistsTable < Migration
version 8
def up(conn : DB::Connection)
if !privacy_type_exists?(conn)
conn.exec <<-SQL
CREATE TYPE public.privacy AS ENUM
(
'Public',
'Unlisted',
'Private'
);
SQL
end
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlists
(
title text,
id text primary key,
author text,
description text,
video_count integer,
created timestamptz,
updated timestamptz,
privacy privacy,
index int8[]
);
SQL
conn.exec <<-SQL
GRANT ALL ON public.playlists TO current_user;
SQL
end
private def privacy_type_exists?(conn : DB::Connection) : Bool
request = <<-SQL
SELECT 1 AS one
FROM pg_type
INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace
WHERE pg_namespace.nspname = 'public'
AND pg_type.typname = 'privacy'
LIMIT 1;
SQL
!conn.query_one?(request, as: Int32).nil?
end
end
end

27
src/invidious/database/migrations/0009_create_playlist_videos_table.cr Normal file
View File

@@ -0,0 +1,27 @@
module Invidious::Database::Migrations
class CreatePlaylistVideosTable < Migration
version 9
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlist_videos
(
title text,
id text,
author text,
ucid text,
length_seconds integer,
published timestamptz,
plid text references playlists(id),
index int8,
live_now boolean,
PRIMARY KEY (index,plid)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.playlist_videos TO current_user;
SQL
end
end
end

11
src/invidious/database/migrations/0010_make_videos_unlogged.cr Normal file
View File

@@ -0,0 +1,11 @@
module Invidious::Database::Migrations
class MakeVideosUnlogged < Migration
version 10
def up(conn : DB::Connection)
conn.exec <<-SQL
ALTER TABLE public.videos SET UNLOGGED;
SQL
end
end
end

49
src/invidious/database/migrator.cr Normal file
View File

@@ -0,0 +1,49 @@
class Invidious::Database::Migrator
MIGRATIONS_TABLE = "public.invidious_migrations"
class_getter migrations = [] of Invidious::Database::Migration.class
def initialize(@db : DB::Database)
end
def migrate
versions = load_versions
ran_migration = false
load_migrations.sort_by(&.version)
.each do |migration|
next if versions.includes?(migration.version)
puts "Running migration: #{migration.class.name}"
migration.migrate
ran_migration = true
end
puts "No migrations to run." unless ran_migration
end
def pending_migrations? : Bool
versions = load_versions
load_migrations.sort_by(&.version)
.any? { |migration| !versions.includes?(migration.version) }
end
private def load_migrations : Array(Invidious::Database::Migration)
self.class.migrations.map(&.new(@db))
end
private def load_versions : Array(Int64)
create_migrations_table
@db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64)
end
private def create_migrations_table
@db.exec <<-SQL
CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} (
id bigserial PRIMARY KEY,
version bigint NOT NULL
)
SQL
end
end

55
src/invidious/database/nonces.cr Normal file
View File

@@ -0,0 +1,55 @@
require "./base.cr"
module Invidious::Database::Nonces
extend self
# -------------------
# Insert / Delete
# -------------------
def insert(nonce : String, expire : Time)
request = <<-SQL
INSERT INTO nonces
VALUES ($1, $2)
ON CONFLICT DO NOTHING
SQL
PG_DB.exec(request, nonce, expire)
end
def delete_expired
request = <<-SQL
DELETE FROM nonces *
WHERE expire < now()
SQL
PG_DB.exec(request)
end
# -------------------
# Update
# -------------------
def update_set_expired(nonce : String)
request = <<-SQL
UPDATE nonces
SET expire = $1
WHERE nonce = $2
SQL
PG_DB.exec(request, Time.utc(1990, 1, 1), nonce)
end
# -------------------
# Select
# -------------------
def select(nonce : String) : Tuple(String, Time)?
request = <<-SQL
SELECT * FROM nonces
WHERE nonce = $1
SQL
return PG_DB.query_one?(request, nonce, as: {String, Time})
end
end

262
src/invidious/database/playlists.cr Normal file
View File

@@ -0,0 +1,262 @@
require "./base.cr"
#
# This module contains functions related to the "playlists" table.
#
module Invidious::Database::Playlists
extend self
# -------------------
# Insert / delete
# -------------------
def insert(playlist : InvidiousPlaylist)
playlist_array = playlist.to_a
request = <<-SQL
INSERT INTO playlists
VALUES (#{arg_array(playlist_array)})
SQL
PG_DB.exec(request, args: playlist_array)
end
# deletes the given playlist and connected playlist videos
def delete(id : String)
PlaylistVideos.delete_by_playlist(id)
request = <<-SQL
DELETE FROM playlists *
WHERE id = $1
SQL
PG_DB.exec(request, id)
end
# -------------------
# Update
# -------------------
def update(id : String, title : String, privacy, description, updated)
request = <<-SQL
UPDATE playlists
SET title = $1, privacy = $2, description = $3, updated = $4
WHERE id = $5
SQL
PG_DB.exec(request, title, privacy, description, updated, id)
end
def update_description(id : String, description)
request = <<-SQL
UPDATE playlists
SET description = $1
WHERE id = $2
SQL
PG_DB.exec(request, description, id)
end
def update_subscription_time(id : String)
request = <<-SQL
UPDATE playlists
SET subscribed = now()
WHERE id = $1
SQL
PG_DB.exec(request, id)
end
def update_video_added(id : String, index : String | Int64)
request = <<-SQL
UPDATE playlists
SET index = array_append(index, $1),
video_count = cardinality(index) + 1,
updated = now()
WHERE id = $2
SQL
PG_DB.exec(request, index, id)
end
def update_video_removed(id : String, index : String | Int64)
request = <<-SQL
UPDATE playlists
SET index = array_remove(index, $1),
video_count = cardinality(index) - 1,
updated = now()
WHERE id = $2
SQL
PG_DB.exec(request, index, id)
end
# -------------------
# Salect
# -------------------
def select(*, id : String) : InvidiousPlaylist?
request = <<-SQL
SELECT * FROM playlists
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: InvidiousPlaylist)
end
def select_all(*, author : String) : Array(InvidiousPlaylist)
request = <<-SQL
SELECT * FROM playlists
WHERE author = $1
SQL
return PG_DB.query_all(request, author, as: InvidiousPlaylist)
end
# -------------------
# Salect (filtered)
# -------------------
def select_like_iv(email : String) : Array(InvidiousPlaylist)
request = <<-SQL
SELECT * FROM playlists
WHERE author = $1 AND id LIKE 'IV%'
ORDER BY created
SQL
PG_DB.query_all(request, email, as: InvidiousPlaylist)
end
def select_not_like_iv(email : String) : Array(InvidiousPlaylist)
request = <<-SQL
SELECT * FROM playlists
WHERE author = $1 AND id NOT LIKE 'IV%'
ORDER BY created
SQL
PG_DB.query_all(request, email, as: InvidiousPlaylist)
end
def select_user_created_playlists(email : String) : Array({String, String})
request = <<-SQL
SELECT id,title FROM playlists
WHERE author = $1 AND id LIKE 'IV%'
ORDER BY title
SQL
PG_DB.query_all(request, email, as: {String, String})
end
# -------------------
# Misc checks
# -------------------
# Check if given playlist ID exists
def exists?(id : String) : Bool
request = <<-SQL
SELECT id FROM playlists
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: String).nil?
end
# Count how many playlist a user has created.
def count_owned_by(author : String) : Int64
request = <<-SQL
SELECT count(*) FROM playlists
WHERE author = $1
SQL
return PG_DB.query_one?(request, author, as: Int64) || 0_i64
end
end
#
# This module contains functions related to the "playlist_videos" table.
#
module Invidious::Database::PlaylistVideos
extend self
private alias VideoIndex = Int64 | Array(Int64)
# -------------------
# Insert / Delete
# -------------------
def insert(video : PlaylistVideo)
video_array = video.to_a
request = <<-SQL
INSERT INTO playlist_videos
VALUES (#{arg_array(video_array)})
SQL
PG_DB.exec(request, args: video_array)
end
def delete(index)
request = <<-SQL
DELETE FROM playlist_videos *
WHERE index = $1
SQL
PG_DB.exec(request, index)
end
def delete_by_playlist(plid : String)
request = <<-SQL
DELETE FROM playlist_videos *
WHERE plid = $1
SQL
PG_DB.exec(request, plid)
end
# -------------------
# Salect
# -------------------
def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo)
request = <<-SQL
SELECT * FROM playlist_videos
WHERE plid = $1
ORDER BY array_position($2, index)
LIMIT $3
OFFSET $4
SQL
return PG_DB.query_all(request, plid, index, limit, offset, as: PlaylistVideo)
end
def select_index(plid : String, vid : String) : Int64?
request = <<-SQL
SELECT index FROM playlist_videos
WHERE plid = $1 AND id = $2
LIMIT 1
SQL
return PG_DB.query_one?(request, plid, vid, as: Int64)
end
def select_one_id(plid : String, index : VideoIndex) : String?
request = <<-SQL
SELECT id FROM playlist_videos
WHERE plid = $1
ORDER BY array_position($2, index)
LIMIT 1
SQL
return PG_DB.query_one?(request, plid, index, as: String)
end
def select_ids(plid : String, index : VideoIndex, limit = 500) : Array(String)
request = <<-SQL
SELECT id FROM playlist_videos
WHERE plid = $1
ORDER BY array_position($2, index)
LIMIT $3
SQL
return PG_DB.query_all(request, plid, index, limit, as: String)
end
end

74
src/invidious/database/sessions.cr Normal file
View File

@@ -0,0 +1,74 @@
require "./base.cr"
module Invidious::Database::SessionIDs
extend self
# -------------------
# Insert
# -------------------
def insert(sid : String, email : String, handle_conflicts : Bool = false)
request = <<-SQL
INSERT INTO session_ids
VALUES ($1, $2, now())
SQL
request += " ON CONFLICT (id) DO NOTHING" if handle_conflicts
PG_DB.exec(request, sid, email)
end
# -------------------
# Delete
# -------------------
def delete(*, sid : String)
request = <<-SQL
DELETE FROM session_ids *
WHERE id = $1
SQL
PG_DB.exec(request, sid)
end
def delete(*, email : String)
request = <<-SQL
DELETE FROM session_ids *
WHERE email = $1
SQL
PG_DB.exec(request, email)
end
def delete(*, sid : String, email : String)
request = <<-SQL
DELETE FROM session_ids *
WHERE id = $1 AND email = $2
SQL
PG_DB.exec(request, sid, email)
end
# -------------------
# Select
# -------------------
def select_email(sid : String) : String?
request = <<-SQL
SELECT email FROM session_ids
WHERE id = $1
SQL
PG_DB.query_one?(request, sid, as: String)
end
def select_all(email : String) : Array({session: String, issued: Time})
request = <<-SQL
SELECT id, issued FROM session_ids
WHERE email = $1
ORDER BY issued DESC
SQL
PG_DB.query_all(request, email, as: {session: String, issued: Time})
end
end

49
src/invidious/database/statistics.cr Normal file
View File

@@ -0,0 +1,49 @@
require "./base.cr"
module Invidious::Database::Statistics
extend self
# -------------------
# User stats
# -------------------
def count_users_total : Int64
request = <<-SQL
SELECT count(*) FROM users
SQL
PG_DB.query_one(request, as: Int64)
end
def count_users_active_6m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '6 months'
SQL
PG_DB.query_one(request, as: Int64)
end
def count_users_active_1m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '1 month'
SQL
PG_DB.query_one(request, as: Int64)
end
# -------------------
# Channel stats
# -------------------
def channel_last_update : Time?
request = <<-SQL
SELECT updated FROM channels
ORDER BY updated DESC
LIMIT 1
SQL
PG_DB.query_one?(request, as: Time)
end
end

228
src/invidious/database/users.cr Normal file
View File

@@ -0,0 +1,228 @@
require "./base.cr"
module Invidious::Database::Users
extend self
# -------------------
# Insert / delete
# -------------------
def insert(user : User, update_on_conflict : Bool = false)
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
request = <<-SQL
INSERT INTO users
VALUES (#{arg_array(user_array)})
SQL
if update_on_conflict
request += <<-SQL
ON CONFLICT (email) DO UPDATE
SET updated = $1, subscriptions = $3
SQL
end
PG_DB.exec(request, args: user_array)
end
def delete(user : User)
request = <<-SQL
DELETE FROM users *
WHERE email = $1
SQL
PG_DB.exec(request, user.email)
end
# -------------------
# Update (history)
# -------------------
def update_watch_history(user : User)
request = <<-SQL
UPDATE users
SET watched = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.watched, user.email)
end
def mark_watched(user : User, vid : String)
request = <<-SQL
UPDATE users
SET watched = array_append(array_remove(watched, $1), $1)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
end
def mark_unwatched(user : User, vid : String)
request = <<-SQL
UPDATE users
SET watched = array_remove(watched, $1)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
end
def clear_watch_history(user : User)
request = <<-SQL
UPDATE users
SET watched = '{}'
WHERE email = $1
SQL
PG_DB.exec(request, user.email)
end
# -------------------
# Update (channels)
# -------------------
def update_subscriptions(user : User)
request = <<-SQL
UPDATE users
SET feed_needs_update = true, subscriptions = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.subscriptions, user.email)
end
def subscribe_channel(user : User, ucid : String)
request = <<-SQL
UPDATE users
SET feed_needs_update = true,
subscriptions = array_append(subscriptions,$1)
WHERE email = $2
SQL
PG_DB.exec(request, ucid, user.email)
end
def unsubscribe_channel(user : User, ucid : String)
request = <<-SQL
UPDATE users
SET feed_needs_update = true,
subscriptions = array_remove(subscriptions, $1)
WHERE email = $2
SQL
PG_DB.exec(request, ucid, user.email)
end
# -------------------
# Update (notifs)
# -------------------
def add_notification(video : ChannelVideo)
request = <<-SQL
UPDATE users
SET notifications = array_append(notifications, $1),
feed_needs_update = true
WHERE $2 = ANY(subscriptions)
SQL
PG_DB.exec(request, video.id, video.ucid)
end
def remove_notification(user : User, vid : String)
request = <<-SQL
UPDATE users
SET notifications = array_remove(notifications, $1)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
end
def clear_notifications(user : User)
request = <<-SQL
UPDATE users
SET notifications = '{}', updated = now()
WHERE email = $1
SQL
PG_DB.exec(request, user.email)
end
# -------------------
# Update (misc)
# -------------------
def feed_needs_update(video : ChannelVideo)
request = <<-SQL
UPDATE users
SET feed_needs_update = true
WHERE $1 = ANY(subscriptions)
SQL
PG_DB.exec(request, video.ucid)
end
def update_preferences(user : User)
request = <<-SQL
UPDATE users
SET preferences = $1
WHERE email = $2
SQL
PG_DB.exec(request, user.preferences.to_json, user.email)
end
def update_password(user : User, pass : String)
request = <<-SQL
UPDATE users
SET password = $1
WHERE email = $2
SQL
PG_DB.exec(request, pass, user.email)
end
# -------------------
# Select
# -------------------
def select(*, email : String) : User?
request = <<-SQL
SELECT * FROM users
WHERE email = $1
SQL
return PG_DB.query_one?(request, email, as: User)
end
# Same as select, but can raise an exception
def select!(*, email : String) : User
request = <<-SQL
SELECT * FROM users
WHERE email = $1
SQL
return PG_DB.query_one(request, email, as: User)
end
def select(*, token : String) : User?
request = <<-SQL
SELECT * FROM users
WHERE token = $1
SQL
return PG_DB.query_one?(request, token, as: User)
end
def select_notifications(user : User) : Array(String)
request = <<-SQL
SELECT notifications
FROM users
WHERE email = $1
SQL
return PG_DB.query_one(request, user.email, as: Array(String))
end
end

52
src/invidious/database/videos.cr Normal file
View File

@@ -0,0 +1,52 @@
require "./base.cr"
module Invidious::Database::Videos
extend self
def insert(video : Video)
request = <<-SQL
INSERT INTO videos
VALUES ($1, $2, $3)
ON CONFLICT (id) DO NOTHING
SQL
PG_DB.exec(request, video.id, video.info.to_json, video.updated)
end
def delete(id)
request = <<-SQL
DELETE FROM videos *
WHERE id = $1
SQL
PG_DB.exec(request, id)
end
def delete_expired
request = <<-SQL
DELETE FROM videos *
WHERE updated < (now() - interval '6 hours')
SQL
PG_DB.exec(request)
end
def update(video : Video)
request = <<-SQL
UPDATE videos
SET (id, info, updated) = ($1, $2, $3)
WHERE id = $1
SQL
PG_DB.exec(request, video.id, video.info.to_json, video.updated)
end
def select(id : String) : Video?
request = <<-SQL
SELECT * FROM videos
WHERE id = $1
SQL
return PG_DB.query_one?(request, id, as: Video)
end
end

40
src/invidious/exceptions.cr Normal file
View File

@@ -0,0 +1,40 @@
# InfoExceptions are for displaying information to the user.
#
# An InfoException might or might not indicate that something went wrong.
# Historically Invidious didn't differentiate between these two options, so to
# maintain previous functionality InfoExceptions do not print backtraces.
class InfoException < Exception
end
# Exception used to hold the bogus UCID during a channel search.
class ChannelSearchException < InfoException
getter channel : String
def initialize(@channel)
end
end
# Exception used to hold the name of the missing item
# Should be used in all parsing functions
class BrokenTubeException < Exception
getter element : String
def initialize(@element)
end
def message
return "Missing JSON element \"#{@element}\""
end
end
# Exception threw when an element is not found.
class NotFoundException < InfoException
end
class VideoNotAvailableException < Exception
end
# Exception used to indicate that the JSON response from YT is missing
# some important informations, and that the query should be sent again.
class RetryOnceException < Exception
end

46
src/invidious/frontend/channel_page.cr Normal file
View File

@@ -0,0 +1,46 @@
module Invidious::Frontend::ChannelPage
extend self
enum TabsAvailable
Videos
Shorts
Streams
Podcasts
Releases
Playlists
Community
Channels
end
def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
return String.build(1500) do |str|
base_url = "/channel/#{channel.ucid}"
TabsAvailable.each do |tab|
# Ignore playlists, as it is not supported for auto-generated channels yet
next if (tab.playlists? && channel.auto_generated)
tab_name = tab.to_s.downcase
if channel.tabs.includes? tab_name
str << %(<div class="pure-u-1 pure-md-1-3">\n)
if tab == selected_tab
str << "\t<b>"
str << translate(locale, "channel_tab_#{tab_name}_label")
str << "</b>\n"
else
# Video tab doesn't have the last path component
url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
str << %(\t<a href=") << url << %(">)
str << translate(locale, "channel_tab_#{tab_name}_label")
str << "</a>\n"
end
str << "</div>"
end
end
end
end
end

50
src/invidious/frontend/comments_reddit.cr Normal file
View File

@@ -0,0 +1,50 @@
module Invidious::Frontend::Comments
extend self
def template_reddit(root, locale)
String.build do |html|
root.each do |child|
if child.data.is_a?(RedditComment)
child = child.data.as(RedditComment)
body_html = HTML.unescape(child.body_html)
replies_html = ""
if child.replies.is_a?(RedditThing)
replies = child.replies.as(RedditThing)
replies_html = self.template_reddit(replies.data.as(RedditListing).children, locale)
end
if child.depth > 0
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1-24">
</div>
<div class="pure-u-23-24">
END_HTML
else
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1">
END_HTML
end
html << <<-END_HTML
<p>
<a href="javascript:void(0)" data-onclick="toggle_parent">[ − ]</a>
<b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b>
#{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)}
<span title="#{child.created_utc.to_s("%a %B %-d %T %Y UTC")}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span>
<a href="https://www.reddit.com#{child.permalink}" title="#{translate(locale, "permalink")}">#{translate(locale, "permalink")}</a>
</p>
<div>
#{body_html}
#{replies_html}
</div>
</div>
</div>
END_HTML
end
end
end
end
end

208
src/invidious/frontend/comments_youtube.cr Normal file
View File

@@ -0,0 +1,208 @@
module Invidious::Frontend::Comments
extend self
def template_youtube(comments, locale, thin_mode, is_replies = false)
String.build do |html|
root = comments["comments"].as_a
root.each do |child|
if child["replies"]?
replies_count_text = translate_count(locale,
"comments_view_x_replies",
child["replies"]["replyCount"].as_i64 || 0,
NumberFormatting::Separator
)
replies_html = <<-END_HTML
<div id="replies" class="pure-g">
<div class="pure-u-1-24"></div>
<div class="pure-u-23-24">
<p>
<a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}"
data-onclick="get_youtube_replies" data-load-replies>#{replies_count_text}</a>
</p>
</div>
</div>
END_HTML
elsif comments["authorId"]? && !comments["singlePost"]?
# for posts we should display a link to the post
replies_count_text = translate_count(locale,
"comments_view_x_replies",
child["replyCount"].as_i64 || 0,
NumberFormatting::Separator
)
replies_html = <<-END_HTML
<div class="pure-g">
<div class="pure-u-1-24"></div>
<div class="pure-u-23-24">
<p>
<a href="/post/#{child["commentId"]}?ucid=#{comments["authorId"]}">#{replies_count_text}</a>
</p>
</div>
</div>
END_HTML
end
if !thin_mode
author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}"
else
author_thumbnail = ""
end
author_name = HTML.escape(child["author"].as_s)
sponsor_icon = ""
if child["verified"]?.try &.as_bool && child["authorIsChannelOwner"]?.try &.as_bool
author_name += "&nbsp;<i class=\"icon ion ion-md-checkmark-circle\"></i>"
elsif child["verified"]?.try &.as_bool
author_name += "&nbsp;<i class=\"icon ion ion-md-checkmark\"></i>"
end
if child["isSponsor"]?.try &.as_bool
sponsor_icon = String.build do |str|
str << %(<img alt="" )
str << %(src="/ggpht) << URI.parse(child["sponsorIconUrl"].as_s).request_target << "\" "
str << %(title=") << translate(locale, "Channel Sponsor") << "\" "
str << %(width="16" height="16" />)
end
end
html << <<-END_HTML
<div class="pure-g" style="width:100%">
<div class="channel-profile pure-u-4-24 pure-u-md-2-24">
<img loading="lazy" style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}" alt="" />
</div>
<div class="pure-u-20-24 pure-u-md-22-24">
<p>
<b>
<a class="#{child["authorIsChannelOwner"] == true ? "channel-owner" : ""}" href="#{child["authorUrl"]}">#{author_name}</a>
</b>
#{sponsor_icon}
<p style="white-space:pre-wrap">#{child["contentHtml"]}</p>
END_HTML
if child["attachment"]?
attachment = child["attachment"]
case attachment["type"]
when "image"
attachment = attachment["imageThumbnails"][1]
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-2">
<img loading="lazy" style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}" alt="" />
</div>
</div>
END_HTML
when "video"
if attachment["error"]?
html << <<-END_HTML
<div class="pure-g video-iframe-wrapper">
<p>#{attachment["error"]}</p>
</div>
END_HTML
else
html << <<-END_HTML
<div class="pure-g video-iframe-wrapper">
<iframe class="video-iframe" src='/embed/#{attachment["videoId"]?}?autoplay=0'></iframe>
</div>
END_HTML
end
when "multiImage"
html << <<-END_HTML
<section class="carousel">
<a class="skip-link" href="#skip-#{child["commentId"]}">#{translate(locale, "carousel_skip")}</a>
<div class="slides">
END_HTML
image_array = attachment["images"].as_a
image_array.each_index do |i|
html << <<-END_HTML
<div class="slides-item slide-#{i + 1}" id="#{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_slide", {"current" => (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0">
<img loading="lazy" src="/ggpht#{URI.parse(image_array[i][1]["url"].as_s).request_target}" alt="" />
</div>
END_HTML
end
html << <<-END_HTML
</div>
<div class="carousel__nav">
END_HTML
attachment["images"].as_a.each_index do |i|
html << <<-END_HTML
<a class="slider-nav" href="##{child["commentId"]}-slide-#{i + 1}" aria-label="#{translate(locale, "carousel_go_to", (i + 1).to_s)}" tabindex="-1" aria-hidden="true">#{i + 1}</a>
END_HTML
end
html << <<-END_HTML
</div>
<div id="skip-#{child["commentId"]}"></div>
</section>
END_HTML
else nil # Ignore
end
end
html << <<-END_HTML
<p>
<span title="#{Time.unix(child["published"].as_i64).to_s(translate(locale, "%A %B %-d, %Y"))}">#{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}</span>
|
END_HTML
if comments["videoId"]?
html << <<-END_HTML
<a rel="noreferrer noopener" href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a>
|
END_HTML
elsif comments["authorId"]?
html << <<-END_HTML
<a rel="noreferrer noopener" href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a>
|
END_HTML
end
html << <<-END_HTML
<i class="icon ion-ios-thumbs-up"></i> #{number_with_separator(child["likeCount"])}
END_HTML
if child["creatorHeart"]?
if !thin_mode
creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}"
else
creator_thumbnail = ""
end
html << <<-END_HTML
&nbsp;
<span class="creator-heart-container" title="#{translate(locale, "`x` marked it with a ❤", child["creatorHeart"]["creatorName"].as_s)}">
<span class="creator-heart">
<img loading="lazy" class="creator-heart-background-hearted" src="#{creator_thumbnail}" alt="" />
<span class="creator-heart-small-hearted">
<span class="icon ion-ios-heart creator-heart-small-container"></span>
</span>
</span>
</span>
END_HTML
end
html << <<-END_HTML
</p>
#{replies_html}
</div>
</div>
END_HTML
end
if comments["continuation"]?
html << <<-END_HTML
<div class="pure-g">
<div class="pure-u-1">
<p>
<a href="javascript:void(0)" data-continuation="#{comments["continuation"]}"
data-onclick="get_youtube_replies" data-load-more #{"data-load-replies" if is_replies}>#{translate(locale, "Load more")}</a>
</p>
</div>
</div>
END_HTML
end
end
end
end

14
src/invidious/frontend/misc.cr Normal file
View File

@@ -0,0 +1,14 @@
module Invidious::Frontend::Misc
extend self
def redirect_url(env : HTTP::Server::Context)
prefs = env.get("preferences").as(Preferences)
if prefs.automatic_instance_redirect
current_page = env.get?("current_page").as(String)
return "/redirect?referer=#{current_page}"
else
return "https://redirect.invidious.io#{env.request.resource}"
end
end
end

97
src/invidious/frontend/pagination.cr Normal file
View File

@@ -0,0 +1,97 @@
require "uri"
module Invidious::Frontend::Pagination
extend self
private def previous_page(str : String::Builder, locale : String?, url : String)
# Link
str << %(<a href=") << url << %(" class="pure-button pure-button-secondary">)
if locale_is_rtl?(locale)
# Inverted arrow ("previous" points to the right)
str << translate(locale, "Previous page")
str << "&nbsp;&nbsp;"
str << %(<i class="icon ion-ios-arrow-forward"></i>)
else
# Regular arrow ("previous" points to the left)
str << %(<i class="icon ion-ios-arrow-back"></i>)
str << "&nbsp;&nbsp;"
str << translate(locale, "Previous page")
end
str << "</a>"
end
private def next_page(str : String::Builder, locale : String?, url : String)
# Link
str << %(<a href=") << url << %(" class="pure-button pure-button-secondary">)
if locale_is_rtl?(locale)
# Inverted arrow ("next" points to the left)
str << %(<i class="icon ion-ios-arrow-back"></i>)
str << "&nbsp;&nbsp;"
str << translate(locale, "Next page")
else
# Regular arrow ("next" points to the right)
str << translate(locale, "Next page")
str << "&nbsp;&nbsp;"
str << %(<i class="icon ion-ios-arrow-forward"></i>)
end
str << "</a>"
end
def nav_numeric(locale : String?, *, base_url : String | URI, current_page : Int, show_next : Bool = true)
return String.build do |str|
str << %(<div class="h-box">\n)
str << %(<div class="page-nav-container flexible">\n)
str << %(<div class="page-prev-container flex-left">)
if current_page > 1
params_prev = URI::Params{"page" => (current_page - 1).to_s}
url_prev = HttpServer::Utils.add_params_to_url(base_url, params_prev)
self.previous_page(str, locale, url_prev.to_s)
end
str << %(</div>\n)
str << %(<div class="page-next-container flex-right">)
if show_next
params_next = URI::Params{"page" => (current_page + 1).to_s}
url_next = HttpServer::Utils.add_params_to_url(base_url, params_next)
self.next_page(str, locale, url_next.to_s)
end
str << %(</div>\n)
str << %(</div>\n)
str << %(</div>\n\n)
end
end
def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?)
return String.build do |str|
str << %(<div class="h-box">\n)
str << %(<div class="page-nav-container flexible">\n)
str << %(<div class="page-prev-container flex-left"></div>\n)
str << %(<div class="page-next-container flex-right">)
if !ctoken.nil?
params_next = URI::Params{"continuation" => ctoken}
url_next = HttpServer::Utils.add_params_to_url(base_url, params_next)
self.next_page(str, locale, url_next.to_s)
end
str << %(</div>\n)
str << %(</div>\n)
str << %(</div>\n\n)
end
end
end

135
src/invidious/frontend/search_filters.cr Normal file
View File

@@ -0,0 +1,135 @@
module Invidious::Frontend::SearchFilters
extend self
# Generate the search filters collapsable widget.
def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String
return String.build(8000) do |str|
str << "<div id='filters'>\n"
str << "\t<details id='filters-collapse'>"
str << "\t\t<summary>" << translate(locale, "search_filters_title") << "</summary>\n"
str << "\t\t<div id='filters-box'><form action='/search' method='get'>\n"
str << "\t\t\t<input type='hidden' name='q' value='" << HTML.escape(query) << "'>\n"
str << "\t\t\t<input type='hidden' name='page' value='" << page << "'>\n"
str << "\t\t\t<div id='filters-flex'>"
filter_wrapper(date)
filter_wrapper(type)
filter_wrapper(duration)
filter_wrapper(features)
filter_wrapper(sort)
str << "\t\t\t</div>\n"
str << "\t\t\t<div id='filters-apply'>"
str << "<button type='submit' class=\"pure-button pure-button-primary\">"
str << translate(locale, "search_filters_apply_button")
str << "</button></div>\n"
str << "\t\t</form></div>\n"
str << "\t</details>\n"
str << "</div>\n"
end
end
# Generate wrapper HTML (`<div>`, filter name, etc...) around the
# `<input>` elements of a search filter
macro filter_wrapper(name)
str << "\t\t\t\t<div class=\"filter-column\"><fieldset>\n"
str << "\t\t\t\t\t<legend><div class=\"filter-name underlined\">"
str << translate(locale, "search_filters_{{name}}_label")
str << "</div></legend>\n"
str << "\t\t\t\t\t<div class=\"filter-options\">\n"
make_{{name}}_filter_options(str, filters.{{name}}, locale)
str << "\t\t\t\t\t</div>"
str << "\t\t\t\t</fieldset></div>\n"
end
# Generates the HTML for the list of radio buttons of the "date" search filter
def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String)
{% for value in Invidious::Search::Filters::Date.constants %}
{% date = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='date' id='filter-date-{{date}}' value='{{date}}'"
str << " checked" if value.{{date}}?
str << '>'
str << "<label for='filter-date-{{date}}'>"
str << translate(locale, "search_filters_date_option_{{date}}")
str << "</label></div>\n"
{% end %}
end
# Generates the HTML for the list of radio buttons of the "type" search filter
def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String)
{% for value in Invidious::Search::Filters::Type.constants %}
{% type = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='type' id='filter-type-{{type}}' value='{{type}}'"
str << " checked" if value.{{type}}?
str << '>'
str << "<label for='filter-type-{{type}}'>"
str << translate(locale, "search_filters_type_option_{{type}}")
str << "</label></div>\n"
{% end %}
end
# Generates the HTML for the list of radio buttons of the "duration" search filter
def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String)
{% for value in Invidious::Search::Filters::Duration.constants %}
{% duration = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='duration' id='filter-duration-{{duration}}' value='{{duration}}'"
str << " checked" if value.{{duration}}?
str << '>'
str << "<label for='filter-duration-{{duration}}'>"
str << translate(locale, "search_filters_duration_option_{{duration}}")
str << "</label></div>\n"
{% end %}
end
# Generates the HTML for the list of checkboxes of the "features" search filter
def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String)
{% for value in Invidious::Search::Filters::Features.constants %}
{% if value.stringify != "All" && value.stringify != "None" %}
{% feature = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='checkbox' name='features' id='filter-feature-{{feature}}' value='{{feature}}'"
str << " checked" if value.{{feature}}?
str << '>'
str << "<label for='filter-feature-{{feature}}'>"
str << translate(locale, "search_filters_features_option_{{feature}}")
str << "</label></div>\n"
{% end %}
{% end %}
end
# Generates the HTML for the list of radio buttons of the "sort" search filter
def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String)
{% for value in Invidious::Search::Filters::Sort.constants %}
{% sort = value.underscore %}
str << "\t\t\t\t\t\t<div>"
str << "<input type='radio' name='sort' id='filter-sort-{{sort}}' value='{{sort}}'"
str << " checked" if value.{{sort}}?
str << '>'
str << "<label for='filter-sort-{{sort}}'>"
str << translate(locale, "search_filters_sort_option_{{sort}}")
str << "</label></div>\n"
{% end %}
end
end

107
src/invidious/frontend/watch_page.cr Normal file
View File

@@ -0,0 +1,107 @@
module Invidious::Frontend::WatchPage
extend self
# A handy structure to pass many elements at
# once to the download widget function
struct VideoAssets
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
getter captions : Array(Invidious::Videos::Captions::Metadata)
def initialize(
@full_videos,
@video_streams,
@audio_streams,
@captions
)
end
end
def download_widget(locale : String, video : Video, video_assets : VideoAssets) : String
if CONFIG.disabled?("downloads")
return "<p id=\"download\">#{translate(locale, "Download is disabled")}</p>"
end
return String.build(4000) do |str|
str << "<form"
str << " class=\"pure-form pure-form-stacked\""
str << " action='/download'"
str << " method='post'"
str << " rel='noopener'"
str << " target='_blank'>"
str << '\n'
# Hidden inputs for video id and title
str << "<input type='hidden' name='id' value='" << video.id << "'/>\n"
str << "<input type='hidden' name='title' value='" << HTML.escape(video.title) << "'/>\n"
str << "\t<div class=\"pure-control-group\">\n"
str << "\t\t<label for='download_widget'>"
str << translate(locale, "Download as: ")
str << "</label>\n"
str << "\t\t<select name='download_widget' id='download_widget'>\n"
# Non-DASH videos (audio+video)
video_assets.full_videos.each do |option|
mimetype = option["mimeType"].as_s.split(";")[0]
height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]?
value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json
str << "\t\t\t<option value='" << value << "'>"
str << (height || "~240") << "p - " << mimetype
str << "</option>\n"
end
# DASH video streams
video_assets.video_streams.each do |option|
mimetype = option["mimeType"].as_s.split(";")[0]
value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json
str << "\t\t\t<option value='" << value << "'>"
str << option["qualityLabel"] << " - " << mimetype << " @ " << option["fps"] << "fps - video only"
str << "</option>\n"
end
# DASH audio streams
video_assets.audio_streams.each do |option|
mimetype = option["mimeType"].as_s.split(";")[0]
value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json
str << "\t\t\t<option value='" << value << "'>"
str << mimetype << " @ " << (option["bitrate"]?.try &.as_i./ 1000) << "k - audio only"
str << "</option>\n"
end
# Subtitles (a.k.a "closed captions")
video_assets.captions.each do |caption|
value = {"label": caption.name, "ext": "#{caption.language_code}.vtt"}.to_json
str << "\t\t\t<option value='" << value << "'>"
str << translate(locale, "download_subtitles", translate(locale, caption.name))
str << "</option>\n"
end
# End of form
str << "\t\t</select>\n"
str << "\t</div>\n"
str << "\t<button type=\"submit\" class=\"pure-button pure-button-primary\">\n"
str << "\t\t<b>" << translate(locale, "Download") << "</b>\n"
str << "\t</button>\n"
str << "</form>\n"
end
end
end

42
src/invidious/hashtag.cr Normal file
View File

@@ -0,0 +1,42 @@
module Invidious::Hashtag
extend self
def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
cursor = (page - 1) * 60
ctoken = generate_continuation(hashtag, cursor)
client_config = YoutubeAPI::ClientConfig.new(region: region)
response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
items, _ = extract_items(response)
return items
end
def generate_continuation(hashtag : String, cursor : Int)
object = {
"80226972:embedded" => {
"2:string" => "FEhashtag",
"3:base64" => {
"1:varint" => 60_i64, # result count
"15:base64" => {
"1:varint" => cursor.to_i64,
"2:varint" => 0_i64,
},
"93:2:embedded" => {
"1:string" => hashtag,
"2:varint" => 0_i64,
"3:varint" => 1_i64,
},
},
"35:string" => "browse-feedFEhashtag",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
end

104
src/invidious/helpers/crystal_class_overrides.cr Normal file
View File

@@ -0,0 +1,104 @@
# Override of the TCPSocket and HTTP::Client classes in order to allow an
# IP family to be selected for domains that resolve to both IPv4 and
# IPv6 addresses.
#
class TCPSocket
def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
connect(addrinfo, timeout: connect_timeout) do |error|
close
error
end
end
end
end
# :ditto:
class HTTP::Client
property family : Socket::Family = Socket::Family::UNSPEC
# Override stdlib to automatically initialize proxy if configured
#
# Accurate as of crystal 1.12.1
def initialize(@host : String, port = nil, tls : TLSContext = nil)
check_host_only(@host)
{% if flag?(:without_openssl) %}
if tls
raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time"
end
@tls = nil
{% else %}
@tls = case tls
when true
OpenSSL::SSL::Context::Client.new
when OpenSSL::SSL::Context::Client
tls
when false, nil
nil
end
{% end %}
@port = (port || (@tls ? 443 : 80)).to_i
self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
end
def initialize(@io : IO, @host = "", @port = 80)
@reconnect = false
self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
end
private def io
io = @io
return io if io
unless @reconnect
raise "This HTTP::Client cannot be reconnected"
end
hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host
io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, family: @family
io.read_timeout = @read_timeout if @read_timeout
io.write_timeout = @write_timeout if @write_timeout
io.sync = false
{% if !flag?(:without_openssl) %}
if tls = @tls
tcp_socket = io
begin
io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host.rchop('.'))
rescue exc
# don't leak the TCP socket when the SSL connection failed
tcp_socket.close
raise exc
end
end
{% end %}
@io = io
end
end
# Mute the ClientError exception raised when a connection is flushed.
# This happends when the connection is unexpectedly closed by the client.
#
class HTTP::Server::Response
class Output
private def unbuffered_flush
@io.flush
rescue ex : IO::Error
unbuffered_close
end
end
end
# TODO: Document this override
#
class PG::ResultSet
def field(index = @column_index)
@fields.not_nil![index]
end
end

165
src/invidious/helpers/errors.cr
View File

@@ -1,13 +1,9 @@
# InfoExceptions are for displaying information to the user.
#
# An InfoException might or might not indicate that something went wrong.
# Historically Invidious didn't differentiate between these two options, so to
# maintain previous functionality InfoExceptions do not print backtraces.
class InfoException < Exception
end
# -------------------
# Issue template
# -------------------
macro error_template(*args)
error_template_helper(env, locale, {{*args}})
error_template_helper(env, {{args.splat}})
end
def github_details(summary : String, content : String)
@@ -22,84 +18,185 @@ def github_details(summary : String, content : String)
return HTML.escape(details)
end
def error_template_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception)
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_template_helper(env, locale, status_code, exception.message || "")
return error_template_helper(env, status_code, exception.message || "")
end
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "text/html"
env.response.status_code = status_code
issue_template = %(Title: `#{exception.message} (#{exception.class})`)
issue_template += %(\nDate: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`)
issue_template += %(\nRoute: `#{env.request.resource}`)
issue_template += %(\nVersion: `#{SOFTWARE["version"]} @ #{SOFTWARE["branch"]}`)
# issue_template += github_details("Preferences", env.get("preferences").as(Preferences).to_pretty_json)
issue_title = "#{exception.message} (#{exception.class})"
issue_template = <<-TEXT
Title: `#{HTML.escape(issue_title)}`
Date: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`
Route: `#{HTML.escape(env.request.resource)}`
Version: `#{SOFTWARE["version"]} @ #{SOFTWARE["branch"]}`
TEXT
issue_template += github_details("Backtrace", exception.inspect_with_backtrace)
# URLs for the error message below
url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md"
url_search_issues = "https://github.com/iv-org/invidious/issues"
url_search_issues += "?q=is:issue+is:open+"
url_search_issues += URI.encode_www_form("[Bug] #{issue_title}")
url_switch = "https://redirect.invidious.io" + env.request.resource
url_new_issue = "https://github.com/iv-org/invidious/issues/new"
url_new_issue += "?labels=bug&template=bug_report.md&title="
url_new_issue += URI.encode_www_form("[Bug] " + issue_title)
error_message = <<-END_HTML
Looks like you've found a bug in Invidious. Please open a new issue
<a href="https://github.com/iv-org/invidious/issues">on GitHub</a>
and include the following text in your message:
<pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre>
<div class="error_message">
<h2>#{translate(locale, "crash_page_you_found_a_bug")}</h2>
<br/><br/>
<p><b>#{translate(locale, "crash_page_before_reporting")}</b></p>
<ul>
<li>#{translate(locale, "crash_page_refresh", env.request.resource)}</li>
<li>#{translate(locale, "crash_page_switch_instance", url_switch)}</li>
<li>#{translate(locale, "crash_page_read_the_faq", url_faq)}</li>
<li>#{translate(locale, "crash_page_search_issue", url_search_issues)}</li>
</ul>
<br/>
<p>#{translate(locale, "crash_page_report_issue", url_new_issue)}</p>
<!-- TODO: Add a "copy to clipboard" button -->
<pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre>
</div>
END_HTML
# Don't show the usual "next steps" widget. The same options are
# proposed above the error message, just worded differently.
next_steps = ""
return templated "error"
end
def error_template_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String)
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "text/html"
env.response.status_code = status_code
locale = env.get("preferences").as(Preferences).locale
error_message = translate(locale, message)
next_steps = error_redirect_helper(env)
return templated "error"
end
# -------------------
# Atom feeds
# -------------------
macro error_atom(*args)
error_atom_helper(env, locale, {{*args}})
error_atom_helper(env, {{args.splat}})
end
def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception)
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_atom_helper(env, locale, status_code, exception.message || "")
return error_atom_helper(env, status_code, exception.message || "")
end
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
return "<error>#{exception.inspect_with_backtrace}</error>"
end
def error_atom_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String)
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
return "<error>#{message}</error>"
end
# -------------------
# JSON
# -------------------
macro error_json(*args)
error_json_helper(env, locale, {{*args}})
error_json_helper(env, {{args.splat}})
end
def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil)
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
additional_fields : Hash(String, Object) | Nil = nil
)
if exception.is_a?(InfoException)
return error_json_helper(env, locale, status_code, exception.message || "", additional_fields)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
end
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception)
return error_json_helper(env, locale, status_code, exception, nil)
end
def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil)
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
additional_fields : Hash(String, Object) | Nil = nil
)
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => message}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
def error_json_helper(env : HTTP::Server::Context, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String)
error_json_helper(env, locale, status_code, message, nil)
# -------------------
# Redirect
# -------------------
def error_redirect_helper(env : HTTP::Server::Context)
request_path = env.request.path
locale = env.get("preferences").as(Preferences).locale
if request_path.starts_with?("/search") || request_path.starts_with?("/watch") ||
request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL")
next_steps_text = translate(locale, "next_steps_error_message")
refresh = translate(locale, "next_steps_error_message_refresh")
go_to_youtube = translate(locale, "next_steps_error_message_go_to_youtube")
switch_instance = translate(locale, "Switch Invidious Instance")
return <<-END_HTML
<p style="margin-bottom: 4px;">#{next_steps_text}</p>
<ul>
<li>
<a href="#{env.request.resource}">#{refresh}</a>
</li>
<li>
<a href="/redirect?referer=#{env.get("current_page")}">#{switch_instance}</a>
</li>
<li>
<a rel="noreferrer noopener" href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a>
</li>
</ul>
END_HTML
else
return ""
end
end

69
src/invidious/helpers/handlers.cr
View File

@@ -97,18 +97,18 @@ class AuthHandler < Kemal::Handler
if token = env.request.headers["Authorization"]?
token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
session = URI.decode_www_form(token["session"].as_s)
scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil)
scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil)
if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String)
user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User)
if email = Invidious::Database::SessionIDs.select_email(session)
user = Invidious::Database::Users.select!(email: email)
end
elsif sid = env.request.cookies["SID"]?.try &.value
if sid.starts_with? "v1:"
raise "Cannot use token as SID"
end
if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String)
user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User)
if email = Invidious::Database::SessionIDs.select_email(sid)
user = Invidious::Database::Users.select!(email: email)
end
scopes = [":*"]
@@ -142,63 +142,8 @@ class APIHandler < Kemal::Handler
exclude ["/api/v1/auth/notifications"], "POST"
def call(env)
return call_next env unless only_match? env
env.response.headers["Access-Control-Allow-Origin"] = "*"
# Since /api/v1/notifications is an event-stream, we don't want
# to wrap the response
return call_next env if exclude_match? env
# Here we swap out the socket IO so we can modify the response as needed
output = env.response.output
env.response.output = IO::Memory.new
begin
call_next env
env.response.output.rewind
if env.response.output.as(IO::Memory).size != 0 &&
env.response.headers.includes_word?("Content-Type", "application/json")
response = JSON.parse(env.response.output)
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
if env.params.query["pretty"]?.try &.== "1"
response = response.to_pretty_json
else
response = response.to_json
end
else
response = env.response.output.gets_to_end
end
rescue ex
env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html")
env.response.status_code = 500
if env.response.headers.includes_word?("Content-Type", "application/json")
response = {"error" => ex.message || "Unspecified error"}
if env.params.query["pretty"]?.try &.== "1"
response = response.to_pretty_json
else
response = response.to_json
end
end
ensure
env.response.output = output
env.response.print response
env.response.flush
end
env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env)
call_next env
end
end

627
src/invidious/helpers/helpers.cr
View File

@@ -22,216 +22,6 @@ struct Annotation
property annotations : String
end
struct ConfigPreferences
include YAML::Serializable
property annotations : Bool = false
property annotations_subscribed : Bool = false
property autoplay : Bool = false
property captions : Array(String) = ["", "", ""]
property comments : Array(String) = ["youtube", ""]
property continue : Bool = false
property continue_autoplay : Bool = true
property dark_mode : String = ""
property latest_only : Bool = false
property listen : Bool = false
property local : Bool = false
property locale : String = "en-US"
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = "invidious"
property quality : String = "hd720"
property quality_dash : String = "auto"
property default_home : String? = "Popular"
property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
property related_videos : Bool = true
property sort : String = "published"
property speed : Float32 = 1.0_f32
property thin_mode : Bool = false
property unseen_only : Bool = false
property video_loop : Bool = false
property extend_desc : Bool = false
property volume : Int32 = 100
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
}
{% end %}
end
end
class Config
include YAML::Serializable
property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions)
property feed_threads : Int32 = 1 # Number of threads to use for updating feeds
property output : String = "STDOUT" # Log file path or STDOUT
property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc)
@[YAML::Field(converter: Preferences::URIConverter)]
property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax
property decrypt_polling : Bool = true # Use polling to keep decryption function up to date
property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel
property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required
property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
property popular_enabled : Bool = true
property captcha_enabled : Bool = true
property login_enabled : Bool = true
property registration_enabled : Bool = true
property statistics_enabled : Bool = false
property admins : Array(String) = [] of String
property external_port : Int32? = nil
property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("")
property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs
property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc.
property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc.
property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument)
property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument)
property bind_unix : String? = nil # Make Invidious listening on UNIX sockets - Example: /tmp/invidious.sock
property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property use_quic : Bool = true # Use quic transport for youtube api
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format
property captcha_key : String? = nil # Key for Anti-Captcha
property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha
def disabled?(option)
case disabled = CONFIG.disable_proxy
when Bool
return disabled
when Array
if disabled.includes? option
return true
else
return false
end
else
return false
end
end
def self.load
# Load config from file or YAML string env var
env_config_file = "INVIDIOUS_CONFIG_FILE"
env_config_yaml = "INVIDIOUS_CONFIG"
config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
config = Config.from_yaml(config_yaml)
# Update config from env vars (upcased and prefixed with "INVIDIOUS_")
{% for ivar in Config.instance_vars %}
{% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
if ENV.has_key?({{env_id}})
# puts %(Config.{{ivar.id}} : Loading from env var {{env_id}})
env_value = ENV.fetch({{env_id}})
success = false
# Use YAML converter if specified
{% ann = ivar.annotation(::YAML::Field) %}
{% if ann && ann[:converter] %}
puts %(Config.{{ivar.id}} : Parsing "#{env_value}" as {{ivar.type}} with {{ann[:converter]}} converter)
config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}})
success = true
# Use regular YAML parser otherwise
{% else %}
{% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %}
# Sort types to avoid parsing nulls and numbers as strings
{% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %}
{{ivar_types}}.each do |ivar_type|
if !success
begin
# puts %(Config.{{ivar.id}} : Trying to parse "#{env_value}" as #{ivar_type})
config.{{ivar.id}} = ivar_type.from_yaml(env_value)
puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}} (#{ivar_type}))
success = true
rescue
# nop
end
end
end
{% end %}
# Exit on fail
if !success
puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}})
exit(1)
end
end
{% end %}
# Build database_url from db.* if it's not set directly
if config.database_url.to_s.empty?
if db = config.db
config.database_url = URI.new(
scheme: "postgres",
user: db.user,
password: db.password,
host: db.host,
port: db.port,
path: db.dbname,
)
else
puts "Config : Either database_url or db.* is required"
exit(1)
end
end
return config
end
end
struct DBConfig
include YAML::Serializable
property user : String
property password : String
property host : String
property port : Int32
property dbname : String
end
def login_req(f_req)
data = {
# Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard
# Generally this is much longer (>1250 characters), see also
# https://github.com/ytdl-org/youtube-dl/commit/baf67a604d912722b0fe03a40e9dc5349a2208cb .
# For now this can be empty.
"bgRequest" => %|["identifier",""]|,
"pstMsg" => "1",
"checkConnection" => "youtube",
"checkedDomains" => "youtube",
"hl" => "en",
"deviceinfo" => %|[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]|,
"f.req" => f_req,
"flowName" => "GlifWebSignIn",
"flowEntry" => "ServiceLogin",
# "cookiesDisabled" => "false",
# "gmscoreversion" => "undefined",
# "continue" => "https://accounts.google.com/ManageAccount",
# "azt" => "",
# "bgHash" => "",
}
return HTTP::Params.encode(data)
end
def html_to_content(description_html : String)
description = description_html.gsub(/(<br>)|(<br\/>)/, {
"<br>": "\n",
@@ -245,287 +35,7 @@ def html_to_content(description_html : String)
return description
end
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
end
def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil)
if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
video_id = i["videoId"].as_s
title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
author_info = i["ownerText"]?.try &.["runs"]?.try &.as_a?.try &.[0]?
author = author_info.try &.["text"].as_s || author_fallback || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local
view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } ||
i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]?
.try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0
live_now = false
paid = false
premium = false
premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) }
i["badges"]?.try &.as_a.each do |badge|
b = badge["metadataBadgeRenderer"]
case b["label"].as_s
when "LIVE NOW"
live_now = true
when "New", "4K", "CC"
# TODO
when "Premium"
paid = true
# TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"]
premium = true
else nil # Ignore
end
end
SearchVideo.new({
title: title,
id: video_id,
author: author,
ucid: author_id,
published: published,
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
paid: paid,
premium: premium,
premiere_timestamp: premiere_timestamp,
})
elsif i = item["channelRenderer"]?
author = i["title"]["simpleText"]?.try &.as_s || author_fallback || ""
author_id = i["channelId"]?.try &.as_s || author_id_fallback || ""
author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try &.["url"]?.try &.as_s || ""
subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0
auto_generated = false
auto_generated = true if !i["videoCountText"]?
video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
SearchChannel.new({
author: author,
ucid: author_id,
author_thumbnail: author_thumbnail,
subscriber_count: subscriber_count,
video_count: video_count,
description_html: description_html,
auto_generated: auto_generated,
})
elsif i = item["gridPlaylistRenderer"]?
title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
plid = i["playlistId"]?.try &.as_s || ""
video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
SearchPlaylist.new({
title: title,
id: plid,
author: author_fallback || "",
ucid: author_id_fallback || "",
video_count: video_count,
videos: [] of SearchPlaylistVideo,
thumbnail: playlist_thumbnail,
})
elsif i = item["playlistRenderer"]?
title = i["title"]["simpleText"]?.try &.as_s || ""
plid = i["playlistId"]?.try &.as_s || ""
video_count = i["videoCount"]?.try &.as_s.to_i || 0
playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || ""
author_info = i["shortBylineText"]?.try &.["runs"]?.try &.as_a?.try &.[0]?
author = author_info.try &.["text"].as_s || author_fallback || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || ""
videos = i["videos"]?.try &.as_a.map do |v|
v = v["childVideoRenderer"]
v_title = v["title"]["simpleText"]?.try &.as_s || ""
v_id = v["videoId"]?.try &.as_s || ""
v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
SearchPlaylistVideo.new({
title: v_title,
id: v_id,
length_seconds: v_length_seconds,
})
end || [] of SearchPlaylistVideo
# TODO: i["publishedTimeText"]?
SearchPlaylist.new({
title: title,
id: plid,
author: author,
ucid: author_id,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail,
})
elsif i = item["radioRenderer"]? # Mix
# TODO
elsif i = item["showRenderer"]? # Show
# TODO
elsif i = item["shelfRenderer"]?
elsif i = item["horizontalCardListRenderer"]?
elsif i = item["searchPyvRenderer"]? # Ad
end
end
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
items = [] of SearchItem
channel_v2_response = initial_data
.try &.["continuationContents"]?
.try &.["gridContinuation"]?
.try &.["items"]?
if channel_v2_response
channel_v2_response.try &.as_a.each { |item|
extract_item(item, author_fallback, author_id_fallback)
.try { |t| items << t }
}
else
initial_data.try { |t| t["contents"]? || t["response"]? }
.try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] ||
t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] ||
t["continuationContents"]? }
.try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? }
.try &.["contents"].as_a
.each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a
.try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a ||
t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t }
.each { |item|
extract_item(item, author_fallback, author_id_fallback)
.try { |t| items << t }
} }
end
items
end
def check_enum(db, enum_name, struct_type = nil)
return # TODO
if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool)
LOGGER.info("check_enum: CREATE TYPE #{enum_name}")
db.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql"))
end
end
end
def check_table(db, table_name, struct_type = nil)
# Create table if it doesn't exist
begin
db.exec("SELECT * FROM #{table_name} LIMIT 0")
rescue ex
LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}")
db.using_connection do |conn|
conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql"))
end
end
return if !struct_type
struct_array = struct_type.type_array
column_array = get_column_array(db, table_name)
column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/)
.try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT")
return if !column_types
struct_array.each_with_index do |name, i|
if name != column_array[i]?
if !column_array[i]?
new_column = column_types.select { |line| line.starts_with? name }[0]
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
next
end
# Column doesn't exist
if !column_array.includes? name
new_column = column_types.select { |line| line.starts_with? name }[0]
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
end
# Column exists but in the wrong position, rotate
if struct_array.includes? column_array[i]
until name == column_array[i]
new_column = column_types.select { |line| line.starts_with? column_array[i] }[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new")
# There's a column we didn't expect
if !new_column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
column_array = get_column_array(db, table_name)
next
end
LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}")
LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}")
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}")
column_array = get_column_array(db, table_name)
end
else
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE")
end
end
end
return if column_array.size <= struct_array.size
column_array.each do |column|
if !struct_array.includes? column
LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE")
end
end
end
class PG::ResultSet
def field(index = @column_index)
@fields.not_nil![index]
end
end
def get_column_array(db, table_name)
column_array = [] of String
db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs|
rs.column_count.times do |i|
column = rs.as(PG::ResultSet).field(i)
column_array << column.name
end
end
return column_array
end
def cache_annotation(db, id, annotations)
def cache_annotation(id, annotations)
if !CONFIG.cache_annotations
return
end
@@ -543,14 +53,14 @@ def cache_annotation(db, id, annotations)
end
end
db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations
Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations
end
def create_notification_stream(env, topics, connection_channel)
connection = Channel(PQ::Notification).new(8)
connection_channel.send({true, connection})
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
locale = env.get("preferences").as(Preferences).locale
since = env.params.query["since"]?.try &.to_i?
id = 0
@@ -564,18 +74,9 @@ def create_notification_stream(env, topics, connection_channel)
published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
video_id = TEST_IDS[rand(TEST_IDS.size)]
video = get_video(video_id, PG_DB)
video = get_video(video_id)
video.published = published
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
response = JSON.parse(video.to_json(locale, nil))
env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}"
@@ -595,22 +96,14 @@ def create_notification_stream(env, topics, connection_channel)
spawn do
begin
if since
since_unix = Time.unix(since.not_nil!)
topics.try &.each do |topic|
case topic
when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video|
Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}"
env.response.puts
@@ -640,18 +133,9 @@ def create_notification_stream(env, topics, connection_channel)
next
end
video = get_video(video_id, PG_DB)
video = get_video(video_id)
video.published = Time.unix(published)
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
response = JSON.parse(video.to_json(locale, nil))
env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}"
@@ -698,84 +182,19 @@ def proxy_file(response, env)
end
end
# See https://github.com/kemalcr/kemal/pull/576
class HTTP::Server::Response::Output
def close
return if closed?
# Fetch the playback requests tracker from the statistics endpoint.
#
# Creates a new tracker when unavailable.
def get_playback_statistic
if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
tracker = {
"totalRequests" => 0_i64,
"successfulRequests" => 0_i64,
"ratio" => 0_f64,
}
unless response.wrote_headers?
response.content_length = @out_count
end
ensure_headers_written
super
if @chunked
@io << "0\r\n\r\n"
@io.flush
end
end
end
class HTTP::Client::Response
def pipe(io)
HTTP.serialize_body(io, headers, @body, @body_io, @version)
end
end
# Supports serialize_body without first writing headers
module HTTP
def self.serialize_body(io, headers, body, body_io, version)
if body
io << body
elsif body_io
content_length = content_length(headers)
if content_length
copied = IO.copy(body_io, io)
if copied != content_length
raise ArgumentError.new("Content-Length header is #{content_length} but body had #{copied} bytes")
end
elsif Client::Response.supports_chunked?(version)
headers["Transfer-Encoding"] = "chunked"
serialize_chunked_body(io, body_io)
else
io << body
end
end
end
end
class HTTP::Client
property family : Socket::Family = Socket::Family::UNSPEC
private def socket
socket = @socket
return socket if socket
hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host
socket = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, @family
socket.read_timeout = @read_timeout if @read_timeout
socket.sync = false
{% if !flag?(:without_openssl) %}
if tls = @tls
socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: @host)
end
{% end %}
@socket = socket
end
end
class TCPSocket
def initialize(host, port, dns_timeout = nil, connect_timeout = nil, family = Socket::Family::UNSPEC)
Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
super(addrinfo.family, addrinfo.type, addrinfo.protocol)
connect(addrinfo, timeout: connect_timeout) do |error|
close
error
end
end
Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
end
return tracker.as(Hash(String, Int64 | Float64))
end

218
src/invidious/helpers/i18n.cr
View File

@@ -1,72 +1,185 @@
LOCALES = {
"ar" => load_locale("ar"),
"de" => load_locale("de"),
"el" => load_locale("el"),
"en-US" => load_locale("en-US"),
"eo" => load_locale("eo"),
"es" => load_locale("es"),
"fa" => load_locale("fa"),
"fi" => load_locale("fi"),
"fr" => load_locale("fr"),
"he" => load_locale("he"),
"hr" => load_locale("hr"),
"id" => load_locale("id"),
"is" => load_locale("is"),
"it" => load_locale("it"),
"ja" => load_locale("ja"),
"nb-NO" => load_locale("nb-NO"),
"nl" => load_locale("nl"),
"pl" => load_locale("pl"),
"pt-BR" => load_locale("pt-BR"),
"pt-PT" => load_locale("pt-PT"),
"ro" => load_locale("ro"),
"ru" => load_locale("ru"),
"sv" => load_locale("sv-SE"),
"tr" => load_locale("tr"),
"uk" => load_locale("uk"),
"zh-CN" => load_locale("zh-CN"),
"zh-TW" => load_locale("zh-TW"),
# Languages requiring a better level of translation (at least 20%)
# to be added to the list below:
#
# "af" => "", # Afrikaans
# "az" => "", # Azerbaijani
# "be" => "", # Belarusian
# "bn_BD" => "", # Bengali (Bangladesh)
# "ia" => "", # Interlingua
# "or" => "", # Odia
# "tk" => "", # Turkmen
# "tok => "", # Toki Pona
#
LOCALES_LIST = {
"ar" => "العربية", # Arabic
"bg" => "български", # Bulgarian
"bn" => "বাংলা", # Bengali
"ca" => "Català", # Catalan
"cs" => "Čeština", # Czech
"cy" => "Cymraeg", # Welsh
"da" => "Dansk", # Danish
"de" => "Deutsch", # German
"el" => "Ελληνικά", # Greek
"en-US" => "English", # English
"eo" => "Esperanto", # Esperanto
"es" => "Español", # Spanish
"et" => "Eesti keel", # Estonian
"eu" => "Euskara", # Basque
"fa" => "فارسی", # Persian
"fi" => "Suomi", # Finnish
"fr" => "Français", # French
"he" => "עברית", # Hebrew
"hi" => "हिन्दी", # Hindi
"hr" => "Hrvatski", # Croatian
"hu-HU" => "Magyar Nyelv", # Hungarian
"id" => "Bahasa Indonesia", # Indonesian
"is" => "Íslenska", # Icelandic
"it" => "Italiano", # Italian
"ja" => "日本語", # Japanese
"ko" => "한국어", # Korean
"lmo" => "Lombard", # Lombard
"lt" => "Lietuvių", # Lithuanian
"nb-NO" => "Norsk bokmål", # Norwegian Bokmål
"nl" => "Nederlands", # Dutch
"pl" => "Polski", # Polish
"pt" => "Português", # Portuguese
"pt-BR" => "Português Brasileiro", # Portuguese (Brazil)
"pt-PT" => "Português de Portugal", # Portuguese (Portugal)
"ro" => "Română", # Romanian
"ru" => "Русский", # Russian
"si" => "සිංහල", # Sinhala
"sk" => "Slovenčina", # Slovak
"sl" => "Slovenščina", # Slovenian
"sq" => "Shqip", # Albanian
"sr" => "Srpski (latinica)", # Serbian (Latin)
"sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic)
"sv-SE" => "Svenska", # Swedish
"tr" => "Türkçe", # Turkish
"uk" => "Українська", # Ukrainian
"vi" => "Tiếng Việt", # Vietnamese
"zh-CN" => "汉语", # Chinese (Simplified)
"zh-TW" => "漢語", # Chinese (Traditional)
}
def load_locale(name)
return JSON.parse(File.read("locales/#{name}.json")).as_h
LOCALES = load_all_locales()
CONTENT_REGIONS = {
"AE", "AR", "AT", "AU", "AZ", "BA", "BD", "BE", "BG", "BH", "BO", "BR", "BY",
"CA", "CH", "CL", "CO", "CR", "CY", "CZ", "DE", "DK", "DO", "DZ", "EC", "EE",
"EG", "ES", "FI", "FR", "GB", "GE", "GH", "GR", "GT", "HK", "HN", "HR", "HU",
"ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KR", "KW",
"KZ", "LB", "LI", "LK", "LT", "LU", "LV", "LY", "MA", "ME", "MK", "MT", "MX",
"MY", "NG", "NI", "NL", "NO", "NP", "NZ", "OM", "PA", "PE", "PG", "PH", "PK",
"PL", "PR", "PT", "PY", "QA", "RO", "RS", "RU", "SA", "SE", "SG", "SI", "SK",
"SN", "SV", "TH", "TN", "TR", "TW", "TZ", "UA", "UG", "US", "UY", "VE", "VN",
"YE", "ZA", "ZW",
}
# Enum for the different types of number formats
enum NumberFormatting
None # Print the number as-is
Separator # Use a separator for thousands
Short # Use short notation (k/M/B)
HtmlSpan # Surround with <span id="count"></span>
end
def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text : String | Nil = nil)
# if locale && !locale[translation]?
# puts "Could not find translation for #{translation.dump}"
# end
def load_all_locales
locales = {} of String => Hash(String, JSON::Any)
if locale && locale[translation]?
case locale[translation]
when .as_h?
match_length = 0
LOCALES_LIST.each_key do |name|
locales[name] = JSON.parse(File.read("locales/#{name}.json")).as_h
end
locale[translation].as_h.each do |key, value|
if md = text.try &.match(/#{key}/)
return locales
end
def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String
# Log a warning if "key" doesn't exist in en-US locale and return
# that key as the text, so this is more or less transparent to the user.
if !LOCALES["en-US"].has_key?(key)
LOGGER.warn("i18n: Missing translation key \"#{key}\"")
return key
end
# Default to english, whenever the locale doesn't exist,
# or the key requested has not been translated
if locale && LOCALES.has_key?(locale) && LOCALES[locale].has_key?(key)
raw_data = LOCALES[locale][key]
else
raw_data = LOCALES["en-US"][key]
end
case raw_data
when .as_h?
# Init
translation = ""
match_length = 0
raw_data.as_h.each do |hash_key, value|
if text.is_a?(String)
if md = text.try &.match(/#{hash_key}/)
if md[0].size >= match_length
translation = value.as_s
match_length = md[0].size
end
end
end
when .as_s?
if !locale[translation].as_s.empty?
translation = locale[translation].as_s
end
else
raise "Invalid translation #{translation}"
end
when .as_s?
translation = raw_data.as_s
else
raise "Invalid translation \"#{raw_data}\""
end
if text
if text.is_a?(String)
translation = translation.gsub("`x`", text)
elsif text.is_a?(Hash(String, String))
# adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic
text.each_key do |hash_key|
translation = translation.gsub("{{#{hash_key}}}", text[hash_key])
end
end
return translation
end
def translate_bool(locale : Hash(String, JSON::Any) | Nil, translation : Bool)
def translate_count(locale : String, key : String, count : Int, format = NumberFormatting::None) : String
# Fallback on english if locale doesn't exist
locale = "en-US" if !LOCALES.has_key?(locale)
# Retrieve suffix
suffix = I18next::Plurals::RESOLVER.get_suffix(locale, count)
plural_key = key + suffix
if LOCALES[locale].has_key?(plural_key)
translation = LOCALES[locale][plural_key].as_s
else
# Try #1: Fallback to singular in the same locale
singular_suffix = I18next::Plurals::RESOLVER.get_suffix(locale, 1)
if LOCALES[locale].has_key?(key + singular_suffix)
translation = LOCALES[locale][key + singular_suffix].as_s
elsif locale != "en-US"
# Try #2: Fallback to english
translation = translate_count("en-US", key, count)
else
# Return key if we're already in english, as the translation is missing
LOGGER.warn("i18n: Missing translation key \"#{key}\"")
return key
end
end
case format
when .separator? then count_txt = number_with_separator(count)
when .short? then count_txt = number_to_short_text(count)
when .html_span? then count_txt = "<span id=\"count\">" + count.to_s + "</span>"
else count_txt = count.to_s
end
return translation.gsub("{{count}}", count_txt)
end
def translate_bool(locale : String?, translation : Bool)
case translation
when true
return translate(locale, "Yes")
@@ -74,3 +187,12 @@ def translate_bool(locale : Hash(String, JSON::Any) | Nil, translation : Bool)
return translate(locale, "No")
end
end
def locale_is_rtl?(locale : String?)
# Fallback to en-US
return false if locale.nil?
# Arabic, Persian, Hebrew
# See https://en.wikipedia.org/wiki/Right-to-left_script#List_of_RTL_scripts
return {"ar", "fa", "he"}.includes? locale
end

566
src/invidious/helpers/i18next.cr Normal file
View File

@@ -0,0 +1,566 @@
# I18next-compatible implementation of plural forms
#
module I18next::Plurals
# -----------------------------------
# I18next plural forms definition
# -----------------------------------
enum PluralForms
# One singular, one plural forms
Single_gt_one = 1 # E.g: French
Single_not_one = 2 # E.g: English
# No plural forms (E.g: Azerbaijani)
None = 3
# One singular, two plural forms
Dual_Slavic = 4 # E.g: Russian
# Special cases (rules used by only one or two language(s))
Special_Arabic = 5
Special_Czech_Slovak = 6
Special_Polish_Kashubian = 7
Special_Welsh = 8
Special_Irish = 10
Special_Scottish_Gaelic = 11
Special_Icelandic = 12
Special_Javanese = 13
Special_Cornish = 14
Special_Lithuanian = 15
Special_Latvian = 16
Special_Macedonian = 17
Special_Mandinka = 18
Special_Maltese = 19
Special_Romanian = 20
Special_Slovenian = 21
Special_Hebrew = 22
Special_Odia = 23
# Mixed v3/v4 rules in Weblate
# `es`, `pt` and `pt-PT` doesn't seem to have been refreshed
# by weblate yet, but I suspect it will happen one day.
# See: https://github.com/translate/translate/issues/4873
Special_French_Portuguese
Special_Hungarian_Serbian
Special_Spanish_Italian
end
private PLURAL_SETS = {
PluralForms::Single_gt_one => [
"ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg",
"mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa",
],
PluralForms::Single_not_one => [
"af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en",
"eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi",
"hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr",
"nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms",
"ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw",
"ta", "te", "tk", "ur", "yo",
],
PluralForms::None => [
"ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky",
"lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh",
],
PluralForms::Dual_Slavic => [
"be", "bs", "cnr", "dz", "ru", "uk",
],
}
private PLURAL_SINGLES = {
"ar" => PluralForms::Special_Arabic,
"cs" => PluralForms::Special_Czech_Slovak,
"csb" => PluralForms::Special_Polish_Kashubian,
"cy" => PluralForms::Special_Welsh,
"ga" => PluralForms::Special_Irish,
"gd" => PluralForms::Special_Scottish_Gaelic,
"he" => PluralForms::Special_Hebrew,
"is" => PluralForms::Special_Icelandic,
"iw" => PluralForms::Special_Hebrew,
"jv" => PluralForms::Special_Javanese,
"kw" => PluralForms::Special_Cornish,
"lt" => PluralForms::Special_Lithuanian,
"lv" => PluralForms::Special_Latvian,
"mk" => PluralForms::Special_Macedonian,
"mnk" => PluralForms::Special_Mandinka,
"mt" => PluralForms::Special_Maltese,
"or" => PluralForms::Special_Odia,
"pl" => PluralForms::Special_Polish_Kashubian,
"ro" => PluralForms::Special_Romanian,
"sk" => PluralForms::Special_Czech_Slovak,
"sl" => PluralForms::Special_Slovenian,
# Mixed v3/v4 rules
"es" => PluralForms::Special_Spanish_Italian,
"fr" => PluralForms::Special_French_Portuguese,
"hr" => PluralForms::Special_Hungarian_Serbian,
"it" => PluralForms::Special_Spanish_Italian,
"pt" => PluralForms::Special_French_Portuguese,
"sr" => PluralForms::Special_Hungarian_Serbian,
}
# These are the v1 and v2 compatible suffixes.
# The array indices matches the PluralForms enum above.
private NUMBERS = [
[1, 2], # 1
[1, 2], # 2
[1], # 3
[1, 2, 5], # 4
[0, 1, 2, 3, 11, 100], # 5
[1, 2, 5], # 6
[1, 2, 5], # 7
[1, 2, 3, 8], # 8
[1, 2], # 9 (not used)
[1, 2, 3, 7, 11], # 10
[1, 2, 3, 20], # 11
[1, 2], # 12
[0, 1], # 13
[1, 2, 3, 4], # 14
[1, 2, 10], # 15
[1, 2, 0], # 16
[1, 2], # 17
[0, 1, 2], # 18
[1, 2, 11, 20], # 19
[1, 2, 20], # 20
[5, 1, 2, 3], # 21
[1, 2, 20, 21], # 22
[2, 1], # 23 (Odia)
]
# -----------------------------------
# I18next plural resolver class
# -----------------------------------
RESOLVER = Resolver.new
class Resolver
private property forms = {} of String => PluralForms
property version : UInt8 = 3
# Options
property simplify_plural_suffix : Bool = true
def initialize(version : Int = 3)
# Sanity checks
# V4 isn't supported, as it requires a full CLDR database.
if version > 4 || version == 0
raise "Invalid i18next version: v#{version}."
elsif version == 4
# Logger.error("Unsupported i18next version: v4. Falling back to v3")
@version = 3_u8
else
@version = version.to_u8
end
self.init_rules
end
def init_rules
# Look into sets
PLURAL_SETS.each do |form, langs|
langs.each { |lang| self.forms[lang] = form }
end
# Add plurals from the "singles" set
self.forms.merge!(PLURAL_SINGLES)
end
def get_plural_form(locale : String) : PluralForms
# Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code
if !locale.matches?(/^pt-PT$/)
locale = locale.split('-')[0]
end
return self.forms[locale] if self.forms[locale]?
# If nothing was found, then use the most common form, i.e
# one singular and one plural, as in english. Not perfect,
# but better than yielding an exception at the user.
return PluralForms::Single_not_one
end
def get_suffix(locale : String, count : Int) : String
# Checked count must be absolute. In i18next, `rule.noAbs` is used to
# determine if comparison should be done on a signed or unsigned integer,
# but this variable is never set, resulting in the comparison always
# being done on absolute numbers.
return get_suffix_retrocompat(locale, count.abs)
end
# Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check
# from original i18next code
private def simple_plural?(form : PluralForms) : Bool
case form
when .single_gt_one? then return true
when .single_not_one? then return true
when .special_icelandic? then return true
when .special_macedonian? then return true
else
return false
end
end
private def get_suffix_retrocompat(locale : String, count : Int) : String
# Get plural form
plural_form = get_plural_form(locale)
# Languages with no plural have the "_0" suffix
return "_0" if plural_form.none?
# Get the index and suffix for this number
idx = SuffixIndex.get_index(plural_form, count)
# Simple plurals are handled differently in all versions (but v4)
if @simplify_plural_suffix && simple_plural?(plural_form)
return (idx == 1) ? "_plural" : ""
end
# More complex plurals
# TODO: support v1 and v2
# TODO: support `options.prepend` (v2 and v3)
# this.options.prepend && suffix.toString() ? this.options.prepend + suffix.toString() : suffix.toString()
#
# case @version
# when 1
# suffix = SUFFIXES_V1_V2[plural_form.to_i][idx]
# return (suffix == 1) ? "" : return "_plural_#{suffix}"
# when 2
# return "_#{suffix}"
# else # v3
return "_#{idx}"
# end
end
end
# -----------------------------
# Plural functions
# -----------------------------
module SuffixIndex
def self.get_index(plural_form : PluralForms, count : Int) : UInt8
case plural_form
when .single_gt_one? then return (count > 1) ? 1_u8 : 0_u8
when .single_not_one? then return (count != 1) ? 1_u8 : 0_u8
when .none? then return 0_u8
when .dual_slavic? then return dual_slavic(count)
when .special_arabic? then return special_arabic(count)
when .special_czech_slovak? then return special_czech_slovak(count)
when .special_polish_kashubian? then return special_polish_kashubian(count)
when .special_welsh? then return special_welsh(count)
when .special_irish? then return special_irish(count)
when .special_scottish_gaelic? then return special_scottish_gaelic(count)
when .special_icelandic? then return special_icelandic(count)
when .special_javanese? then return special_javanese(count)
when .special_cornish? then return special_cornish(count)
when .special_lithuanian? then return special_lithuanian(count)
when .special_latvian? then return special_latvian(count)
when .special_macedonian? then return special_macedonian(count)
when .special_mandinka? then return special_mandinka(count)
when .special_maltese? then return special_maltese(count)
when .special_romanian? then return special_romanian(count)
when .special_slovenian? then return special_slovenian(count)
when .special_hebrew? then return special_hebrew(count)
when .special_odia? then return special_odia(count)
# Mixed v3/v4 forms
when .special_spanish_italian? then return special_cldr_spanish_italian(count)
when .special_french_portuguese? then return special_cldr_french_portuguese(count)
when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count)
else
# default, if nothing matched above
return 0_u8
end
end
# Plural form of Slavic languages (E.g: Russian)
#
# Corresponds to i18next rule #4
# Rule: (n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2)
#
def self.dual_slavic(count : Int) : UInt8
n_mod_10 = count % 10
n_mod_100 = count % 100
if n_mod_10 == 1 && n_mod_100 != 11
return 0_u8
elsif n_mod_10 >= 2 && n_mod_10 <= 4 && (n_mod_100 < 10 || n_mod_100 >= 20)
return 1_u8
else
return 2_u8
end
end
# Plural form for Arabic language
#
# Corresponds to i18next rule #5
# Rule: (n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5)
#
def self.special_arabic(count : Int) : UInt8
return count.to_u8 if (count == 0 || count == 1 || count == 2)
n_mod_100 = count % 100
return 3_u8 if (n_mod_100 >= 3 && n_mod_100 <= 10)
return 4_u8 if (n_mod_100 >= 11)
return 5_u8
end
# Plural form for Czech and Slovak languages
#
# Corresponds to i18next rule #6
# Rule: ((n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2)
#
def self.special_czech_slovak(count : Int) : UInt8
return 0_u8 if (count == 1)
return 1_u8 if (count >= 2 && count <= 4)
return 2_u8
end
# Plural form for Polish and Kashubian languages
#
# Corresponds to i18next rule #7
# Rule: (n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2)
#
def self.special_polish_kashubian(count : Int) : UInt8
return 0_u8 if (count == 1)
n_mod_10 = count % 10
n_mod_100 = count % 100
if n_mod_10 >= 2 && n_mod_10 <= 4 && (n_mod_100 < 10 || n_mod_100 >= 20)
return 1_u8
else
return 2_u8
end
end
# Plural form for Welsh language
#
# Corresponds to i18next rule #8
# Rule: ((n==1) ? 0 : (n==2) ? 1 : (n != 8 && n != 11) ? 2 : 3)
#
def self.special_welsh(count : Int) : UInt8
return 0_u8 if (count == 1)
return 1_u8 if (count == 2)
return 2_u8 if (count != 8 && count != 11)
return 3_u8
end
# Plural form for Irish language
#
# Corresponds to i18next rule #10
# Rule: (n==1 ? 0 : n==2 ? 1 : n<7 ? 2 : n<11 ? 3 : 4)
#
def self.special_irish(count : Int) : UInt8
return 0_u8 if (count == 1)
return 1_u8 if (count == 2)
return 2_u8 if (count < 7)
return 3_u8 if (count < 11)
return 4_u8
end
# Plural form for Gaelic language
#
# Corresponds to i18next rule #11
# Rule: ((n==1 || n==11) ? 0 : (n==2 || n==12) ? 1 : (n > 2 && n < 20) ? 2 : 3)
#
def self.special_scottish_gaelic(count : Int) : UInt8
return 0_u8 if (count == 1 || count == 11)
return 1_u8 if (count == 2 || count == 12)
return 2_u8 if (count > 2 && count < 20)
return 3_u8
end
# Plural form for Icelandic language
#
# Corresponds to i18next rule #12
# Rule: (n%10!=1 || n%100==11)
#
def self.special_icelandic(count : Int) : UInt8
if (count % 10) != 1 || (count % 100) == 11
return 1_u8
else
return 0_u8
end
end
# Plural form for Javanese language
#
# Corresponds to i18next rule #13
# Rule: (n !== 0)
#
def self.special_javanese(count : Int) : UInt8
return (count != 0) ? 1_u8 : 0_u8
end
# Plural form for Cornish language
#
# Corresponds to i18next rule #14
# Rule: ((n==1) ? 0 : (n==2) ? 1 : (n == 3) ? 2 : 3)
#
def self.special_cornish(count : Int) : UInt8
return 0_u8 if count == 1
return 1_u8 if count == 2
return 2_u8 if count == 3
return 3_u8
end
# Plural form for Lithuanian language
#
# Corresponds to i18next rule #15
# Rule: (n%10==1 && n%100!=11 ? 0 : n%10>=2 && (n%100<10 || n%100>=20) ? 1 : 2)
#
def self.special_lithuanian(count : Int) : UInt8
n_mod_10 = count % 10
n_mod_100 = count % 100
if n_mod_10 == 1 && n_mod_100 != 11
return 0_u8
elsif n_mod_10 >= 2 && (n_mod_100 < 10 || n_mod_100 >= 20)
return 1_u8
else
return 2_u8
end
end
# Plural form for Latvian language
#
# Corresponds to i18next rule #16
# Rule: (n%10==1 && n%100!=11 ? 0 : n !== 0 ? 1 : 2)
#
def self.special_latvian(count : Int) : UInt8
if (count % 10) == 1 && (count % 100) != 11
return 0_u8
elsif count != 0
return 1_u8
else
return 2_u8
end
end
# Plural form for Macedonian language
#
# Corresponds to i18next rule #17
# Rule: (n==1 || n%10==1 && n%100!=11 ? 0 : 1)
#
def self.special_macedonian(count : Int) : UInt8
if count == 1 || ((count % 10) == 1 && (count % 100) != 11)
return 0_u8
else
return 1_u8
end
end
# Plural form for Mandinka language
#
# Corresponds to i18next rule #18
# Rule: (n==0 ? 0 : n==1 ? 1 : 2)
#
def self.special_mandinka(count : Int) : UInt8
return (count == 0 || count == 1) ? count.to_u8 : 2_u8
end
# Plural form for Maltese language
#
# Corresponds to i18next rule #19
# Rule: (n==1 ? 0 : n==0 || ( n%100>1 && n%100<11) ? 1 : (n%100>10 && n%100<20 ) ? 2 : 3)
#
def self.special_maltese(count : Int) : UInt8
return 0_u8 if count == 1
return 1_u8 if count == 0
n_mod_100 = count % 100
return 1_u8 if (n_mod_100 > 1 && n_mod_100 < 11)
return 2_u8 if (n_mod_100 > 10 && n_mod_100 < 20)
return 3_u8
end
# Plural form for Romanian language
#
# Corresponds to i18next rule #20
# Rule: (n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < 20)) ? 1 : 2)
#
def self.special_romanian(count : Int) : UInt8
return 0_u8 if count == 1
return 1_u8 if count == 0
n_mod_100 = count % 100
return 1_u8 if (n_mod_100 > 0 && n_mod_100 < 20)
return 2_u8
end
# Plural form for Slovenian language
#
# Corresponds to i18next rule #21
# Rule: (n%100==1 ? 1 : n%100==2 ? 2 : n%100==3 || n%100==4 ? 3 : 0)
#
def self.special_slovenian(count : Int) : UInt8
n_mod_100 = count % 100
return 1_u8 if (n_mod_100 == 1)
return 2_u8 if (n_mod_100 == 2)
return 3_u8 if (n_mod_100 == 3 || n_mod_100 == 4)
return 0_u8
end
# Plural form for Hebrew language
#
# Corresponds to i18next rule #22
# Rule: (n==1 ? 0 : n==2 ? 1 : (n<0 || n>10) && n%10==0 ? 2 : 3)
#
def self.special_hebrew(count : Int) : UInt8
return 0_u8 if (count == 1)
return 1_u8 if (count == 2)
if (count < 0 || count > 10) && (count % 10) == 0
return 2_u8
else
return 3_u8
end
end
# Plural form for Odia ("or") language
#
# This one is a bit special. It should use rule #2 (like english)
# but the "numbers" (suffixes?) it has are inverted, so we'll make a
# special rule for it.
#
def self.special_odia(count : Int) : UInt8
return (count == 1) ? 0_u8 : 1_u8
end
# -------------------
# "v3.5" rules
# -------------------
# Plural form for Spanish & Italian languages
#
# This rule is mostly compliant to CLDR v42
#
def self.special_cldr_spanish_italian(count : Int) : UInt8
return 0_u8 if (count == 1) # one
return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many
return 2_u8 # other
end
# Plural form for French and Portuguese
#
# This rule is mostly compliant to CLDR v42
#
def self.special_cldr_french_portuguese(count : Int) : UInt8
return 0_u8 if (count == 0 || count == 1) # one
return 1_u8 if (count % 1_000_000 == 0) # many
return 2_u8 # other
end
# Plural form for Hungarian and Serbian
#
# This rule is mostly compliant to CLDR v42
#
def self.special_cldr_hungarian_serbian(count : Int) : UInt8
n_mod_10 = count % 10
n_mod_100 = count % 100
return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one
return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few
return 2_u8 # other
end
end
end

248
src/invidious/helpers/json_filter.cr
View File

@@ -1,248 +0,0 @@
module JSONFilter
alias BracketIndex = Hash(Int64, Int64)
alias GroupedFieldsValue = String | Array(GroupedFieldsValue)
alias GroupedFieldsList = Array(GroupedFieldsValue)
class FieldsParser
class ParseError < Exception
end
# Returns the `Regex` pattern used to match nest groups
def self.nest_group_pattern : Regex
# uses a '.' character to match json keys as they are allowed
# to contain any unicode codepoint
/(?:|,)(?<groupname>[^,\n]*?)\(/
end
# Returns the `Regex` pattern used to check if there are any empty nest groups
def self.unnamed_nest_group_pattern : Regex
/^\(|\(\(|\/\(/
end
def self.parse_fields(fields_text : String) : Nil
if fields_text.empty?
raise FieldsParser::ParseError.new "Fields is empty"
end
opening_bracket_count = fields_text.count('(')
closing_bracket_count = fields_text.count(')')
if opening_bracket_count != closing_bracket_count
bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing"
raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})"
elsif match_result = unnamed_nest_group_pattern.match(fields_text)
raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}"
end
# first, handle top-level single nested properties: items/id, playlistItems/snippet, etc
parse_single_nests(fields_text) { |nest_list| yield nest_list }
# next, handle nest groups: items(id, etag, etc)
parse_nest_groups(fields_text) { |nest_list| yield nest_list }
end
def self.parse_single_nests(fields_text : String) : Nil
single_nests = remove_nest_groups(fields_text)
if !single_nests.empty?
property_nests = single_nests.split(',')
property_nests.each do |nest|
nest_list = nest.split('/')
if nest_list.includes? ""
raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}"
end
yield nest_list
end
# else
# raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}"
end
end
def self.parse_nest_groups(fields_text : String) : Nil
nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
bracket_pairs = get_bracket_pairs(fields_text, true)
text_index = 0
regex_index = 0
while regex_result = self.nest_group_pattern.match(fields_text, regex_index)
raw_match = regex_result[0]
group_name = regex_result["groupname"]
text_index = regex_result.begin
regex_index = regex_result.end
if text_index.nil? || regex_index.nil?
raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}"
end
offset = raw_match.starts_with?(',') ? 1 : 0
opening_bracket_index = (text_index + group_name.size) + offset
closing_bracket_index = bracket_pairs[opening_bracket_index]
content_start = opening_bracket_index + 1
content = fields_text[content_start...closing_bracket_index]
if content.empty?
raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}"
else
content = remove_nest_groups(content)
end
while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index]
if nest_stack.size
nest_stack.pop
end
end
group_name.split('/').each do |group_name|
nest_stack.push({
group_name: group_name,
closing_bracket_index: closing_bracket_index,
})
end
if !content.empty?
properties = content.split(',')
properties.each do |prop|
nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] }
if !prop.empty?
if prop.includes?('/')
parse_single_nests(prop) { |list| nest_list += list }
else
nest_list.push prop
end
else
raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}"
end
yield nest_list
end
end
end
end
def self.remove_nest_groups(text : String) : String
content_bracket_pairs = get_bracket_pairs(text, false)
content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket|
closing_bracket = content_bracket_pairs[opening_bracket]
last_comma = text.rindex(',', opening_bracket) || 0
text = text[0...last_comma] + text[closing_bracket + 1...text.size]
end
return text.starts_with?(',') ? text[1...text.size] : text
end
def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex
istart = [] of Int64
bracket_index = BracketIndex.new
text.each_char_with_index do |char, index|
if char == '('
istart.push(index.to_i64)
end
if char == ')'
begin
opening = istart.pop
if recursive || (!recursive && istart.size == 0)
bracket_index[opening] = index.to_i64
end
rescue
raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}"
end
end
end
if istart.size != 0
idx = istart.pop
raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}"
end
return bracket_index
end
end
class FieldsGrouper
alias SkeletonValue = Hash(String, SkeletonValue)
def self.create_json_skeleton(fields_text : String) : SkeletonValue
root_hash = {} of String => SkeletonValue
FieldsParser.parse_fields(fields_text) do |nest_list|
current_item = root_hash
nest_list.each do |key|
if current_item[key]?
current_item = current_item[key]
else
current_item[key] = {} of String => SkeletonValue
current_item = current_item[key]
end
end
end
root_hash
end
def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList
grouped_fields_list = GroupedFieldsList.new
json_skeleton.each do |key, value|
grouped_fields_list.push key
nested_keys = create_grouped_fields_list(value)
grouped_fields_list.push nested_keys unless nested_keys.empty?
end
return grouped_fields_list
end
end
class FilterError < Exception
end
def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true)
skeleton = FieldsGrouper.create_json_skeleton(fields_text)
grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton)
filter(item, grouped_fields_list, in_place)
end
def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any
item = item.clone unless in_place
if !item.as_h? && !item.as_a?
raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}"
end
top_level_keys = Array(String).new
grouped_fields_list.each do |value|
if value.is_a? String
top_level_keys.push value
elsif value.is_a? Array
if !top_level_keys.empty?
key_to_filter = top_level_keys.last
if item.as_h?
filter(item[key_to_filter], value, in_place: true)
elsif item.as_a?
item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) }
end
else
raise FilterError.new "Tried to filter while top level keys list is empty"
end
end
end
if item.as_h?
item.as_h.select! top_level_keys
elsif item.as_a?
item.as_a.map { |value| filter(value, top_level_keys, in_place: true) }
end
item
end
end

43
src/invidious/helpers/logger.cr
View File

@@ -1,3 +1,5 @@
require "colorize"
enum LogLevel
All = 0
Trace = 1
@@ -10,40 +12,53 @@ enum LogLevel
end
class Invidious::LogHandler < Kemal::BaseLogHandler
def initialize(@io : IO = STDOUT, @level = LogLevel::Debug)
def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true)
Colorize.enabled = use_color
Colorize.on_tty_only!
end
def call(context : HTTP::Server::Context)
elapsed_time = Time.measure { call_next(context) }
elapsed_text = elapsed_text(elapsed_time)
info("#{context.response.status_code} #{context.request.method} #{context.request.resource} #{elapsed_text}")
# Default: full path with parameters
requested_url = context.request.resource
# Try not to log search queries passed as GET parameters during normal use
# (They will still be logged if log level is 'Debug' or 'Trace')
if @level > LogLevel::Debug && (
requested_url.downcase.includes?("search") || requested_url.downcase.includes?("q=")
)
# Log only the path
requested_url = context.request.path
end
info("#{context.response.status_code} #{context.request.method} #{requested_url} #{elapsed_text}")
context
end
def puts(message : String)
@io << message << '\n'
@io.flush
end
def write(message : String)
@io << message
@io.flush
end
def set_log_level(level : String)
@level = LogLevel.parse(level)
end
def set_log_level(level : LogLevel)
@level = level
def color(level)
case level
when LogLevel::Trace then :cyan
when LogLevel::Debug then :green
when LogLevel::Info then :white
when LogLevel::Warn then :yellow
when LogLevel::Error then :red
when LogLevel::Fatal then :magenta
else :default
end
end
{% for level in %w(trace debug info warn error fatal) %}
def {{level.id}}(message : String)
if LogLevel::{{level.id.capitalize}} >= @level
puts("#{Time.utc} [{{level.id}}] #{message}")
puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})))
end
end
{% end %}

21
src/invidious/helpers/macros.cr
View File

@@ -48,11 +48,26 @@ module JSON::Serializable
end
end
macro templated(filename, template = "template", navbar_search = true)
macro templated(_filename, template = "template", navbar_search = true)
navbar_search = {{navbar_search}}
render "src/invidious/views/#{{{filename}}}.ecr", "src/invidious/views/#{{{template}}}.ecr"
{{ filename = "src/invidious/views/" + _filename + ".ecr" }}
{{ layout = "src/invidious/views/" + template + ".ecr" }}
__content_filename__ = {{filename}}
content = Kilt.render({{filename}})
Kilt.render({{layout}})
end
macro rendered(filename)
render "src/invidious/views/#{{{filename}}}.ecr"
Kilt.render("src/invidious/views/#{{{filename}}}.ecr")
end
# Similar to Kemals halt method but works in a
# method.
macro haltf(env, status_code = 200, response = "")
{{env}}.response.status_code = {{status_code}}
{{env}}.response.print {{response}}
{{env}}.response.close
return
end

316
src/invidious/helpers/proxy.cr
View File

File diff suppressed because one or more lines are too long

317
src/invidious/helpers/serialized_yt_data.cr Normal file
View File

@@ -0,0 +1,317 @@
@[Flags]
enum VideoBadges
LiveNow
Premium
ThreeD
FourK
New
EightK
VR180
VR360
ClosedCaptions
end
struct SearchVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property premiere_timestamp : Time?
property author_verified : Bool
property badges : VideoBadges
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
xml.element("media:community") do
xml.element("media:statistics", views: self.views)
end
end
end
def to_xml(auto_generated, query_params, _xml : Nil)
XML.build do |xml|
to_xml(auto_generated, query_params, xml)
end
end
def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorVerified", self.author_verified
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
json.field "viewCountText", translate_count(locale, "generic_views_count", self.views, NumberFormatting::Short)
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.badges.live_now?
json.field "premium", self.badges.premium?
json.field "isUpcoming", self.upcoming?
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
json.field "isNew", self.badges.new?
json.field "is4k", self.badges.four_k?
json.field "is8k", self.badges.eight_k?
json.field "isVr180", self.badges.vr180?
json.field "isVr360", self.badges.vr360?
json.field "is3d", self.badges.three_d?
json.field "hasCaptions", self.badges.closed_captions?
end
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
to_json(locale, json)
end
end
def to_json(json : JSON::Builder)
to_json(nil, json)
end
def upcoming?
premiere_timestamp ? true : false
end
end
struct SearchPlaylistVideo
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
property author_verified : Bool
def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorVerified", self.author_verified
json.field "videoCount", self.video_count
json.field "videos" do
json.array do
self.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end
end
end
end
end
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
to_json(locale, json)
end
end
def to_json(json : JSON::Builder)
to_json(nil, json)
end
end
struct SearchChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property channel_handle : String?
property description_html : String
property auto_generated : Bool
property author_verified : Bool
def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorVerified", self.author_verified
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
json.field "channelHandle", self.channel_handle
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
end
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
to_json(locale, json)
end
end
def to_json(json : JSON::Builder)
to_json(nil, json)
end
end
struct SearchHashtag
include DB::Serializable
property title : String
property url : String
property video_count : Int64
property channel_count : Int64
def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "hashtag"
json.field "title", self.title
json.field "url", self.url
json.field "videoCount", self.video_count
json.field "channelCount", self.channel_count
end
end
end
class Category
include DB::Serializable
property title : String
property contents : Array(SearchItem) | Array(Video)
property url : String?
property description_html : String
property badges : Array(Tuple(String, String))?
def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "category"
json.field "title", self.title
json.field "contents" do
json.array do
self.contents.each do |item|
item.to_json(locale, json)
end
end
end
end
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
to_json(locale, json)
end
end
def to_json(json : JSON::Builder)
to_json(nil, json)
end
end
struct Continuation
getter token
def initialize(@token : String)
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category

349
src/invidious/helpers/sig_helper.cr Normal file
View File

@@ -0,0 +1,349 @@
require "uri"
require "socket"
require "socket/tcp_socket"
require "socket/unix_socket"
{% if flag?(:advanced_debug) %}
require "io/hexdump"
{% end %}
private alias NetworkEndian = IO::ByteFormat::NetworkEndian
module Invidious::SigHelper
enum UpdateStatus
Updated
UpdateNotRequired
Error
end
# -------------------
# Payload types
# -------------------
abstract struct Payload
end
struct StringPayload < Payload
getter string : String
def initialize(str : String)
raise Exception.new("SigHelper: String can't be empty") if str.empty?
@string = str
end
def self.from_bytes(slice : Bytes)
size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice)
if size == 0 # Error code
raise Exception.new("SigHelper: Server encountered an error")
end
if (slice.bytesize - 2) != size
raise Exception.new("SigHelper: String size mismatch")
end
if str = String.new(slice[2..])
return self.new(str)
else
raise Exception.new("SigHelper: Can't read string from socket")
end
end
def to_io(io)
# `.to_u16` raises if there is an overflow during the conversion
io.write_bytes(@string.bytesize.to_u16, NetworkEndian)
io.write(@string.to_slice)
end
end
private enum Opcode
FORCE_UPDATE = 0
DECRYPT_N_SIGNATURE = 1
DECRYPT_SIGNATURE = 2
GET_SIGNATURE_TIMESTAMP = 3
GET_PLAYER_STATUS = 4
PLAYER_UPDATE_TIMESTAMP = 5
end
private record Request,
opcode : Opcode,
payload : Payload?
# ----------------------
# High-level functions
# ----------------------
class Client
@mux : Multiplexor
def initialize(uri_or_path)
@mux = Multiplexor.new(uri_or_path)
end
# Forces the server to re-fetch the YouTube player, and extract the necessary
# components from it (nsig function code, sig function code, signature timestamp).
def force_update : UpdateStatus
request = Request.new(Opcode::FORCE_UPDATE, nil)
value = send_request(request) do |bytes|
IO::ByteFormat::NetworkEndian.decode(UInt16, bytes)
end
case value
when 0x0000 then return UpdateStatus::Error
when 0xFFFF then return UpdateStatus::UpdateNotRequired
when 0xF44F then return UpdateStatus::Updated
else
code = value.nil? ? "nil" : value.to_s(base: 16)
raise Exception.new("SigHelper: Invalid status code received #{code}")
end
end
# Decrypt a provided n signature using the server's current nsig function
# code, and return the result (or an error).
def decrypt_n_param(n : String) : String?
request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n))
n_dec = self.send_request(request) do |bytes|
StringPayload.from_bytes(bytes).string
end
return n_dec
end
# Decrypt a provided s signature using the server's current sig function
# code, and return the result (or an error).
def decrypt_sig(sig : String) : String?
request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig))
sig_dec = self.send_request(request) do |bytes|
StringPayload.from_bytes(bytes).string
end
return sig_dec
end
# Return the signature timestamp from the server's current player
def get_signature_timestamp : UInt64?
request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil)
return self.send_request(request) do |bytes|
IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
end
end
# Return the current player's version
def get_player : UInt32?
request = Request.new(Opcode::GET_PLAYER_STATUS, nil)
return self.send_request(request) do |bytes|
has_player = (bytes[0] == 0xFF)
player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4])
has_player ? player_version : nil
end
end
# Return when the player was last updated
def get_player_timestamp : UInt64?
request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil)
return self.send_request(request) do |bytes|
IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
end
end
private def send_request(request : Request, &)
channel = @mux.send(request)
slice = channel.receive
return yield slice
rescue ex
LOGGER.debug("SigHelper: Error when sending a request")
LOGGER.trace(ex.inspect_with_backtrace)
return nil
end
end
# ---------------------
# Low level functions
# ---------------------
class Multiplexor
alias TransactionID = UInt32
record Transaction, channel = ::Channel(Bytes).new
@prng = Random.new
@mutex = Mutex.new
@queue = {} of TransactionID => Transaction
@conn : Connection
@uri_or_path : String
def initialize(@uri_or_path)
@conn = Connection.new(uri_or_path)
listen
end
def listen : Nil
raise "Socket is closed" if @conn.closed?
LOGGER.debug("SigHelper: Multiplexor listening")
spawn do
loop do
begin
receive_data
rescue ex
LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")
# We close the socket because for some reason is not closed.
@conn.close
loop do
begin
@conn = Connection.new(@uri_or_path)
LOGGER.info("SigHelper: Reconnected to SigHelper!")
rescue ex
LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
sleep 500.milliseconds
next
end
break if !@conn.closed?
end
end
Fiber.yield
end
end
end
def send(request : Request)
transaction = Transaction.new
transaction_id = @prng.rand(TransactionID)
# Add transaction to queue
@mutex.synchronize do
# On a 32-bits random integer, this should never happen. Though, just in case, ...
if @queue[transaction_id]?
raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!")
end
@queue[transaction_id] = transaction
end
write_packet(transaction_id, request)
return transaction.channel
end
def receive_data
transaction_id, slice = read_packet
@mutex.synchronize do
if transaction = @queue.delete(transaction_id)
# Remove transaction from queue and send data to the channel
transaction.channel.send(slice)
LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel")
else
raise Exception.new("SigHelper: Received transaction was not in queue")
end
end
end
# Read a single packet from the socket
private def read_packet : {TransactionID, Bytes}
# Header
transaction_id = @conn.read_bytes(UInt32, NetworkEndian)
length = @conn.read_bytes(UInt32, NetworkEndian)
LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}")
if length > 67_000
raise Exception.new("SigHelper: Packet longer than expected (#{length})")
end
# Payload
slice = Bytes.new(length)
@conn.read(slice) if length > 0
LOGGER.trace("SigHelper: payload = #{slice}")
LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done")
return transaction_id, slice
end
# Write a single packet to the socket
private def write_packet(transaction_id : TransactionID, request : Request)
LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}")
io = IO::Memory.new(1024)
io.write_bytes(request.opcode.to_u8, NetworkEndian)
io.write_bytes(transaction_id, NetworkEndian)
if payload = request.payload
payload.to_io(io)
end
@conn.send(io)
@conn.flush
LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done")
end
end
class Connection
@socket : UNIXSocket | TCPSocket
{% if flag?(:advanced_debug) %}
@io : IO::Hexdump
{% end %}
def initialize(host_or_path : String)
case host_or_path
when .starts_with?('/')
# Make sure that the file exists
if File.exists?(host_or_path)
@socket = UNIXSocket.new(host_or_path)
else
raise Exception.new("SigHelper: '#{host_or_path}' no such file")
end
when .starts_with?("tcp://")
uri = URI.parse(host_or_path)
@socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
else
uri = URI.parse("tcp://#{host_or_path}")
@socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
end
LOGGER.info("SigHelper: Using helper at '#{host_or_path}'")
{% if flag?(:advanced_debug) %}
@io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true)
{% end %}
@socket.sync = false
@socket.blocking = false
end
def closed? : Bool
return @socket.closed?
end
def close : Nil
@socket.close if !@socket.closed?
end
def flush(*args, **options)
@socket.flush(*args, **options)
end
def send(*args, **options)
@socket.send(*args, **options)
end
# Wrap IO functions, with added debug tooling if needed
{% for function in %w(read read_bytes write write_bytes) %}
def {{function.id}}(*args, **options)
{% if flag?(:advanced_debug) %}
@io.{{function.id}}(*args, **options)
{% else %}
@socket.{{function.id}}(*args, **options)
{% end %}
end
{% end %}
end
end

104
src/invidious/helpers/signatures.cr
View File

@@ -1,73 +1,53 @@
alias SigProc = Proc(Array(String), Int32, Array(String))
require "http/params"
require "./sig_helper"
struct DecryptFunction
@decrypt_function = [] of {SigProc, Int32}
@decrypt_time = Time.monotonic
class Invidious::DecryptFunction
@last_update : Time = Time.utc - 42.days
def initialize(@use_polling = true)
def initialize(uri_or_path)
@client = SigHelper::Client.new(uri_or_path)
self.check_update
end
def update_decrypt_function
@decrypt_function = fetch_decrypt_function
def check_update
# If we have updated in the last 5 minutes, do nothing
return if (Time.utc - @last_update) < 5.minutes
# Get the amount of time elapsed since when the player was updated, in the
# event where multiple invidious processes are run in parallel.
update_time_elapsed = (@client.get_player_timestamp || 301).seconds
if update_time_elapsed > 5.minutes
LOGGER.debug("Signature: Player might be outdated, updating")
@client.force_update
@last_update = Time.utc
end
end
private def fetch_decrypt_function(id = "CvFH_6DNRCY")
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
player = YT_POOL.client &.get(url).body
function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
function_body = function_body.split(";")[1..-2]
var_name = function_body[0][0, 2]
var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
operations = {} of String => SigProc
var_body.split("},").each do |operation|
op_name = operation.match(/^[^:]+/).not_nil![0]
op_body = operation.match(/\{[^}]+/).not_nil![0]
case op_body
when "{a.reverse()"
operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
when "{a.splice(0,b)"
operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
else
operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
end
end
decrypt_function = [] of {SigProc, Int32}
function_body.each do |function|
function = function.lchop(var_name).delete("[].")
op_name = function.match(/[^\(]+/).not_nil![0]
value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
decrypt_function << {operations[op_name], value}
end
return decrypt_function
def decrypt_nsig(n : String) : String?
self.check_update
return @client.decrypt_n_param(n)
rescue ex
LOGGER.debug(ex.message || "Signature: Unknown error")
LOGGER.trace(ex.inspect_with_backtrace)
return nil
end
def decrypt_signature(fmt : Hash(String, JSON::Any))
return "" if !fmt["s"]? || !fmt["sp"]?
def decrypt_signature(str : String) : String?
self.check_update
return @client.decrypt_sig(str)
rescue ex
LOGGER.debug(ex.message || "Signature: Unknown error")
LOGGER.trace(ex.inspect_with_backtrace)
return nil
end
sp = fmt["sp"].as_s
sig = fmt["s"].as_s.split("")
if !@use_polling
now = Time.monotonic
if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0
@decrypt_function = fetch_decrypt_function
@decrypt_time = Time.monotonic
end
end
@decrypt_function.each do |proc, value|
sig = proc.call(sig, value)
end
return "&#{sp}=#{sig.join("")}"
def get_sts : UInt64?
self.check_update
return @client.get_signature_timestamp
rescue ex
LOGGER.debug(ex.message || "Signature: Unknown error")
LOGGER.trace(ex.inspect_with_backtrace)
return nil
end
end

25
src/invidious/helpers/tokens.cr
View File

@@ -1,8 +1,8 @@
require "crypto/subtle"
def generate_token(email, scopes, expire, key, db)
def generate_token(email, scopes, expire, key)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc)
Invidious::Database::SessionIDs.insert(session, email)
token = {
"session" => session,
@@ -19,7 +19,7 @@ def generate_token(email, scopes, expire, key, db)
return token.to_json
end
def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = false)
def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false)
expire = Time.utc + expire
token = {
@@ -30,7 +30,7 @@ def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = fa
if use_nonce
nonce = Random::Secure.hex(16)
db.exec("INSERT INTO nonces VALUES ($1, $2) ON CONFLICT DO NOTHING", nonce, expire)
Invidious::Database::Nonces.insert(nonce, expire)
token["nonce"] = nonce
end
@@ -42,11 +42,14 @@ end
def sign_token(key, hash)
string_to_sign = [] of String
# TODO: figure out which "key" variable is used
# Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this
# variable, but it's preferable to not touch that (works fine atm).
hash.each do |key, value|
next if key == "signature"
if value.is_a?(JSON::Any) && value.as_a?
value = value.as_a.map { |i| i.as_s }
value = value.as_a.map(&.as_s)
end
case value
@@ -63,7 +66,7 @@ def sign_token(key, hash)
return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip
end
def validate_request(token, session, request, key, db, locale = nil)
def validate_request(token, session, request, key, locale = nil)
case token
when String
token = JSON.parse(URI.decode_www_form(token)).as_h
@@ -82,7 +85,7 @@ def validate_request(token, session, request, key, db, locale = nil)
raise InfoException.new("Erroneous token")
end
scopes = token["scopes"].as_a.map { |v| v.as_s }
scopes = token["scopes"].as_a.map(&.as_s)
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
raise InfoException.new("Invalid scope")
@@ -92,9 +95,9 @@ def validate_request(token, session, request, key, db, locale = nil)
raise InfoException.new("Invalid signature")
end
if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s))
if nonce[1] > Time.utc
db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
Invidious::Database::Nonces.update_set_expired(nonce[0])
else
raise InfoException.new("Erroneous token")
end
@@ -105,11 +108,11 @@ end
def scope_includes_scope(scope, subset)
methods, endpoint = scope.split(":")
methods = methods.split(";").map { |method| method.upcase }.reject { |method| method.empty? }.sort
methods = methods.split(";").map(&.upcase).reject(&.empty?).sort!
endpoint = endpoint.downcase
subset_methods, subset_endpoint = subset.split(":")
subset_methods = subset_methods.split(";").map { |method| method.upcase }.sort
subset_methods = subset_methods.split(";").map(&.upcase).sort!
subset_endpoint = subset_endpoint.downcase
if methods.empty?

278
src/invidious/helpers/utils.cr
View File

@@ -1,70 +1,3 @@
require "lsquic"
require "pool/connection"
def add_yt_headers(request)
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["accept-language"] ||= "en-us,en;q=0.5"
return if request.resource.starts_with? "/sorry/index"
request.headers["x-youtube-client-name"] ||= "1"
request.headers["x-youtube-client-version"] ||= "2.20200609"
# Preserve original cookies and add new YT consent cookie for EU servers
request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+"
if !CONFIG.cookies.empty?
request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
end
struct YoutubeConnectionPool
property! url : URI
property! capacity : Int32
property! timeout : Float64
property pool : ConnectionPool(QUIC::Client | HTTP::Client)
def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
@url = url
@pool = build_pool(use_quic)
end
def client(region = nil, &block)
if region
conn = make_client(url, region)
response = yield conn
else
conn = pool.checkout
begin
response = yield conn
rescue ex
conn.close
conn = QUIC::Client.new(url)
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
response = yield conn
ensure
pool.checkin(conn)
end
end
response
end
private def build_pool(use_quic)
ConnectionPool(QUIC::Client | HTTP::Client).new(capacity: capacity, timeout: timeout) do
if use_quic
conn = QUIC::Client.new(url)
else
conn = HTTP::Client.new(url)
end
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
conn
end
end
end
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
def ci_lower_bound(pos, n)
if n == 0
@@ -85,42 +18,18 @@ def elapsed_text(elapsed)
"#{(millis * 1000).round(2)}µs"
end
def make_client(url : URI, region = nil)
# TODO: Migrate any applicable endpoints to QUIC
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
if region
PROXY_LIST[region]?.try &.sample(40).each do |proxy|
begin
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
client.set_proxy(proxy)
break
rescue ex
end
end
end
return client
end
def make_client(url : URI, region = nil, &block)
client = make_client(url, region)
begin
yield client
ensure
client.close
end
end
def decode_length_seconds(string)
length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i
length_seconds = string.gsub(/[^0-9:]/, "")
return 0_i32 if length_seconds.empty?
length_seconds = length_seconds.split(":").map { |x| x.to_i? || 0 }
length_seconds = [0] * (3 - length_seconds.size) + length_seconds
length_seconds = Time::Span.new hours: length_seconds[0], minutes: length_seconds[1], seconds: length_seconds[2]
length_seconds = length_seconds.total_seconds.to_i
length_seconds = Time::Span.new(
hours: length_seconds[0],
minutes: length_seconds[1],
seconds: length_seconds[2]
).total_seconds.to_i32
return length_seconds
end
@@ -142,6 +51,24 @@ def recode_length_seconds(time)
end
end
def decode_interval(string : String) : Time::Span
raw_minutes = string.try &.to_i32?
if !raw_minutes
hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_i32
hours ||= 0
minutes = /(?<minutes>\d+)m(?!s)/.match(string).try &.["minutes"].try &.to_i32
minutes ||= 0
time = Time::Span.new(hours: hours, minutes: minutes)
else
time = Time::Span.new(minutes: raw_minutes)
end
return time
end
def decode_time(string)
time = string.try &.to_f?
@@ -184,24 +111,27 @@ def decode_date(string : String)
else nil # Continue
end
# String matches format "20 hours ago", "4 months ago"...
date = string.split(" ")[-3, 3]
delta = date[0].to_i
# String matches format "20 hours ago", "4 months ago", "20s ago", "15min ago"...
match = string.match(/(?<count>\d+) ?(?<span>[smhdwy]\w*) ago/)
case date[1]
when .includes? "second"
raise "Could not parse #{string}" if match.nil?
delta = match["count"].to_i
case match["span"]
when .starts_with? "s" # second(s)
delta = delta.seconds
when .includes? "minute"
when .starts_with? "mi" # minute(s)
delta = delta.minutes
when .includes? "hour"
when .starts_with? "h" # hour(s)
delta = delta.hours
when .includes? "day"
when .starts_with? "d" # day(s)
delta = delta.days
when .includes? "week"
when .starts_with? "w" # week(s)
delta = delta.weeks
when .includes? "month"
when .starts_with? "mo" # month(s)
delta = delta.months
when .includes? "year"
when .starts_with? "y" # year(s)
delta = delta.years
else
raise "Could not parse #{string}"
@@ -214,51 +144,47 @@ def recode_date(time : Time, locale)
span = Time.utc - time
if span.total_days > 365.0
span = translate(locale, "`x` years", (span.total_days.to_i // 365).to_s)
return translate_count(locale, "generic_count_years", span.total_days.to_i // 365)
elsif span.total_days > 30.0
span = translate(locale, "`x` months", (span.total_days.to_i // 30).to_s)
return translate_count(locale, "generic_count_months", span.total_days.to_i // 30)
elsif span.total_days > 7.0
span = translate(locale, "`x` weeks", (span.total_days.to_i // 7).to_s)
return translate_count(locale, "generic_count_weeks", span.total_days.to_i // 7)
elsif span.total_hours > 24.0
span = translate(locale, "`x` days", (span.total_days.to_i).to_s)
return translate_count(locale, "generic_count_days", span.total_days.to_i)
elsif span.total_minutes > 60.0
span = translate(locale, "`x` hours", (span.total_hours.to_i).to_s)
return translate_count(locale, "generic_count_hours", span.total_hours.to_i)
elsif span.total_seconds > 60.0
span = translate(locale, "`x` minutes", (span.total_minutes.to_i).to_s)
return translate_count(locale, "generic_count_minutes", span.total_minutes.to_i)
else
span = translate(locale, "`x` seconds", (span.total_seconds.to_i).to_s)
return translate_count(locale, "generic_count_seconds", span.total_seconds.to_i)
end
return span
end
def number_with_separator(number)
number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
end
def short_text_to_number(short_text : String) : Int32
case short_text
when .ends_with? "M"
number = short_text.rstrip(" mM").to_f
number *= 1000000
when .ends_with? "K"
number = short_text.rstrip(" kK").to_f
number *= 1000
else
number = short_text.rstrip(" ")
def short_text_to_number(short_text : String) : Int64
matches = /(?<number>\d+(\.\d+)?)\s?(?<suffix>[mMkKbB]?)/.match(short_text)
number = matches.try &.["number"].to_f || 0.0
case matches.try &.["suffix"].downcase
when "k" then number *= 1_000
when "m" then number *= 1_000_000
when "b" then number *= 1_000_000_000
end
number = number.to_i
return number
return number.to_i64
rescue ex
return 0_i64
end
def number_to_short_text(number)
seperated = number_with_separator(number).gsub(",", ".").split("")
text = seperated.first(2).join
separated = number_with_separator(number).gsub(",", ".").split("")
text = separated.first(2).join
if seperated[2]? && seperated[2] != "."
text += seperated[2]
if separated[2]? && separated[2] != "."
text += separated[2]
end
text = text.rchop(".0")
@@ -298,7 +224,7 @@ def make_host_url(kemal_config)
# Add if non-standard port
if port != 80 && port != 443
port = ":#{kemal_config.port}"
port = ":#{port}"
else
port = ""
end
@@ -336,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.request_target
referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback
@@ -385,8 +311,8 @@ def parse_range(range)
end
ranges = range.lchop("bytes=").split(',')
ranges.each do |range|
start_range, end_range = range.split('-')
ranges.each do |r|
start_range, end_range = r.split('-')
start_range = start_range.to_i64? || 0_i64
end_range = end_range.to_i64?
@@ -397,15 +323,63 @@ def parse_range(range)
return 0_i64, nil
end
def convert_theme(theme)
case theme
when "true"
"dark"
when "false"
"light"
when "", nil
nil
else
theme
def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String
str = uri.to_s.sub(/^https?:\/\//, "")
if str.size > max_length
str = "#{str[0, max_length]}#{suffix}"
end
return str
end
# Get the html link from a NavigationEndpoint or an innertubeCommand
def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
if url = endpoint.dig?("urlEndpoint", "url").try &.as_s
url = URI.parse(url)
displayed_url = text
if url.host == "youtu.be"
url = "/watch?v=#{url.request_target.lstrip('/')}"
elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com")
if url.path == "/redirect"
# Sometimes, links can be corrupted (why?) so make sure to fallback
# nicely. See https://github.com/iv-org/invidious/issues/2682
url = url.query_params["q"]? || ""
displayed_url = url
else
url = url.request_target
displayed_url = "youtube.com#{url}"
end
end
text = %(<a href="#{url}">#{reduce_uri(displayed_url)}</a>)
elsif watch_endpoint = endpoint.dig?("watchEndpoint")
start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i
link_video_id = watch_endpoint["videoId"].as_s
url = "/watch?v=#{link_video_id}"
url += "&t=#{start_time}" if !start_time.nil?
# If the current video ID (passed through from the caller function)
# is the same as the video ID in the link, add HTML attributes for
# the JS handler function that bypasses page reload.
#
# See: https://github.com/iv-org/invidious/issues/3063
if link_video_id == video_id
start_time ||= 0
text = %(<a href="#{url}" data-onclick="jump_to_time" data-jump-time="#{start_time}">#{reduce_uri(text)}</a>)
else
text = %(<a href="#{url}">#{text}</a>)
end
elsif url = endpoint.dig?("commandMetadata", "webCommandMetadata", "url").try &.as_s
if text.starts_with?(/\s?[@#]/)
# Handle "pings" in comments and hasthags differently
# See:
# - https://github.com/iv-org/invidious/issues/3038
# - https://github.com/iv-org/invidious/issues/3062
text = %(<a href="#{url}">#{text}</a>)
else
text = %(<a href="#{url}">#{reduce_uri(text)}</a>)
end
end
return text
end

81
src/invidious/helpers/webvtt.cr Normal file
View File

@@ -0,0 +1,81 @@
# Namespace for logic relating to generating WebVTT files
#
# Probably not compliant to WebVTT's specs but it is enough for Invidious.
module WebVTT
# A WebVTT builder generates WebVTT files
private class Builder
# See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload
private ESCAPE_SUBSTITUTIONS = {
'&' => "&amp;",
'<' => "&lt;",
'>' => "&gt;",
'\u200E' => "&lrm;",
'\u200F' => "&rlm;",
'\u00A0' => "&nbsp;",
}
def initialize(@io : IO)
end
# Writes an vtt cue with the specified time stamp and contents
def cue(start_time : Time::Span, end_time : Time::Span, text : String)
timestamp(start_time, end_time)
@io << self.escape(text)
@io << "\n\n"
end
private def timestamp(start_time : Time::Span, end_time : Time::Span)
timestamp_component(start_time)
@io << " --> "
timestamp_component(end_time)
@io << '\n'
end
private def timestamp_component(timestamp : Time::Span)
@io << timestamp.hours.to_s.rjust(2, '0')
@io << ':' << timestamp.minutes.to_s.rjust(2, '0')
@io << ':' << timestamp.seconds.to_s.rjust(2, '0')
@io << '.' << timestamp.milliseconds.to_s.rjust(3, '0')
end
private def escape(text : String) : String
return text.gsub(ESCAPE_SUBSTITUTIONS)
end
def document(setting_fields : Hash(String, String)? = nil, &)
@io << "WEBVTT\n"
if setting_fields
setting_fields.each do |name, value|
@io << name << ": " << value << '\n'
end
end
@io << '\n'
yield
end
end
# Returns the resulting `String` of writing WebVTT to the yielded `WebVTT::Builder`
#
# ```
# string = WebVTT.build do |vtt|
# vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1")
# vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2")
# end
#
# string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n"
# ```
#
# Accepts an optional settings fields hash to add settings attribute to the resulting vtt file.
def self.build(setting_fields : Hash(String, String)? = nil, &)
String.build do |str|
builder = Builder.new(str)
builder.document(setting_fields) do
yield builder
end
end
end
end

31
src/invidious/helpers/youtube_api.cr
View File

@@ -1,31 +0,0 @@
#
# This file contains youtube API wrappers
#
# Hard-coded constants required by the API
HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
HARDCODED_CLIENT_VERS = "2.20210318.08.00"
def request_youtube_api_browse(continuation)
# JSON Request data, required by the API
data = {
"context": {
"client": {
"hl": "en",
"gl": "US",
"clientName": "WEB",
"clientVersion": HARDCODED_CLIENT_VERS,
},
},
"continuation": continuation,
}
# Send the POST request and return result
response = YT_POOL.client &.post(
"/youtubei/v1/browse?key=#{HARDCODED_API_KEY}",
headers: HTTP::Headers{"content-type" => "application/json"},
body: data.to_json
)
return response.body
end

41
src/invidious/http_server/utils.cr Normal file
View File

@@ -0,0 +1,41 @@
require "uri"
module Invidious::HttpServer
module Utils
extend self
def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
url = URI.parse(raw_url)
# Add some URL parameters
params = url.query_params
params["host"] = url.host.not_nil! # Should never be nil, in theory
params["region"] = region if !region.nil?
url.query_params = params
if absolute
return "#{HOST_URL}#{url.request_target}"
else
return url.request_target
end
end
def add_params_to_url(url : String | URI, params : URI::Params) : URI
url = URI.parse(url) if url.is_a?(String)
url_query = url.query || ""
# Append the parameters
url.query = String.build do |str|
if !url_query.empty?
str << url_query
str << '&'
end
str << params
end
return url
end
end
end

27
src/invidious/jobs.cr
View File

@@ -1,12 +1,39 @@
module Invidious::Jobs
JOBS = [] of BaseJob
# Automatically generate a structure that wraps the various
# jobs' configs, so that the following YAML config can be used:
#
# jobs:
# job_name:
# enabled: true
# some_property: "value"
#
macro finished
struct JobsConfig
include YAML::Serializable
{% for sc in BaseJob.subclasses %}
# Voodoo macro to transform `Some::Module::CustomJob` to `custom`
{% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %}
getter {{ class_name }} = {{ sc.name }}::Config.new
{% end %}
def initialize
end
end
end
def self.register(job : BaseJob)
JOBS << job
end
def self.start_all
JOBS.each do |job|
# Don't run the main rountine if the job is disabled by config
next if job.disabled?
spawn { job.begin }
end
end

30
src/invidious/jobs/base_job.cr
View File

@@ -1,3 +1,33 @@
abstract class Invidious::Jobs::BaseJob
abstract def begin
# When this base job class is inherited, make sure to define
# a basic "Config" structure, that contains the "enable" property,
# and to create the associated instance property.
#
macro inherited
macro finished
# This config structure can be expanded as required.
struct Config
include YAML::Serializable
property enable = true
def initialize
end
end
property cfg = Config.new
# Return true if job is enabled by config
protected def enabled? : Bool
return (@cfg.enable == true)
end
# Return true if job is disabled by config
protected def disabled? : Bool
return (@cfg.enable == false)
end
end
end
end

131
src/invidious/jobs/bypass_captcha_job.cr
View File

@@ -1,131 +0,0 @@
class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
def begin
loop do
begin
{"/watch?v=jNQXAC9IVRw&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UC4QobU6STFB0P71PMvOGN5A")}.each do |path|
response = YT_POOL.client &.get(path)
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
headers = response.cookies.add_request_headers(HTTP::Headers.new)
response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => "https://www.youtube.com#{path}",
"websiteKey" => site_key,
"recaptchaDataSValue" => s_value,
},
}.to_json).body)
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
response.cookies
.select { |cookie| cookie.name != "PREF" }
.each { |cookie| CONFIG.cookies << cookie }
# Persist cookies between runs
File.write("config/config.yml", CONFIG.to_yaml)
elsif response.headers["Location"]?.try &.includes?("/sorry/index")
location = response.headers["Location"].try { |u| URI.parse(u) }
headers = HTTP::Headers{":authority" => location.host.not_nil!}
response = YT_POOL.client &.get(location.request_target, headers)
html = XML.parse_html(response.body)
form = html.xpath_node(%(//form[@action="index"])).not_nil!
site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
inputs = {} of String => String
form.xpath_nodes(%(.//input[@name])).map do |node|
inputs[node["name"]] = node["value"]
end
captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url))
captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
response = JSON.parse(captcha_client.post("/createTask",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
"recaptchaDataSValue" => s_value,
},
}.to_json).body)
captcha_client.close
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
loop do
sleep 10.seconds
response = JSON.parse(captcha_client.post("/getTaskResult",
headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
if response["status"]?.try &.== "ready"
break
elsif response["errorId"]?.try &.as_i != 0
raise response["errorDescription"].as_s
end
end
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
headers = HTTP::Headers{
"Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
}
cookies = HTTP::Cookies.from_headers(headers)
cookies.each { |cookie| CONFIG.cookies << cookie }
# Persist cookies between runs
File.write("config/config.yml", CONFIG.to_yaml)
end
end
rescue ex
LOGGER.error("BypassCaptchaJob: #{ex.message}")
ensure
sleep 1.minute
Fiber.yield
end
end
end
end

27
src/invidious/jobs/clear_expired_items_job.cr Normal file
View File

@@ -0,0 +1,27 @@
class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob
# Remove items (videos, nonces, etc..) whose cache is outdated every hour.
# Removes the need for a cron job.
def begin
loop do
failed = false
LOGGER.info("jobs: running ClearExpiredItems job")
begin
Invidious::Database::Videos.delete_expired
Invidious::Database::Nonces.delete_expired
rescue DB::Error
failed = true
end
# Retry earlier than scheduled on DB error
if failed
LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.")
sleep 10.minutes
else
LOGGER.info("jobs: ClearExpiredItems done.")
sleep 1.hour
end
end
end
end

97
src/invidious/jobs/instance_refresh_job.cr Normal file
View File

@@ -0,0 +1,97 @@
class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob
# We update the internals of a constant as so it can be accessed from anywhere
# within the codebase
#
# "INSTANCES" => Array(Tuple(String, String)) # region, instance
INSTANCES = {"INSTANCES" => [] of Tuple(String, String)}
def initialize
end
def begin
loop do
refresh_instances
LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes")
sleep 30.minute
Fiber.yield
end
end
# Refreshes the list of instances used for redirects.
#
# Does the following three checks for each instance
# - Is it a clear-net instance?
# - Is it an instance with a good uptime?
# - Is it an updated instance?
private def refresh_instances
raw_instance_list = self.fetch_instances
filtered_instance_list = [] of Tuple(String, String)
raw_instance_list.each do |instance_data|
# TODO allow Tor hidden service instances when the current instance
# is also a hidden service. Same for i2p and any other non-clearnet instances.
begin
domain = instance_data[0]
info = instance_data[1]
stats = info["stats"]
next unless info["type"] == "https"
next if bad_uptime?(info["monitor"])
next if outdated?(stats["software"]["version"])
filtered_instance_list << {info["region"].as_s, domain.as_s}
rescue ex
if domain
LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ")
else
LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ")
end
end
end
if !filtered_instance_list.empty?
INSTANCES["INSTANCES"] = filtered_instance_list
end
end
# Fetches information regarding instances from api.invidious.io or an otherwise configured URL
private def fetch_instances : Array(JSON::Any)
begin
# We directly call the stdlib HTTP::Client here as it allows us to negate the effects
# of the force_resolve config option. This is needed as api.invidious.io does not support ipv6
# and as such the following request raises if we were to use force_resolve with the ipv6 value.
instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io"))
# Timeouts
instance_api_client.connect_timeout = 10.seconds
instance_api_client.dns_timeout = 10.seconds
raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a
instance_api_client.close
rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException
raw_instance_list = [] of JSON::Any
end
return raw_instance_list
end
# Checks if the given target instance is outdated
private def outdated?(target_instance_version) : Bool
remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/)
return false if !remote_commit_date
remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC)
local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC)
return (remote_commit_date - local_commit_date).abs.days > 30
end
# Checks if the uptime of the target instance is greater than 90% over a 30 day period
private def bad_uptime?(target_instance_health_monitor) : Bool
return true if !target_instance_health_monitor["down"].as_bool == false
return true if target_instance_health_monitor["uptime"].as_f < 90
return false
end
end

4
src/invidious/jobs/notification_job.cr
View File

@@ -1,12 +1,12 @@
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
def initialize(@connection_channel, @pg_url)
end
def begin
connections = [] of Channel(PQ::Notification)
connections = [] of ::Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }

13
src/invidious/jobs/pull_popular_videos_job.cr
View File

@@ -1,11 +1,4 @@
class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
QUERY = <<-SQL
SELECT DISTINCT ON (ucid) *
FROM channel_videos
WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC
SQL
POPULAR_VIDEOS = Atomic.new([] of ChannelVideo)
private getter db : DB::Database
@@ -14,9 +7,9 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
def begin
loop do
videos = db.query_all(QUERY, as: ChannelVideo)
.sort_by(&.published)
.reverse
videos = Invidious::Database::ChannelVideos.select_popular_videos
.sort_by!(&.published)
.reverse!
POPULAR_VIDEOS.set(videos)

16
src/invidious/jobs/refresh_channels_job.cr
View File

@@ -8,12 +8,12 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
active_channel = Channel(Bool).new
backoff = 1.seconds
active_channel = ::Channel(Bool).new
backoff = 2.minutes
loop do
LOGGER.debug("RefreshChannelsJob: Refreshing all channels")
db.query("SELECT id FROM channels ORDER BY updated") do |rs|
PG_DB.query("SELECT id FROM channels ORDER BY updated") do |rs|
rs.each do
id = rs.read(String)
@@ -30,16 +30,16 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
spawn do
begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, db, CONFIG.full_refresh)
channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
lim_fibers = max_fibers
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
Invidious::Database::Channels.update_author(id, channel.author)
rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
Invidious::Database::Channels.update_mark_deleted(id)
else
lim_fibers = 1
LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")
@@ -58,8 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
end
end
LOGGER.debug("RefreshChannelsJob: Done, sleeping for one minute")
sleep 1.minute
LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_interval}")
sleep CONFIG.channel_refresh_interval
Fiber.yield
end
end

4
src/invidious/jobs/refresh_feeds_job.cr
View File

@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
@@ -25,7 +25,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
spawn do
begin
# Drop outdated views
column_array = get_column_array(db, view_name)
column_array = Invidious::Database.get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")

22
src/invidious/jobs/statistics_refresh_job.cr
View File

@@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
#
# "totalRequests" => 0_i64,
# "successfulRequests" => 0_i64
# "ratio" => 0_i64
#
"playback" => {} of String => Int64 | Float64,
}
private getter db : DB::Database
@@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
loop do
refresh_stats
sleep 1.minute
sleep 10.minute
Fiber.yield
end
end
@@ -47,12 +54,17 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
private def refresh_stats
users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64)
users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64)
users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64)
users["total"] = Invidious::Database::Statistics.count_users_total
users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m
users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m
STATISTICS["metadata"] = {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
# Reset playback requests tracker
STATISTICS["playback"] = {} of String => Int64 | Float64
end
end

2
src/invidious/jobs/subscribe_to_feeds_job.cr
View File

@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|

14
src/invidious/jobs/update_decrypt_function_job.cr
View File

@@ -1,14 +0,0 @@
class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
def begin
loop do
begin
DECRYPT_FUNCTION.update_decrypt_function
rescue ex
LOGGER.error("UpdateDecryptFunctionJob : #{ex.message}")
ensure
sleep 1.minute
Fiber.yield
end
end
end
end

18
src/invidious/jsonify/api_v1/common.cr Normal file
View File

@@ -0,0 +1,18 @@
require "json"
module Invidious::JSONify::APIv1
extend self
def thumbnails(json : JSON::Builder, id : String)
json.array do
build_thumbnails(id).each do |thumbnail|
json.object do
json.field "quality", thumbnail[:name]
json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
json.field "width", thumbnail[:width]
json.field "height", thumbnail[:height]
end
end
end
end
end

295
src/invidious/jsonify/api_v1/video_json.cr Normal file
View File

@@ -0,0 +1,295 @@
require "json"
module Invidious::JSONify::APIv1
extend self
def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false)
json.object do
json.field "type", video.video_type
json.field "title", video.title
json.field "videoId", video.id
json.field "error", video.info["reason"] if video.info["reason"]?
json.field "videoThumbnails" do
self.thumbnails(json, video.id)
end
json.field "storyboards" do
self.storyboards(json, video.id, video.storyboards)
end
json.field "description", video.description
json.field "descriptionHtml", video.description_html
json.field "published", video.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale))
json.field "keywords", video.keywords
json.field "viewCount", video.views
json.field "likeCount", video.likes
json.field "dislikeCount", 0_i64
json.field "paid", video.paid
json.field "premium", video.premium
json.field "isFamilyFriendly", video.is_family_friendly
json.field "allowedRegions", video.allowed_regions
json.field "genre", video.genre
json.field "genreUrl", video.genre_url
json.field "author", video.author
json.field "authorId", video.ucid
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "authorVerified", video.author_verified
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "subCountText", video.sub_count_text
json.field "lengthSeconds", video.length_seconds
json.field "allowRatings", video.allow_ratings
json.field "rating", 0_i64
json.field "isListed", video.is_listed
json.field "liveNow", video.live_now
json.field "isPostLiveDvr", video.post_live_dvr
json.field "isUpcoming", video.upcoming?
if video.premiere_timestamp
json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix
end
if hlsvp = video.hls_manifest_url
hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}"
json.field "adaptiveFormats" do
json.array do
video.adaptive_fmts.each do |fmt|
json.object do
# Only available on regular videos, not livestreams/OTF streams
if init_range = fmt["initRange"]?
json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
end
if index_range = fmt["indexRange"]?
json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
end
# Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
if proxy
json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
fmt["url"].to_s, absolute: true
)
else
json.field "url", fmt["url"]
end
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "clen", fmt["contentLength"]? || "-1"
# Last modified is a unix timestamp with µS, with the dot omitted.
# E.g: 1638056732(.)141582
#
# On livestreams, it's not present, so always fall back to the
# current unix timestamp (up to mS precision) for compatibility.
last_modified = fmt["lastModified"]?
last_modified ||= "#{Time.utc.to_unix_ms}000"
json.field "lmt", last_modified
json.field "projectionType", fmt["projectionType"]
height = fmt["height"]?.try &.as_i
width = fmt["width"]?.try &.as_i
fps = fmt["fps"]?.try &.as_i
if fps
json.field "fps", fps
end
if height && width
json.field "size", "#{width}x#{height}"
json.field "resolution", "#{height}p"
quality_label = "#{width > height ? height : width}p"
if fps && fps > 30
quality_label += fps.to_s
end
json.field "qualityLabel", quality_label
end
if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
end
# Livestream chunk infos
json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
# Audio-related data
json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
# Extra misc stuff
json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
end
end
end
end
json.field "formatStreams" do
json.array do
video.fmt_stream.each do |fmt|
json.object do
if proxy
json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
fmt["url"].to_s, absolute: true
)
else
json.field "url", fmt["url"]
end
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
height = fmt["height"]?.try &.as_i
width = fmt["width"]?.try &.as_i
fps = fmt["fps"]?.try &.as_i
if fps
json.field "fps", fps
end
if height && width
json.field "size", "#{width}x#{height}"
json.field "resolution", "#{height}p"
quality_label = "#{width > height ? height : width}p"
if fps && fps > 30
quality_label += fps.to_s
end
json.field "qualityLabel", quality_label
end
if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
end
end
end
end
end
json.field "captions" do
json.array do
video.captions.each do |caption|
json.object do
json.field "label", caption.name
json.field "language_code", caption.language_code
json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}"
end
end
end
end
if !video.music.empty?
json.field "musicTracks" do
json.array do
video.music.each do |music|
json.object do
json.field "song", music.song
json.field "artist", music.artist
json.field "album", music.album
json.field "license", music.license
end
end
end
end
end
json.field "recommendedVideos" do
json.array do
video.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
self.thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
json.field "authorUrl", "/channel/#{rv["ucid"]?}"
json.field "authorId", rv["ucid"]?
json.field "authorVerified", rv["author_verified"] == "true"
if rv["author_thumbnail"]?
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
json.field "viewCountText", rv["short_view_count"]?
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
end
end
end
end
def storyboards(json, id, storyboards)
json.array do
storyboards.each do |sb|
json.object do
json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}"
json.field "templateUrl", sb.url.to_s
json.field "width", sb.width
json.field "height", sb.height
json.field "count", sb.count
json.field "interval", sb.interval
json.field "storyboardWidth", sb.columns
json.field "storyboardHeight", sb.rows
json.field "storyboardCount", sb.images_count
end
end
end
end
end

4
src/invidious/mixes.cr
View File

@@ -72,7 +72,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
videos += next_page.videos
end
videos.uniq! { |video| video.id }
videos.uniq!(&.id)
videos = videos.first(50)
return Mix.new({
title: mix_title,
@@ -97,7 +97,7 @@ def template_mix(mix)
<li class="pure-menu-item">
<a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}">
<div class="thumbnail">
<img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg">
<img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
<p style="width:100%">#{video["title"]}</p>

213
src/invidious/playlists.cr
View File

@@ -11,7 +11,7 @@ struct PlaylistVideo
property index : Int64
property live_now : Bool
def to_xml(auto_generated, xml : XML::Builder)
def to_xml(xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
@@ -20,13 +20,8 @@ struct PlaylistVideo
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
@@ -47,18 +42,18 @@ struct PlaylistVideo
end
end
def to_xml(auto_generated, xml : XML::Builder? = nil)
if xml
to_xml(auto_generated, xml)
else
XML.build do |json|
to_xml(auto_generated, xml)
end
end
def to_xml(_xml : Nil = nil)
XML.build { |xml| to_xml(xml) }
end
def to_json(locale, json : JSON::Builder, index : Int32?)
def to_json(locale : String?, json : JSON::Builder)
to_json(json)
end
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
@@ -67,7 +62,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index
@@ -78,17 +73,12 @@ struct PlaylistVideo
end
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
end
end
def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil)
if json
to_json(locale, json, index: index)
else
JSON.build do |json|
to_json(locale, json, index: index)
end
end
def to_json(_json : Nil, index : Int32? = nil)
JSON.build { |json| to_json(json, index: index) }
end
end
@@ -106,8 +96,9 @@ struct Playlist
property views : Int64
property updated : Time
property thumbnail : String?
property subtitle : String?
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@@ -117,6 +108,7 @@ struct Playlist
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
@@ -142,22 +134,18 @@ struct Playlist
json.field "videos" do
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, json)
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, json, continuation: continuation)
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
@@ -196,7 +184,7 @@ struct InvidiousPlaylist
end
end
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
@@ -217,32 +205,29 @@ struct InvidiousPlaylist
json.field "videos" do
json.array do
if !offset || offset == 0
index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64)
if (!offset || offset == 0) && !video_id.nil?
index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id)
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, json, offset + index)
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each_with_index do |video, idx|
video.to_json(json, offset + idx)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, json, continuation: continuation)
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
# TODO: Get playlist thumbnail from playlist data rather than first video
@thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
@@ -259,11 +244,11 @@ struct InvidiousPlaylist
end
def description_html
HTML.escape(self.description).gsub("\n", "<br>")
HTML.escape(self.description)
end
end
def create_playlist(db, title, privacy, user)
def create_playlist(title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
@@ -278,17 +263,14 @@ def create_playlist(db, title, privacy, user)
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def subscribe_playlist(db, user, playlist)
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title.byte_slice(0, 150),
title: playlist.title[..150],
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
@@ -299,10 +281,7 @@ def subscribe_playlist(db, user, playlist)
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
Invidious::Database::Playlists.insert(playlist)
return playlist
end
@@ -322,21 +301,19 @@ def produce_playlist_continuation(id, index)
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
data_wrapper = {"1:varint" => request_count, "15:string" => "PT:#{data}"}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object = {
"80226972:embedded" => {
"2:string" => plid,
"3:string" => data_wrapper,
"2:string" => plid,
"3:base64" => {
"1:varint" => request_count,
"15:string" => "PT:#{data}",
"104:embedded" => {"1:0:varint" => 0_i64},
},
"35:string" => id,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(object) }
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
@@ -344,41 +321,32 @@ def produce_playlist_continuation(id, index)
return continuation
end
def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
def get_playlist(plid : String)
if plid.starts_with? "IV"
if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
raise InfoException.new("Playlist does not exist.")
raise NotFoundException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid, locale)
return fetch_playlist(plid)
end
end
def fetch_playlist(plid, locale)
def fetch_playlist(plid : String)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en")
if response.status_code != 200
if response.headers["location"]?.try &.includes? "/sorry/index"
raise InfoException.new("Could not extract playlist info. Instance is likely blocked.")
else
raise InfoException.new("Not a playlist.")
end
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
initial_data = extract_initial_data(response.body)
playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?
playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items")
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer[0]["playlistSidebarPrimaryInfoRenderer"]?
playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer")
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || ""
title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || ""
desc_item = playlist_info["description"]?
@@ -388,18 +356,25 @@ def fetch_playlist(plid, locale)
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]?
.try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s
thumbnail = playlist_info.dig?(
"thumbnailRenderer", "playlistVideoThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "episode"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
else
@@ -412,12 +387,15 @@ def fetch_playlist(plid, locale)
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1]["playlistSidebarSecondaryInfoRenderer"]?.try &.["videoOwner"]["videoOwnerRenderer"]?
author_info = playlist_sidebar_renderer[1].dig?(
"playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer"
)
raise InfoException.new("Could not extract author info") if !author_info
author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
author = author_info.dig?("title", "runs", 0, "text").try &.as_s || ""
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || ""
ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
return Playlist.new({
@@ -432,36 +410,40 @@ def fetch_playlist(plid, locale)
views: views,
updated: updated,
thumbnail: thumbnail,
subtitle: subtitle,
})
end
def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
# Show empy playlist if requested page is out of range
def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil)
# Show empty playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
end
if playlist.is_a? InvidiousPlaylist
db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3",
playlist.id, playlist.index, offset, as: PlaylistVideo)
Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100)
else
if offset >= 100
# Normalize offset to match youtube's behavior (100 videos chunck per request)
offset = (offset / 100).to_i64 * 100_i64
if video_id
initial_data = YoutubeAPI.next({
"videoId" => video_id,
"playlistId" => playlist.id,
})
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
videos = [] of PlaylistVideo
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
ctoken = produce_playlist_continuation(playlist.id, offset)
initial_data = JSON.parse(request_youtube_api_browse(ctoken)).as_h
else
response = YT_POOL.client &.get("/playlist?list=#{playlist.id}&gl=US&hl=en")
initial_data = extract_initial_data(response.body)
initial_data = YoutubeAPI.browse(ctoken)
videos += extract_playlist_videos(initial_data)
offset += 100
end
if initial_data
return extract_playlist_videos(initial_data)
else
return [] of PlaylistVideo
end
return videos
end
end
@@ -495,7 +477,6 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s
index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64
thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s
title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || ""
author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || ""
ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || ""
@@ -537,10 +518,10 @@ def template_playlist(playlist)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
<li class="pure-menu-item">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}">
<li class="pure-menu-item" id="#{video["videoId"]}">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}">
<div class="thumbnail">
<img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg">
<img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
<p style="width:100%">#{video["title"]}</p>

354
src/invidious/routes/account.cr Normal file
View File

@@ -0,0 +1,354 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Account
extend self
# -------------------
# Password update
# -------------------
# Show the password change interface (GET request)
def get_change_password(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY)
templated "user/change_password"
end
# Handle the password change (POST request)
def post_change_password(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
password = env.params.body["password"]?
if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
end
new_password = new_passwords.uniq[0]
if new_password.empty?
return error_template(401, "Password cannot be empty")
end
if new_password.bytesize > 55
return error_template(400, "Password cannot be longer than 55 characters")
end
if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
return error_template(401, "Incorrect password")
end
new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10)
Invidious::Database::Users.update_password(user, new_password.to_s)
env.redirect referer
end
# -------------------
# Account deletion
# -------------------
# Show the account deletion confirmation prompt (GET request)
def get_delete(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY)
templated "user/delete_account"
end
# Handle the account deletion (POST request)
def post_delete(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
view_name = "subscriptions_#{sha256(user.email)}"
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)
env.response.cookies << cookie
end
env.redirect referer
end
# -------------------
# Clear history
# -------------------
# Show the watch history deletion confirmation prompt (GET request)
def get_clear_history(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY)
templated "user/clear_watch_history"
end
# Handle the watch history clearing (POST request)
def post_clear_history(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
Invidious::Database::Users.clear_watch_history(user)
env.redirect referer
end
# -------------------
# Authorize tokens
# -------------------
# Show the "authorize token?" confirmation prompt (GET request)
def get_authorize_token(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}"
end
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY)
scopes = env.params.query["scopes"]?.try &.split(",")
scopes ||= [] of String
callback_url = env.params.query["callback_url"]?
if callback_url
callback_url = URI.parse(callback_url)
end
expire = env.params.query["expire"]?.try &.to_i?
templated "user/authorize_token"
end
# Handle token authorization (POST request)
def post_authorize_token(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = env.get("user").as(User)
sid = sid.as(String)
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
access_token = generate_token(user.email, scopes, expire, HMAC_KEY)
if callback_url
access_token = URI.encode_www_form(access_token)
url = URI.parse(callback_url)
if url.query
query = HTTP::Params.parse(url.query.not_nil!)
else
query = HTTP::Params.new
end
query["token"] = access_token
query["username"] = URI.encode_path_segment(user.email)
url.query = query.to_s
env.redirect url.to_s
else
csrf_token = ""
env.set "access_token", access_token
templated "user/authorize_token"
end
end
# -------------------
# Manage tokens
# -------------------
# Show the token manager page (GET request)
def token_manager(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env, "/subscription_manager")
if !user
return env.redirect referer
end
user = user.as(User)
tokens = Invidious::Database::SessionIDs.select_all(user.email)
templated "user/token_manager"
end
# -------------------
# AJAX for tokens
# -------------------
# Handle internal (non-API) token actions (POST request)
def token_ajax(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
redirect = env.params.query["redirect"]?
redirect ||= "true"
redirect = redirect == "true"
if !user
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
end
end
user = user.as(User)
sid = sid.as(String)
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
else
return error_json(400, ex)
end
end
if env.params.query["action_revoke_token"]?
action = "action_revoke_token"
else
return env.redirect referer
end
session = env.params.query["session"]?
session ||= ""
case action
when .starts_with? "action_revoke_token"
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
end
if redirect
return env.redirect referer
else
env.response.content_type = "application/json"
return "{}"
end
end
end

241
src/invidious/routes/api/manifest.cr Normal file
View File

@@ -0,0 +1,241 @@
module Invidious::Routes::API::Manifest
# /api/manifest/dash/id/:id
def self.get_dash_video_id(env)
env.response.headers.add("Access-Control-Allow-Origin", "*")
env.response.content_type = "application/dash+xml"
local = env.params.query["local"]?.try &.== "true"
id = env.params.url["id"]
region = env.params.query["region"]?
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, region: region)
rescue ex : NotFoundException
haltf env, status_code: 404
rescue ex
haltf env, status_code: 403
end
if dashmpd = video.dash_manifest_url
response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
if response.status_code != 200
haltf env, status_code: response.status_code
end
manifest = response.body
manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
url = baseurl.lchop("<BaseURL>")
url = url.rchop("</BaseURL>")
if local
uri = URI.parse(url)
url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
end
"<BaseURL>#{url}</BaseURL>"
end
return manifest
end
adaptive_fmts = video.adaptive_fmts
if local
adaptive_fmts.each do |fmt|
fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
end
end
audio_streams = video.audio_streams.sort_by { |stream| {stream["bitrate"].as_i} }.reverse!
video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse!
manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011",
"profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static",
mediaPresentationDuration: "PT#{video.length_seconds}S") do
xml.element("Period") do
i = 0
{"audio/mp4"}.each do |mime_type|
mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
mime_streams.each do |fmt|
# OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
# Different representations of the same audio should be groupped into one AdaptationSet.
# However, most players don't support auto quality switching, so we have to trick them
# into providing a quality selector.
# See https://github.com/iv-org/invidious/issues/3074 for more details.
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate")
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
value: "2")
xml.element("BaseURL") { xml.text url }
xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
i += 1
end
end
potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144}
{"video/mp4"}.each do |mime_type|
mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
heights = [] of Int32
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do
mime_streams.each do |fmt|
# OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
width = fmt["width"].as_i
height = fmt["height"].as_i
# Resolutions reported by YouTube player (may not accurately reflect source)
height = potential_heights.min_by { |x| (height - x).abs }
next if unique_res && heights.includes? height
heights << height
xml.element("Representation", id: itag, codecs: codecs, width: width, height: height,
startWithSAP: "1", maxPlayoutRate: "1",
bandwidth: bandwidth, frameRate: fmt["fps"]) do
xml.element("BaseURL") { xml.text url }
xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
end
i += 1
end
end
end
end
return manifest
end
# /api/manifest/dash/id/videoplayback
def self.get_dash_video_playback(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect "/videoplayback?#{env.params.query}"
end
# /api/manifest/dash/id/videoplayback/*
def self.get_dash_video_playback_greedy(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect env.request.path.lchop("/api/manifest/dash/id")
end
# /api/manifest/dash/id/videoplayback && /api/manifest/dash/id/videoplayback/*
def self.options_dash_video_playback(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
# /api/manifest/hls_playlist/*
def self.get_hls_playlist(env)
response = YT_POOL.client &.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
end
local = env.params.query["local"]?.try &.== "true"
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
manifest = response.body
if local
manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
path = path.gsub(/mime\/\w+\/\w+/) do |mimetype|
mimetype = mimetype.split("/")
mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2]
end
path = path.split("/")
raw_params = {} of String => Array(String)
path.each_slice(2) do |pair|
key, value = pair
value = URI.decode_www_form(value)
if raw_params[key]?
raw_params[key] << value
else
raw_params[key] = [value]
end
end
raw_params = HTTP::Params.new(raw_params)
if fvip = raw_params["hls_chunk_host"].match(/r(?<fvip>\d+)---/)
raw_params["fvip"] = fvip["fvip"]
end
raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
end
manifest
end
# /api/manifest/hls_variant/*
def self.get_hls_variant(env)
response = YT_POOL.client &.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
end
local = env.params.query["local"]?.try &.== "true"
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
manifest = response.body
if local
manifest = manifest.gsub("https://www.youtube.com", HOST_URL)
manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true")
end
manifest
end
end

490
src/invidious/routes/api/v1/authenticated.cr Normal file
View File

@@ -0,0 +1,490 @@
module Invidious::Routes::API::V1::Authenticated
# The notification APIs cannot be extracted yet!
# They require the *local* notifications constant defined in invidious.cr
#
# def self.notifications(env)
# env.response.content_type = "text/event-stream"
# topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000)
# topics ||= [] of String
# create_notification_stream(env, topics, connection_channel)
# end
def self.get_preferences(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
user.preferences.to_json
end
def self.set_preferences(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
begin
user.preferences = Preferences.from_json(env.request.body || "{}")
rescue
end
Invidious::Database::Users.update_preferences(user)
env.response.status_code = 204
end
def self.export_invidious(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
return Invidious::User::Export.to_invidious(user)
end
def self.import_invidious(env)
user = env.get("user").as(User)
begin
if body = env.request.body
body = env.request.body.not_nil!.gets_to_end
else
body = "{}"
end
Invidious::User::Import.from_invidious(user, body)
rescue
end
env.response.status_code = 204
end
def self.get_history(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX)
page ||= 1
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
start_index = (page - 1) * max_results
if user.watched[start_index]?
watched = user.watched.reverse[start_index, max_results]
end
watched ||= [] of String
return watched.to_json
end
def self.mark_watched(env)
user = env.get("user").as(User)
if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.")
end
id = env.params.url["id"]
if !id.match(/^[a-zA-Z0-9_-]{11}$/)
return error_json(400, "Invalid video id.")
end
Invidious::Database::Users.mark_watched(user, id)
env.response.status_code = 204
end
def self.mark_unwatched(env)
user = env.get("user").as(User)
if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.")
end
id = env.params.url["id"]
if !id.match(/^[a-zA-Z0-9_-]{11}$/)
return error_json(400, "Invalid video id.")
end
Invidious::Database::Users.mark_unwatched(user, id)
env.response.status_code = 204
end
def self.clear_history(env)
user = env.get("user").as(User)
Invidious::Database::Users.clear_watch_history(user)
env.response.status_code = 204
end
def self.feed(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
locale = env.get("preferences").as(Preferences).locale
max_results = env.params.query["max_results"]?.try &.to_i?
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
page = env.params.query["page"]?.try &.to_i?
page ||= 1
videos, notifications = get_subscription_feed(user, max_results, page)
JSON.build do |json|
json.object do
json.field "notifications" do
json.array do
notifications.each do |video|
video.to_json(locale, json)
end
end
end
json.field "videos" do
json.array do
videos.each do |video|
video.to_json(locale, json)
end
end
end
end
end
end
def self.get_subscriptions(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
subscriptions = Invidious::Database::Channels.select(user.subscriptions)
JSON.build do |json|
json.array do
subscriptions.each do |subscription|
json.object do
json.field "author", subscription.author
json.field "authorId", subscription.id
end
end
end
end
end
def self.subscribe_channel(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
ucid = env.params.url["ucid"]
if !user.subscriptions.includes? ucid
get_channel(ucid)
Invidious::Database::Users.subscribe_channel(user, ucid)
end
env.response.status_code = 204
end
def self.unsubscribe_channel(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
ucid = env.params.url["ucid"]
Invidious::Database::Users.unsubscribe_channel(user, ucid)
env.response.status_code = 204
end
def self.list_playlists(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
playlists = Invidious::Database::Playlists.select_all(author: user.email)
JSON.build do |json|
json.array do
playlists.each do |playlist|
playlist.to_json(0, json)
end
end
end
end
def self.create_playlist(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150)
if !title
return error_json(400, "Invalid title.")
end
privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) }
if !privacy
return error_json(400, "Invalid privacy setting.")
end
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_json(400, "User cannot have more than 100 playlists.")
end
playlist = create_playlist(title, privacy, user)
env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}"
env.response.status_code = 201
{
"title" => title,
"playlistId" => playlist.id,
}.to_json
end
def self.update_playlist_attribute(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
plid = env.params.url["plid"]?
if !plid || plid.empty?
return error_json(400, "A playlist ID is required")
end
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
end
title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title
privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) } || playlist.privacy
description = env.params.json["description"]?.try &.as(String).delete("\r") || playlist.description
if title != playlist.title ||
privacy != playlist.privacy ||
description != playlist.description
updated = Time.utc
else
updated = playlist.updated
end
Invidious::Database::Playlists.update(plid, title, privacy, description, updated)
env.response.status_code = 204
end
def self.delete_playlist(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
plid = env.params.url["plid"]
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
end
Invidious::Database::Playlists.delete(plid)
env.response.status_code = 204
end
def self.insert_video_into_playlist(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
plid = env.params.url["plid"]
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
end
if playlist.index.size >= CONFIG.playlist_length_limit
return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
video_id = env.params.json["videoId"].try &.as(String)
if !video_id
return error_json(403, "Invalid videoId")
end
begin
video = get_video(video_id)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
playlist_video = PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
length_seconds: video.length_seconds,
published: video.published,
plid: plid,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX),
})
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(plid, playlist_video.index)
env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}"
env.response.status_code = 201
JSON.build do |json|
playlist_video.to_json(json, index: playlist.index.size)
end
end
def self.delete_video_in_playlist(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
plid = env.params.url["plid"]
index = env.params.url["index"].to_i64(16)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
end
if !playlist.index.includes? index
return error_json(404, "Playlist does not contain index")
end
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(plid, index)
env.response.status_code = 204
end
# Invidious::Routing.patch "/api/v1/auth/playlists/:plid/videos/:index"
# def modify_playlist_at(env)
# TODO
# end
def self.get_tokens(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
scopes = env.get("scopes").as(Array(String))
tokens = Invidious::Database::SessionIDs.select_all(user.email)
JSON.build do |json|
json.array do
tokens.each do |token|
json.object do
json.field "session", token[:session]
json.field "issued", token[:issued].to_unix
end
end
end
end
end
def self.register_token(env)
user = env.get("user").as(User)
locale = env.get("preferences").as(Preferences).locale
case env.request.headers["Content-Type"]?
when "application/x-www-form-urlencoded"
scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
when "application/json"
scopes = env.params.json["scopes"].as(Array).map(&.as_s)
callback_url = env.params.json["callbackUrl"]?.try &.as(String)
expire = env.params.json["expire"]?.try &.as(Int64)
else
return error_json(400, "Invalid or missing header 'Content-Type'")
end
if callback_url && callback_url.empty?
callback_url = nil
end
if callback_url
callback_url = URI.parse(callback_url)
end
if sid = env.get?("sid").try &.as(String)
env.response.content_type = "text/html"
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true)
return templated "user/authorize_token"
else
env.response.content_type = "application/json"
superset_scopes = env.get("scopes").as(Array(String))
authorized_scopes = [] of String
scopes.each do |scope|
if scopes_include_scope(superset_scopes, scope)
authorized_scopes << scope
end
end
access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY)
if callback_url
access_token = URI.encode_www_form(access_token)
if query = callback_url.query
query = HTTP::Params.parse(query.not_nil!)
else
query = HTTP::Params.new
end
query["token"] = access_token
callback_url.query = query.to_s
env.redirect callback_url.to_s
else
access_token
end
end
end
def self.unregister_token(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
scopes = env.get("scopes").as(Array(String))
session = env.params.json["session"]?.try &.as(String)
session ||= env.get("session").as(String)
# Allow tokens to revoke other tokens with correct scope
if session == env.get("session").as(String)
Invidious::Database::SessionIDs.delete(sid: session)
elsif scopes_include_scope(scopes, "GET:tokens")
Invidious::Database::SessionIDs.delete(sid: session)
else
return error_json(400, "Cannot revoke session #{session}")
end
env.response.status_code = 204
end
def self.notifications(env)
env.response.content_type = "text/event-stream"
raw_topics = env.params.body["topics"]? || env.params.query["topics"]?
topics = raw_topics.try &.split(",").uniq.first(1000)
topics ||= [] of String
create_notification_stream(env, topics, CONNECTION_CHANNEL)
end
end

516
src/invidious/routes/api/v1/channels.cr Normal file
View File

@@ -0,0 +1,516 @@
module Invidious::Routes::API::V1::Channels
# Macro to avoid duplicating some code below
# This sets the `channel` variable, or handles Exceptions.
private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
end
def self.home(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve "sort by" setting from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
videos = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
end
next_continuation = nil
else
begin
videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex
return error_json(500, ex)
end
end
JSON.build do |json|
# TODO: Refactor into `to_json` for InvidiousChannel
json.object do
json.field "author", channel.author
json.field "authorId", channel.ucid
json.field "authorUrl", channel.author_url
json.field "authorBanners" do
json.array do
if channel.banner
qualities = {
{width: 2560, height: 424},
{width: 2120, height: 351},
{width: 1060, height: 175},
}
qualities.each do |quality|
json.object do
json.field "url", channel.banner.not_nil!.gsub("=w1060-", "=w#{quality[:width]}-")
json.field "width", quality[:width]
json.field "height", quality[:height]
end
end
json.object do
json.field "url", channel.banner.not_nil!.split("=w1060-")[0]
json.field "width", 512
json.field "height", 288
end
end
end
end
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "subCount", channel.sub_count
json.field "totalViews", channel.total_views
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
json.field "tabs", channel.tabs
json.field "tags", channel.tags
json.field "authorVerified", channel.verified
json.field "latestVideos" do
json.array do
videos.each do |video|
video.to_json(locale, json)
end
end
end
json.field "relatedChannels" do
json.array do
# Fetch related channels
begin
related_channels, _ = fetch_related_channels(channel)
rescue ex
related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
related_channel.to_json(locale, json)
end
end
end # relatedChannels
end
end
end
def self.latest(env)
# Remove parameters that could affect this endpoint's behavior
env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
return self.videos(env)
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve some URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
videos = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
end
next_continuation = nil
else
begin
videos, next_continuation = Channel::Tabs.get_60_videos(
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
end
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.shorts(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve continuation from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
videos = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
end
next_continuation = nil
else
begin
videos, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
end
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.streams(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve continuation from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
videos = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
end
next_continuation = nil
else
begin
videos, next_continuation = Channel::Tabs.get_60_livestreams(
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
end
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.playlists(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
continuation = env.params.query["continuation"]?
sort_by = env.params.query["sort"]?.try &.downcase ||
env.params.query["sort_by"]?.try &.downcase ||
"last"
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json|
json.object do
json.field "playlists" do
json.array do
items.each do |item|
item.to_json(locale, json) if item.is_a?(SearchPlaylist)
end
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.podcasts(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
continuation = env.params.query["continuation"]?
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation)
JSON.build do |json|
json.object do
json.field "playlists" do
json.array do
items.each do |item|
item.to_json(locale, json) if item.is_a?(SearchPlaylist)
end
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.releases(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
continuation = env.params.query["continuation"]?
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation)
JSON.build do |json|
json.object do
json.field "playlists" do
json.array do
items.each do |item|
item.to_json(locale, json) if item.is_a?(SearchPlaylist)
end
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.community(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
thin_mode = env.params.query["thin_mode"]?
thin_mode = thin_mode == "true"
format = env.params.query["format"]?
format ||= "json"
continuation = env.params.query["continuation"]?
# sort_by = env.params.query["sort_by"]?.try &.downcase
begin
fetch_channel_community(ucid, continuation, locale, format, thin_mode)
rescue ex
return error_json(500, ex)
end
end
def self.post(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
id = env.params.url["id"].to_s
ucid = env.params.query["ucid"]?
thin_mode = env.params.query["thin_mode"]?
thin_mode = thin_mode == "true"
format = env.params.query["format"]?
format ||= "json"
if ucid.nil?
response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
return error_json(400, "Invalid post ID") if response["error"]?
ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
else
ucid = ucid.to_s
end
begin
fetch_channel_community_post(ucid, id, locale, format, thin_mode)
rescue ex
return error_json(500, ex)
end
end
def self.post_comments(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
id = env.params.url["id"]
thin_mode = env.params.query["thin_mode"]?
thin_mode = thin_mode == "true"
format = env.params.query["format"]?
format ||= "json"
continuation = env.params.query["continuation"]?
case continuation
when nil, ""
ucid = env.params.query["ucid"]
comments = Comments.fetch_community_post_comments(ucid, id)
else
comments = YoutubeAPI.browse(continuation: continuation)
end
return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true)
end
def self.channels(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
continuation = env.params.query["continuation"]?
begin
items, next_continuation = fetch_related_channels(channel, continuation)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.object do
json.field "relatedChannels" do
json.array do
items.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
env.response.content_type = "application/json"
query = Invidious::Search::Query.new(env.params.query, :channel, region)
# Required because we can't (yet) pass multiple parameter to the
# `Search::Query` initializer (in this case, an URL segment)
query.channel = env.params.url["ucid"]
begin
search_results = query.process
rescue ex
return error_json(400, ex)
end
JSON.build do |json|
json.array do
search_results.each do |item|
item.to_json(locale, json)
end
end
end
end
# 301 redirect from /api/v1/channels/comments/:ucid
# and /api/v1/channels/:ucid/comments to new /api/v1/channels/:ucid/community and
# corresponding equivalent URL structure of the other one.
def self.channel_comments_redirect(env)
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
env.response.headers["Location"] = "/api/v1/channels/#{ucid}/community?#{env.params.query}"
env.response.status_code = 301
return
end
end

45
src/invidious/routes/api/v1/feeds.cr Normal file
View File

@@ -0,0 +1,45 @@
module Invidious::Routes::API::V1::Feeds
def self.trending(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
region = env.params.query["region"]?
trending_type = env.params.query["type"]?
begin
trending, plid = fetch_trending(trending_type, region, locale)
rescue ex
return error_json(500, ex)
end
videos = JSON.build do |json|
json.array do
trending.each do |video|
video.to_json(locale, json)
end
end
end
videos
end
def self.popular(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
haltf env, 403, error_message
end
JSON.build do |json|
json.array do
popular_videos.each do |video|
video.to_json(locale, json)
end
end
end
end
end

203
src/invidious/routes/api/v1/misc.cr Normal file
View File

@@ -0,0 +1,203 @@
module Invidious::Routes::API::V1::Misc
# Stats API endpoint for Invidious
def self.stats(env)
env.response.content_type = "application/json"
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
# Calculate playback success rate
if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
tracker = tracker.as(Hash(String, Int64 | Float64))
if !tracker.empty?
total_requests = tracker["totalRequests"]
success_count = tracker["successfulRequests"]
if total_requests.zero?
tracker["ratio"] = 1_i64
else
tracker["ratio"] = (success_count / (total_requests)).round(2)
end
end
end
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
# APIv1 currently uses the same logic for both
# user playlists and Invidious playlists. This means that we can't
# reasonably split them yet. This should be addressed in APIv2
def self.get_playlist(env : HTTP::Server::Context)
env.response.content_type = "application/json"
plid = env.params.url["plid"]
offset = env.params.query["index"]?.try &.to_i?
offset ||= env.params.query["page"]?.try &.to_i?.try { |page| (page - 1) * 100 }
offset ||= 0
video_id = env.params.query["continuation"]?
format = env.params.query["format"]?
format ||= "json"
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
begin
playlist = get_playlist(plid)
rescue ex : InfoException
return error_json(404, ex)
rescue ex
return error_json(404, "Playlist does not exist.")
end
user = env.get?("user").try &.as(User)
if !playlist || playlist.privacy.private? && playlist.author != user.try &.email
return error_json(404, "Playlist does not exist.")
end
# includes into the playlist a maximum of 20 videos, before the offset
if offset > 0
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
json_response = JSON.parse(response)
else
# Unless the continuation is really the offset 0, it becomes expensive.
# It happens when the offset is not set.
# First we find the actual offset, and then we lookback
# it shouldn't happen often though
lookback = 0
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
if json_response["videos"].as_a.empty?
json_response = JSON.parse(response)
elsif json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
json_response = JSON.parse(response)
end
end
if format == "html"
playlist_html = template_playlist(json_response)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
"playlistHtml" => playlist_html,
"index" => index,
"nextVideo" => next_video,
}.to_json
end
response
end
def self.mixes(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
rdid = env.params.url["rdid"]
continuation = env.params.query["continuation"]?
continuation ||= rdid.lchop("RD")[0, 11]
format = env.params.query["format"]?
format ||= "json"
begin
mix = fetch_mix(rdid, continuation, locale: locale)
if !rdid.ends_with? continuation
mix = fetch_mix(rdid, mix.videos[1].id)
index = mix.videos.index(mix.videos.select { |video| video.id == continuation }[0]?)
end
mix.videos = mix.videos[index..-1]
rescue ex
return error_json(500, ex)
end
response = JSON.build do |json|
json.object do
json.field "title", mix.title
json.field "mixId", mix.id
json.field "videos" do
json.array do
mix.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "author", video.author
json.field "authorId", video.ucid
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
json.array do
Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end
json.field "index", video.index
json.field "lengthSeconds", video.length_seconds
end
end
end
end
end
end
if format == "html"
response = JSON.parse(response)
playlist_html = template_mix(response)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
"playlistHtml" => playlist_html,
"nextVideo" => next_video,
}.to_json
end
response
end
# resolve channel and clip urls, return the UCID
def self.resolve_url(env)
env.response.content_type = "application/json"
url = env.params.query["url"]?
return error_json(400, "Missing URL to resolve") if !url
begin
resolved_url = YoutubeAPI.resolve_url(url.as(String))
endpoint = resolved_url["endpoint"]
page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
if page_type == "WEB_PAGE_TYPE_UNKNOWN"
return error_json(400, "Unknown url")
end
sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
params = sub_endpoint.try &.dig?("params")
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.object do
json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
json.field "params", params.try &.as_s
json.field "pageType", page_type
end
end
end
end

87
src/invidious/routes/api/v1/search.cr Normal file
View File

@@ -0,0 +1,87 @@
module Invidious::Routes::API::V1::Search
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
env.response.content_type = "application/json"
query = Invidious::Search::Query.new(env.params.query, :regular, region)
begin
search_results = query.process
rescue ex
return error_json(400, ex)
end
JSON.build do |json|
json.array do
search_results.each do |item|
item.to_json(locale, json)
end
end
end
end
def self.search_suggestions(env)
preferences = env.get("preferences").as(Preferences)
region = env.params.query["region"]? || preferences.region
env.response.content_type = "application/json"
query = env.params.query["q"]? || ""
begin
client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
response = client.get(url).body
client.close
body = JSON.parse(response[19..-2]).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
json.object do
json.field "query", body[0].as_s
json.field "suggestions" do
json.array do
suggestions.each do |suggestion|
json.string suggestion[0].as_s
end
end
end
end
end
rescue ex
return error_json(500, ex)
end
end
def self.hashtag(env)
hashtag = env.params.url["hashtag"]
page = env.params.query["page"]?.try &.to_i? || 1
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
env.response.content_type = "application/json"
begin
results = Invidious::Hashtag.fetch(hashtag, page, region)
rescue ex
return error_json(400, ex)
end
JSON.build do |json|
json.object do
json.field "results" do
json.array do
results.each do |item|
item.to_json(locale, json)
end
end
end
end
end
end
end

432
src/invidious/routes/api/v1/videos.cr Normal file
View File

@@ -0,0 +1,432 @@
require "html"
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]?
proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
end
end
def self.captions(env)
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]? || env.params.body["region"]?
if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/)
return error_json(400, "Invalid video ID")
end
# See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354
# It is possible to use `/api/timedtext?type=list&v=#{id}` and
# `/api/timedtext?type=track&v=#{id}&lang=#{lang_code}` directly,
# but this does not provide links for auto-generated captions.
#
# In future this should be investigated as an alternative, since it does not require
# getting video info.
begin
video = get_video(id, region: region)
rescue ex : NotFoundException
haltf env, 404
rescue ex
haltf env, 500
end
captions = video.captions
label = env.params.query["label"]?
lang = env.params.query["lang"]?
tlang = env.params.query["tlang"]?
if !label && !lang
response = JSON.build do |json|
json.object do
json.field "captions" do
json.array do
captions.each do |caption|
json.object do
json.field "label", caption.name
json.field "languageCode", caption.language_code
json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
end
end
end
end
end
end
return response
end
env.response.content_type = "text/vtt; charset=UTF-8"
if lang
caption = captions.select(&.language_code.== lang)
else
caption = captions.select(&.name.== label)
end
if caption.empty?
haltf env, 404
else
caption = caption[0]
end
if CONFIG.use_innertube_for_captions
params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
transcript = Invidious::Videos::Transcript.from_raw(
YoutubeAPI.get_transcript(params),
caption.language_code,
caption.auto_generated
)
webvtt = transcript.to_vtt
else
# Timedtext API handling
url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
# Auto-generated captions often have cues that aren't aligned properly with the video,
# as well as some other markup that makes it cumbersome, so we try to fix that here
if caption.name.includes? "auto-generated"
caption_xml = YT_POOL.client &.get(url).body
settings_field = {
"Kind" => "captions",
"Language" => "#{tlang || caption.language_code}",
}
if caption_xml.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(caption_xml, tlang)
else
caption_xml = XML.parse(caption_xml)
webvtt = WebVTT.build(settings_field) do |builder|
caption_nodes = caption_xml.xpath_nodes("//transcript/text")
caption_nodes.each_with_index do |node, i|
start_time = node["start"].to_f.seconds
duration = node["dur"]?.try &.to_f.seconds
duration ||= start_time
if caption_nodes.size > i + 1
end_time = caption_nodes[i + 1]["start"].to_f.seconds
else
end_time = start_time + duration
end
text = HTML.unescape(node.content)
text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
text = text.gsub(/<\/font>/, "")
if md = text.match(/(?<name>.*) : (?<text>.*)/)
text = "<v #{md["name"]}>#{md["text"]}</v>"
end
builder.cue(start_time, end_time, text)
end
end
end
else
uri = URI.parse(url)
query_params = uri.query_params
query_params["fmt"] = "vtt"
uri.query_params = query_params
webvtt = YT_POOL.client &.get(uri.request_target).body
if webvtt.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(webvtt)
else
# Some captions have "align:[start/end]" and "position:[num]%"
# attributes. Those are causing issues with VideoJS, which is unable
# to properly align the captions on the video, so we remove them.
#
# See: https://github.com/iv-org/invidious/issues/2391
webvtt = webvtt.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
end
end
if title = env.params.query["title"]?
# https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
end
webvtt
end
# Fetches YouTube storyboards
#
# Which are sprites containing x * y preview
# thumbnails for individual scenes in a video.
# See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails
def self.storyboards(env)
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]?
begin
video = get_video(id, region: region)
rescue ex : NotFoundException
haltf env, 404
rescue ex
haltf env, 500
end
width = env.params.query["width"]?.try &.to_i
height = env.params.query["height"]?.try &.to_i
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards)
end
end
end
return response
end
env.response.content_type = "text/vtt"
# Select a storyboard matching the user's provided width/height
storyboard = video.storyboards.select { |x| x.width == width || x.height == height }
haltf env, 404 if storyboard.empty?
# Alias variable, to make the code below esaier to read
sb = storyboard[0]
# Some base URL segments that we'll use to craft the final URLs
work_url = sb.proxied_url.dup
template_path = sb.proxied_url.path
# Initialize cue timing variables
# NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap
# (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000)
time_delta = sb.interval.milliseconds
start_time = 0.milliseconds
end_time = time_delta
# Build a VTT file for VideoJS-vtt plugin
vtt_file = WebVTT.build do |vtt|
sb.images_count.times do |i|
# Replace the variable component part of the path
work_url.path = template_path.sub("$M", i)
sb.rows.times do |j|
sb.columns.times do |k|
# The URL fragment represents the offset of the thumbnail inside the storyboard image
work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}"
vtt.cue(start_time, end_time, work_url.to_s)
start_time += time_delta
end_time += time_delta
end
end
end
end
# videojs-vtt-thumbnails is not compliant to the VTT specification, it
# doesn't unescape the HTML entities, so we have to do it here:
# TODO: remove this when we migrate to VideoJS 8
return HTML.unescape(vtt_file)
end
def self.annotations(env)
env.response.content_type = "text/xml"
id = env.params.url["id"]
source = env.params.query["source"]?
source ||= "archive"
if !id.match(/[a-zA-Z0-9_-]{11}/)
haltf env, 400
end
annotations = ""
case source
when "archive"
if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
annotations = cached_annotation.annotations
else
index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
# IA doesn't handle leading hyphens,
# so we use https://archive.org/details/youtubeannotations_64
if index == "62"
index = "64"
id = id.sub(/^-/, 'A')
end
file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
if !location.headers["Location"]?
env.response.status_code = location.status_code
end
response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
if response.body.empty?
haltf env, 404
end
if response.status_code != 200
haltf env, response.status_code
end
annotations = response.body
cache_annotation(id, annotations)
end
else # "youtube"
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
if response.status_code != 200
haltf env, response.status_code
end
annotations = response.body
end
etag = sha256(annotations)[0, 16]
if env.request.headers["If-None-Match"]?.try &.== etag
haltf env, 304
else
env.response.headers["ETag"] = etag
annotations
end
end
def self.comments(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
env.response.content_type = "application/json"
id = env.params.url["id"]
source = env.params.query["source"]?
source ||= "youtube"
thin_mode = env.params.query["thin_mode"]?
thin_mode = thin_mode == "true"
format = env.params.query["format"]?
format ||= "json"
action = env.params.query["action"]?
action ||= "action_get_comments"
continuation = env.params.query["continuation"]?
sort_by = env.params.query["sort_by"]?.try &.downcase
if source == "youtube"
sort_by ||= "top"
begin
comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
return comments
elsif source == "reddit"
sort_by ||= "confidence"
begin
comments, reddit_thread = Comments.fetch_reddit(id, sort_by: sort_by)
rescue ex
comments = nil
reddit_thread = nil
end
if !reddit_thread || !comments
return error_json(404, "No reddit threads found")
end
if format == "json"
reddit_thread = JSON.parse(reddit_thread.to_json).as_h
reddit_thread["comments"] = JSON.parse(comments.to_json)
return reddit_thread.to_json
else
content_html = Frontend::Comments.template_reddit(comments, locale)
content_html = Comments.fill_links(content_html, "https", "www.reddit.com")
content_html = Comments.replace_links(content_html)
response = {
"title" => reddit_thread.title,
"permalink" => reddit_thread.permalink,
"contentHtml" => content_html,
}
return response.to_json
end
end
end
def self.clips(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
clip_id = env.params.url["id"]
region = env.params.query["region"]?
proxy = {"1", "true"}.any? &.== env.params.query["local"]?
response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
return error_json(400, "Invalid clip ID") if response["error"]?
video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
return error_json(400, "Invalid clip ID") if video_id.nil?
start_time = nil
end_time = nil
clip_title = nil
if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
start_time, end_time, clip_title = parse_clip_parameters(params)
end
begin
video = get_video(video_id, region: region)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
json.object do
json.field "startTime", start_time
json.field "endTime", end_time
json.field "clipTitle", clip_title
json.field "video" do
Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
end
end
end
end
end

2
src/invidious/routes/base_route.cr
View File

@@ -1,2 +0,0 @@
abstract class Invidious::Routes::BaseRoute
end

126
src/invidious/routes/before_all.cr Normal file
View File

@@ -0,0 +1,126 @@
module Invidious::Routes::BeforeAll
def self.handle(env)
preferences = Preferences.from_json("{}")
begin
if prefs_cookie = env.request.cookies["PREFS"]?
preferences = Preferences.from_json(URI.decode_www_form(prefs_cookie.value))
else
if language_header = env.request.headers["Accept-Language"]?
if language = ANG.language_negotiator.best(language_header, LOCALES.keys)
preferences.locale = language.header
end
end
end
rescue
preferences = Preferences.from_json("{}")
end
env.set "preferences", preferences
env.response.headers["X-XSS-Protection"] = "1; mode=block"
env.response.headers["X-Content-Type-Options"] = "nosniff"
# Allow media resources to be loaded from google servers
# TODO: check if *.youtube.com can be removed
if CONFIG.disabled?("local") || !preferences.local
extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443"
else
extra_media_csp = ""
end
# Only allow the pages at /embed/* to be embedded
if env.request.resource.starts_with?("/embed")
frame_ancestors = "'self' file: http: https:"
else
frame_ancestors = "'none'"
end
# TODO: Remove style-src's 'unsafe-inline', requires to remove all
# inline styles (<style> [..] </style>, style=" [..] ")
env.response.headers["Content-Security-Policy"] = {
"default-src 'none'",
"script-src 'self'",
"style-src 'self' 'unsafe-inline'",
"img-src 'self' data:",
"font-src 'self' data:",
"connect-src 'self'",
"manifest-src 'self'",
"media-src 'self' blob:" + extra_media_csp,
"child-src 'self' blob:",
"frame-src 'self'",
"frame-ancestors " + frame_ancestors,
}.join("; ")
env.response.headers["Referrer-Policy"] = "same-origin"
# Ask the chrom*-based browsers to disable FLoC
# See: https://blog.runcloud.io/google-floc/
env.response.headers["Permissions-Policy"] = "interest-cohort=()"
if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts
env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload"
end
return if {
"/sb/",
"/vi/",
"/s_p/",
"/yts/",
"/ggpht/",
"/api/manifest/",
"/videoplayback",
"/latest_version",
"/download",
}.any? { |r| env.request.resource.starts_with? r }
if env.request.cookies.has_key? "SID"
sid = env.request.cookies["SID"].value
if sid.starts_with? "v1:"
raise "Cannot use token as SID"
end
if email = Database::SessionIDs.select_email(sid)
user = Database::Users.select!(email: email)
csrf_token = generate_response(sid, {
":authorize_token",
":playlist_ajax",
":signout",
":subscription_ajax",
":token_ajax",
":watch_ajax",
}, HMAC_KEY, 1.week)
preferences = user.preferences
env.set "preferences", preferences
env.set "sid", sid
env.set "csrf_token", csrf_token
env.set "user", user
end
end
dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s
thin_mode = thin_mode == "true"
locale = env.params.query["hl"]? || preferences.locale
preferences.dark_mode = dark_mode
preferences.thin_mode = thin_mode
preferences.locale = locale
env.set "preferences", preferences
current_page = env.request.path
if env.request.query
query = HTTP::Params.parse(env.request.query.not_nil!)
if query["referer"]?
query["referer"] = get_referer(env, "/")
end
current_page += "?#{query}"
end
env.set "current_page", URI.encode_www_form(current_page)
end
end

423
src/invidious/routes/channels.cr Normal file
View File

@@ -0,0 +1,423 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Channels
# Redirection for unsupported routes ("tabs")
def self.redirect_home(env)
ucid = env.params.url["ucid"]
return env.redirect "/channel/#{URI.encode_www_form(ucid)}"
end
def self.home(env)
self.videos(env)
end
def self.videos(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_by ||= "last"
sort_options = {"last", "oldest", "newest"}
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, sort_by
)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
elsif item.responds_to?(:author)
item.author
end
end
items = items.select(SearchPlaylist)
items.each(&.author = "")
else
# Fetch items and continuation token
if channel.is_age_gated
sort_by = ""
sort_options = [] of String
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
items = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
items = [] of PlaylistVideo
end
next_continuation = nil
else
sort_by ||= "newest"
sort_options = {"newest", "oldest", "popular"}
items, next_continuation = Channel::Tabs.get_60_videos(
channel, continuation: continuation, sort_by: sort_by
)
end
end
selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
templated "channel"
end
def self.shorts(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if !channel.tabs.includes? "shorts"
return env.redirect "/channel/#{channel.ucid}"
end
if channel.is_age_gated
sort_by = ""
sort_options = [] of String
begin
playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
items = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
items = [] of PlaylistVideo
end
next_continuation = nil
else
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
sort_options = {"newest", "oldest", "popular"}
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation, sort_by: sort_by
)
end
selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
templated "channel"
end
def self.streams(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if !channel.tabs.includes? "streams"
return env.redirect "/channel/#{channel.ucid}"
end
if channel.is_age_gated
sort_by = ""
sort_options = [] of String
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
items = get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
items = [] of PlaylistVideo
end
next_continuation = nil
else
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
sort_options = {"newest", "oldest", "popular"}
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_60_livestreams(
channel, continuation: continuation, sort_by: sort_by
)
end
selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
)
items = items.select(SearchPlaylist)
items.each(&.author = "")
selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
templated "channel"
end
def self.podcasts(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
sort_by = ""
sort_options = [] of String
items, next_continuation = fetch_channel_podcasts(
channel.ucid, channel.author, continuation
)
items = items.select(SearchPlaylist)
items.each(&.author = "")
selected_tab = Frontend::ChannelPage::TabsAvailable::Podcasts
templated "channel"
end
def self.releases(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
sort_by = ""
sort_options = [] of String
items, next_continuation = fetch_channel_releases(
channel.ucid, channel.author, continuation
)
items = items.select(SearchPlaylist)
items.each(&.author = "")
selected_tab = Frontend::ChannelPage::TabsAvailable::Releases
templated "channel"
end
def self.community(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
# redirect to post page
if lb = env.params.query["lb"]?
env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
end
thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort options for community posts
sort_by = ""
sort_options = [] of String
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
env.response.status_code = 500
error_message = ex.message
rescue ex : NotFoundException
env.response.status_code = 404
error_message = ex.message
rescue ex
return error_template(500, ex)
end
templated "community"
end
def self.post(env)
# /post/{postId}
id = env.params.url["id"]
ucid = env.params.query["ucid"]?
prefs = env.get("preferences").as(Preferences)
locale = prefs.locale
thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode
thin_mode = thin_mode == "true"
nojs = env.params.query["nojs"]?
nojs ||= "0"
nojs = nojs == "1"
if !ucid.nil?
ucid = ucid.to_s
post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
else
# resolve the url to get the author's UCID
response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
return error_template(400, "Invalid post ID") if response["error"]?
ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
end
post_response = JSON.parse(post_response)
if nojs
comments = Comments.fetch_community_post_comments(ucid, id)
comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"]
end
templated "post"
end
def self.channels(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
items, next_continuation = fetch_related_channels(channel, continuation)
# Featured/related channels can't be sorted
sort_options = [] of String
sort_by = nil
selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
templated "channel"
end
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
env.redirect "/channel/#{ucid}"
end
private KNOWN_TABS = {
"home", "videos", "shorts", "streams", "podcasts",
"releases", "playlists", "community", "channels", "about",
}
# Redirects brand url channels to a normal /channel/:ucid route
def self.brand_redirect(env)
locale = env.get("preferences").as(Preferences).locale
# /attribution_link endpoint needs both the `a` and `u` parameter
# and in order to avoid detection from YouTube we should only send the required ones
# without any of the additional url parameters that only Invidious uses.
yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"]))
# Retrieves URL params that only Invidious uses
invidious_url_params = env.params.query.dup
invidious_url_params.delete_all("a")
invidious_url_params.delete_all("u")
invidious_url_params.delete_all("user")
begin
resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"]
rescue ex : InfoException | KeyError
return error_template(404, translate(locale, "This channel does not exist."))
end
selected_tab = env.params.url["tab"]?
if KNOWN_TABS.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
end
url += "?#{invidious_url_params}" if !invidious_url_params.empty?
return env.redirect url
end
# Handles redirects for the /profile endpoint
def self.profile(env)
# The /profile endpoint is special. If passed into the resolve_url
# endpoint YouTube would return a sign in page instead of an /channel/:ucid
# thus we'll add an edge case and handle it here.
uri_params = env.params.query.size > 0 ? "?#{env.params.query}" : ""
user = env.params.query["user"]?
if !user
return error_template(404, "This channel does not exist.")
else
env.redirect "/user/#{user}#{uri_params}"
end
end
def self.live(env)
locale = env.get("preferences").as(Preferences).locale
# Appears to be a bug in routing, having several routes configured
# as `/a/:a`, `/b/:a`, `/c/:a` results in 404
value = env.request.resource.split("/")[2]
body = ""
{"channel", "user", "c"}.each do |type|
response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
if response.status_code == 200
body = response.body
end
end
video_id = body.match(/'VIDEO_ID': "(?<id>[a-zA-Z0-9_-]{11})"/).try &.["id"]?
if video_id
params = [] of String
env.params.query.each do |k, v|
params << "#{k}=#{v}"
end
params = params.join("&")
url = "/watch?v=#{video_id}"
if !params.empty?
url += "&#{params}"
end
env.redirect url
else
env.redirect "/channel/#{value}"
end
end
private def self.fetch_basic_information(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
if user
user = user.as(User)
subscriptions = user.subscriptions
end
subscriptions ||= [] of String
ucid = env.params.url["ucid"]
continuation = env.params.query["continuation"]?
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
env.set "search", "channel:#{ucid} "
return {locale, user, subscriptions, continuation, ucid, channel}
end
end

51
src/invidious/routes/embed.cr
View File

@@ -1,12 +1,19 @@
class Invidious::Routes::Embed < Invidious::Routes::BaseRoute
def redirect(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Embed
def self.redirect(env)
locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(PG_DB, plid, locale: locale)
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
videos = get_playlist_videos(playlist, offset: offset)
if videos.empty?
url = "/playlist?list=#{plid}"
raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
end
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
@@ -23,12 +30,12 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute
env.redirect url
end
def show(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.show(env)
locale = env.get("preferences").as(Preferences).locale
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(PG_DB, env.params.query, plid, id)
continuation = process_continuation(env.params.query, plid, id)
if md = env.params.query["playlist"]?
.try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/)
@@ -58,9 +65,15 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute
if plid
begin
playlist = get_playlist(PG_DB, plid, locale: locale)
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
videos = get_playlist_videos(playlist, offset: offset)
if videos.empty?
url = "/playlist?list=#{plid}"
raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
end
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
@@ -117,9 +130,9 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute
subscriptions ||= [] of String
begin
video = get_video(id, PG_DB, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
video = get_video(id, region: params.region)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
@@ -134,8 +147,8 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
if notifications && notifications.includes? id
PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email)
if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
end
@@ -165,12 +178,12 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute
captions = video.captions
preferred_captions = captions.select { |caption|
params.preferred_captions.includes?(caption.name.simpleText) ||
params.preferred_captions.includes?(caption.languageCode.split("-")[0])
params.preferred_captions.includes?(caption.name) ||
params.preferred_captions.includes?(caption.language_code.split("-")[0])
}
preferred_captions.sort_by! { |caption|
(params.preferred_captions.index(caption.name.simpleText) ||
params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil!
(params.preferred_captions.index(caption.name) ||
params.preferred_captions.index(caption.language_code.split("-")[0])).not_nil!
}
captions = captions - preferred_captions

52
src/invidious/routes/errors.cr Normal file
View File

@@ -0,0 +1,52 @@
module Invidious::Routes::ErrorRoutes
def self.error_404(env)
# Workaround for #3117
if HOST_URL.empty? && env.request.path.starts_with?("/v1/storyboards/sb")
return env.redirect "#{env.request.path[15..]}?#{env.params.query}"
end
if md = env.request.path.match(/^\/(?<id>([a-zA-Z0-9_-]{11})|(\w+))$/)
item = md["id"]
# Check if item is branding URL e.g. https://youtube.com/gaming
response = YT_POOL.client &.get("/#{item}")
if response.status_code == 301
response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
end
if response.body.empty?
env.response.headers["Location"] = "/"
haltf env, status_code: 302
end
html = XML.parse_html(response.body)
ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
if ucid
env.response.headers["Location"] = "/channel/#{ucid}"
haltf env, status_code: 302
end
params = [] of String
env.params.query.each do |k, v|
params << "#{k}=#{v}"
end
params = params.join("&")
url = "/watch?v=#{item}"
if !params.empty?
url += "&#{params}"
end
# Check if item is video ID
if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
env.response.headers["Location"] = url
haltf env, status_code: 302
end
end
env.response.headers["Location"] = "/"
haltf env, status_code: 302
end
end

462
src/invidious/routes/feeds.cr Normal file
View File

@@ -0,0 +1,462 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Feeds
def self.view_all_playlists_redirect(env)
env.redirect "/feed/playlists"
end
def self.playlists(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
referer = get_referer(env)
return env.redirect "/" if user.nil?
user = user.as(User)
# TODO: make a single DB call and separate the items here?
items_created = Invidious::Database::Playlists.select_like_iv(user.email)
items_created.map! do |item|
item.author = ""
item
end
items_saved = Invidious::Database::Playlists.select_not_like_iv(user.email)
items_saved.map! do |item|
item.author = ""
item
end
templated "feeds/playlists"
end
def self.popular(env)
locale = env.get("preferences").as(Preferences).locale
if CONFIG.popular_enabled
templated "feeds/popular"
else
message = translate(locale, "The Popular feed has been disabled by the administrator.")
templated "message"
end
end
def self.trending(env)
locale = env.get("preferences").as(Preferences).locale
trending_type = env.params.query["type"]?
trending_type ||= "Default"
region = env.params.query["region"]?
region ||= env.get("preferences").as(Preferences).region
begin
trending, plid = fetch_trending(trending_type, region, locale)
rescue ex
return error_template(500, ex)
end
templated "feeds/trending"
end
def self.subscriptions(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
sid = sid.as(String)
token = user.token
if user.preferences.unseen_only
env.set "show_watched", true
end
# Refresh account
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
page = env.params.query["page"]?.try &.to_i?
page ||= 1
videos, notifications = get_subscription_feed(user, max_results, page)
if CONFIG.enable_user_notifications
# "updated" here is used for delivering new notifications, so if
# we know a user has looked at their feed e.g. in the past 10 minutes,
# they've already seen a video posted 20 minutes ago, and don't need
# to be notified.
Invidious::Database::Users.clear_notifications(user)
user.notifications = [] of String
end
env.set "user", user
# Used for pagination links
base_url = "/feed/subscriptions"
base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
templated "feeds/subscriptions"
end
def self.history(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
referer = get_referer(env)
page = env.params.query["page"]?.try &.to_i?
page ||= 1
if !user
return env.redirect referer
end
user = user.as(User)
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
if user.watched[(page - 1) * max_results]?
watched = user.watched.reverse[(page - 1) * max_results, max_results]
end
watched ||= [] of String
# Used for pagination links
base_url = "/feed/history"
base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
templated "feeds/history"
end
# RSS feeds
def self.rss_channel(env)
locale = env.get("preferences").as(Preferences).locale
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
ucid = env.params.url["ucid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
rescue ex : NotFoundException
return error_atom(404, ex)
rescue ex
return error_atom(500, ex)
end
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
title = entry.xpath_node("default:title", namespaces).not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
SearchVideo.new({
title: title,
id: video_id,
author: author,
ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
premiere_timestamp: nil,
author_verified: false,
badges: VideoBadges::None,
})
end
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
xml.element("yt:channelId") { xml.text channel.ucid }
xml.element("icon") { xml.text channel.author_thumbnail }
xml.element("title") { xml.text channel.author }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
xml.element("author") do
xml.element("name") { xml.text channel.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
end
xml.element("image") do
xml.element("url") { xml.text channel.author_thumbnail }
xml.element("title") { xml.text channel.author }
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
video.to_xml(channel.auto_generated, params, xml)
end
end
end
end
def self.rss_private(env)
locale = env.get("preferences").as(Preferences).locale
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
token = env.params.query["token"]?
if !token
haltf env, status_code: 403
end
user = Invidious::Database::Users.select(token: token.strip)
if !user
haltf env, status_code: 403
end
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
page = env.params.query["page"]?.try &.to_i?
page ||= 1
params = HTTP::Params.parse(env.params.query["params"]? || "")
videos, notifications = get_subscription_feed(user, max_results, page)
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions")
xml.element("link", "type": "application/atom+xml", rel: "self",
href: "#{HOST_URL}#{env.request.resource}")
xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) }
(notifications + videos).each do |video|
video.to_xml(locale, params, xml)
end
end
end
end
def self.rss_playlist(env)
locale = env.get("preferences").as(Preferences).locale
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
plid = env.params.url["plid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
path = env.request.path
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
videos = get_playlist_videos(playlist, offset: 0)
return XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
xml.element("id") { xml.text "iv:playlist:#{plid}" }
xml.element("iv:playlistId") { xml.text plid }
xml.element("title") { xml.text playlist.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}")
xml.element("author") do
xml.element("name") { xml.text playlist.author }
end
videos.each &.to_xml(xml)
end
end
else
haltf env, status_code: 404
end
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
when "url", "href"
request_target = URI.parse(node[attribute.name]).request_target
query_string_opt = request_target.starts_with?("/watch?v=") ? "&#{params}" : ""
node[attribute.name] = "#{HOST_URL}#{request_target}#{query_string_opt}"
else nil # Skip
end
end
end
document = document.to_xml(options: XML::SaveOptions::NO_DECL)
document.scan(/<uri>(?<url>[^<]+)<\/uri>/).each do |match|
content = "#{HOST_URL}#{URI.parse(match["url"]).request_target}"
document = document.gsub(match[0], "<uri>#{content}</uri>")
end
document
end
def self.rss_videos(env)
if ucid = env.params.query["channel_id"]?
env.redirect "/feed/channel/#{ucid}"
elsif user = env.params.query["user"]?
env.redirect "/feed/channel/#{user}"
elsif plid = env.params.query["playlist_id"]?
env.redirect "/feed/playlist/#{plid}"
end
end
# Push notifications via PubSub
def self.push_notifications_get(env)
verify_token = env.params.url["token"]
mode = env.params.query["hub.mode"]?
topic = env.params.query["hub.topic"]?
challenge = env.params.query["hub.challenge"]?
if !mode || !topic || !challenge
haltf env, status_code: 400
else
mode = mode.not_nil!
topic = topic.not_nil!
challenge = challenge.not_nil!
end
case verify_token
when .starts_with? "v1"
_, time, nonce, signature = verify_token.split(":")
data = "#{time}:#{nonce}"
when .starts_with? "v2"
time, signature = verify_token.split(":")
data = "#{time}"
else
haltf env, status_code: 400
end
# The hub will sometimes check if we're still subscribed after delivery errors,
# so we reply with a 200 as long as the request hasn't expired
if Time.utc.to_unix - time.to_i > 432000
haltf env, status_code: 400
end
if OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, data) != signature
haltf env, status_code: 400
end
if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]?
Invidious::Database::Channels.update_subscription_time(ucid)
elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]?
Invidious::Database::Playlists.update_subscription_time(plid)
else
haltf env, status_code: 400
end
env.response.status_code = 200
challenge
end
def self.push_notifications_post(env)
locale = env.get("preferences").as(Preferences).locale
token = env.params.url["token"]
body = env.request.body.not_nil!.gets_to_end
signature = env.request.headers["X-Hub-Signature"].lchop("sha1=")
if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body)
LOGGER.error("/feed/webhook/#{token} : Invalid signature")
haltf env, status_code: 200
end
spawn do
# TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"default" => "http://www.w3.org/2005/Atom",
}
rss = XML.parse(body)
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
begin
video = get_video(id, force_refresh: true)
rescue
next # skip this video since it raised an exception (e.g. it is a scheduled live event)
end
if CONFIG.enable_user_notifications
# Deliver notifications to `/api/v1/auth/notifications`
payload = {
"topic" => video.ucid,
"videoId" => video.id,
"published" => published.to_unix,
}.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'")
end
video = ChannelVideo.new({
id: id,
title: video.title,
published: published,
updated: updated,
ucid: video.ucid,
author: author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
if was_insert
if CONFIG.enable_user_notifications
Invidious::Database::Users.add_notification(video)
else
Invidious::Database::Users.feed_needs_update(video)
end
end
end
end
env.response.status_code = 200
end
end

153
src/invidious/routes/images.cr Normal file
View File

@@ -0,0 +1,153 @@
module Invidious::Routes::Images
# Avatars, banners and other large image assets.
def self.ggpht(env)
url = env.request.path.lchop("/ggpht")
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
begin
GGPHT_POOL.client &.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
end
end
def self.options_storyboard(env)
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
def self.get_storyboard(env)
authority = env.params.url["authority"]
id = env.params.url["id"]
storyboard = env.params.url["storyboard"]
index = env.params.url["index"]
url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}"
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
begin
get_ytimg_pool(authority).client &.get(url, headers) do |resp|
env.response.headers["Connection"] = "close"
return self.proxy_image(env, resp)
end
rescue ex
end
end
# ??? maybe also for storyboards?
def self.s_p_image(env)
id = env.params.url["id"]
name = env.params.url["name"]
url = env.request.resource
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
begin
get_ytimg_pool("i9").client &.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
end
end
def self.yts_image(env)
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
begin
YT_POOL.client &.get(env.request.resource, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
env.response.headers.delete("Transfer-Encoding")
break
end
proxy_file(response, env)
end
rescue ex
end
end
def self.thumbnails(env)
id = env.params.url["id"]
name = env.params.url["name"]
headers = HTTP::Headers.new
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
end
end
url = "/vi/#{id}/#{name}"
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
begin
get_ytimg_pool("i").client &.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
end
end
private def self.proxy_image(env, response)
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
return env.response.headers.delete("Transfer-Encoding")
end
return proxy_file(response, env)
end
end

368
src/invidious/routes/login.cr
View File

@@ -1,17 +1,19 @@
class Invidious::Routes::Login < Invidious::Routes::BaseRoute
def login_page(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Login
def self.login_page(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
return env.redirect "/feed/subscriptions" if user
referer = get_referer(env, "/feed/subscriptions")
return env.redirect referer if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
end
referer = get_referer(env, "/feed/subscriptions")
email = nil
password = nil
captcha = nil
@@ -22,14 +24,11 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
captcha_type = env.params.query["captcha"]?
captcha_type ||= "image"
tfa = env.params.query["tfa"]?
prompt = nil
templated "login"
templated "user/login"
end
def login(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.login(env)
locale = env.get("preferences").as(Preferences).locale
referer = get_referer(env, "/feed/subscriptions")
@@ -45,304 +44,23 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
account_type ||= "invidious"
case account_type
when "google"
tfa_code = env.params.body["tfa"]?.try &.lchop("G-")
traceback = IO::Memory.new
# See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
begin
client = QUIC::Client.new(LOGIN_URL)
headers = HTTP::Headers.new
login_page = client.get("/ServiceLogin")
headers = login_page.cookies.add_request_headers(headers)
lookup_req = {
email, nil, [] of String, nil, "US", nil, nil, 2, false, true,
{nil, nil,
{2, 1, nil, 1,
"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
nil, [] of String, 4},
1,
{nil, nil, [] of String},
nil, nil, nil, true,
},
email,
}.to_json
traceback << "Getting lookup..."
headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
headers["Google-Accounts-XSRF"] = "1"
response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
lookup_results = JSON.parse(response.body[5..-1])
traceback << "done, returned #{response.status_code}.<br/>"
user_hash = lookup_results[0][2]
if token = env.params.body["token"]?
answer = env.params.body["answer"]?
captcha = {token, answer}
else
captcha = nil
end
challenge_req = {
user_hash, nil, 1, nil,
{1, nil, nil, nil,
{password, captcha, true},
},
{nil, nil,
{2, 1, nil, 1,
"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
nil, [] of String, 4},
1,
{nil, nil, [] of String},
nil, nil, nil, true,
},
}.to_json
traceback << "Getting challenge..."
response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req))
headers = response.cookies.add_request_headers(headers)
challenge_results = JSON.parse(response.body[5..-1])
traceback << "done, returned #{response.status_code}.<br/>"
headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
if challenge_results[0][3]?.try &.== 7
return error_template(423, "Account has temporarily been disabled")
end
if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s
account_type = "google"
captcha_type = "image"
prompt = nil
tfa = tfa_code
captcha = {tokens: [token], question: ""}
return templated "login"
end
if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
return error_template(401, "Incorrect password")
end
prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]?
if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type
traceback << "Handling prompt #{prompt_type}.<br/>"
case prompt_type
when "TWO_STEP_VERIFICATION"
prompt_type = 2
else # "LOGIN_CHALLENGE"
prompt_type = 4
end
# Prefer Authenticator app and SMS over unsupported protocols
if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2
tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0]
traceback << "Selecting challenge #{tfa[8]}..."
select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json
tl = challenge_results[1][2]
tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body
tfa = tfa[5..-1]
tfa = JSON.parse(tfa)[0][-1]
traceback << "done.<br/>"
else
traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.<br/>"
tfa = challenge_results[0][-1][0][0]
end
if tfa[5] == "QUOTA_EXCEEDED"
return error_template(423, "Quota exceeded, try again in a few hours")
end
if !tfa_code
account_type = "google"
captcha_type = "image"
case tfa[8]
when 6, 9
prompt = "Google verification code"
when 12
prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
when 15
prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
else
prompt = "Google verification code"
end
tfa = nil
captcha = nil
return templated "login"
end
tl = challenge_results[1][2]
request_type = tfa[8]
case request_type
when 6 # Authenticator app
tfa_req = {
user_hash, nil, 2, nil,
{6, nil, nil, nil, nil,
{tfa_code, false},
},
}.to_json
when 9 # Voice or text message
tfa_req = {
user_hash, nil, 2, nil,
{9, nil, nil, nil, nil, nil, nil, nil,
{nil, tfa_code, false, 2},
},
}.to_json
when 12 # Recovery email
tfa_req = {
user_hash, nil, 4, nil,
{12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
{tfa_code},
},
}.to_json
when 15 # Security question
tfa_req = {
user_hash, nil, 5, nil,
{15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
{tfa_code},
},
}.to_json
else
return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.")
end
traceback << "Submitting challenge..."
response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req))
headers = response.cookies.add_request_headers(headers)
challenge_results = JSON.parse(response.body[5..-1])
if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") ||
(challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT")
return error_template(401, "Invalid TFA code")
end
traceback << "done.<br/>"
end
traceback << "Logging in..."
location = URI.parse(challenge_results[0][-1][2].to_s)
cookies = HTTP::Cookies.from_headers(headers)
headers.delete("Content-Type")
headers.delete("Google-Accounts-XSRF")
loop do
if !location || location.path == "/ManageAccount"
break
end
# Occasionally there will be a second page after login confirming
# the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle.
if location.path.starts_with? "/b/0/SmsAuthInterstitial"
traceback << "Unhandled dialog /b/0/SmsAuthInterstitial."
end
login = client.get(location.request_target, headers)
headers = login.cookies.add_request_headers(headers)
location = login.headers["Location"]?.try { |u| URI.parse(u) }
end
cookies = HTTP::Cookies.from_headers(headers)
sid = cookies["SID"]?.try &.value
if !sid
raise "Couldn't get SID."
end
user, sid = get_user(sid, headers, PG_DB)
# We are now logged in
traceback << "done.<br/>"
host = URI.parse(env.request.headers["Host"]).host
if Kemal.config.ssl || CONFIG.https_only
secure = true
else
secure = false
end
cookies.each do |cookie|
if Kemal.config.ssl || CONFIG.https_only
cookie.secure = secure
else
cookie.secure = secure
end
if cookie.extension
cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "")
end
env.response.cookies << cookie
end
if env.request.cookies["PREFS"]?
preferences = env.get("preferences").as(Preferences)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email)
cookie = env.request.cookies["PREFS"]
cookie.expires = Time.utc(1990, 1, 1)
env.response.cookies << cookie
end
env.redirect referer
rescue ex
traceback.rewind
# error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.")
error_message = %(#{ex.message}<br/>Traceback:<br/><div style="padding-left:2em" id="traceback">#{traceback.gets_to_end}</div>)
return error_template(500, error_message)
end
when "invidious"
if !email
if email.nil? || email.empty?
return error_template(401, "User ID is a required field")
end
if !password
if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User)
user = Invidious::Database::Users.select(email: email)
if user
if !user.password
return error_template(400, "Please sign in using 'Log in with Google'")
end
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc)
Invidious::Database::SessionIDs.insert(sid, email)
if Kemal.config.ssl || CONFIG.https_only
secure = true
else
secure = false
end
if CONFIG.domain
env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years,
secure: secure, http_only: true)
else
env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years,
secure: secure, http_only: true)
end
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
else
return error_template(401, "Wrong username or password")
end
@@ -381,19 +99,17 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
captcha_type ||= "image"
account_type = "invidious"
tfa = false
prompt = ""
if captcha_type == "image"
captcha = generate_captcha(HMAC_KEY, PG_DB)
captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
else
captcha = generate_text_captcha(HMAC_KEY, PG_DB)
captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
end
return templated "login"
return templated "user/login"
end
tokens = env.params.body.select { |k, v| k.match(/^token\[\d+\]$/) }.map { |k, v| v }
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
answer ||= ""
captcha_type ||= "image"
@@ -404,7 +120,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
begin
validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale)
validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
@@ -417,9 +133,9 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
found_valid_captcha = false
error_exception = Exception.new
tokens.each_with_index do |token, i|
tokens.each do |tok|
begin
validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale)
validate_request(tok, answer, env.request, HMAC_KEY, locale)
found_valid_captcha = true
rescue ex
error_exception = ex
@@ -434,34 +150,24 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user, sid = create_user(sid, email, password)
user_array = user.to_a
user_array[4] = user_array[4].to_json # User preferences
args = arg_array(user_array)
if language_header = env.request.headers["Accept-Language"]?
if language = ANG.language_negotiator.best(language_header, LOCALES.keys)
user.preferences.locale = language.header
end
end
PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array)
PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc)
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
if Kemal.config.ssl || CONFIG.https_only
secure = true
else
secure = false
end
if CONFIG.domain
env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years,
secure: secure, http_only: true)
else
env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years,
secure: secure, http_only: true)
end
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
if env.request.cookies["PREFS"]?
preferences = env.get("preferences").as(Preferences)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email)
user.preferences = env.get("preferences").as(Preferences)
Invidious::Database::Users.update_preferences(user)
cookie = env.request.cookies["PREFS"]
cookie.expires = Time.utc(1990, 1, 1)
@@ -475,8 +181,8 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
end
end
def signout(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.signout(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -491,12 +197,12 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid)
Invidious::Database::SessionIDs.delete(sid: sid)
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)

33
src/invidious/routes/misc.cr
View File

@@ -1,7 +1,9 @@
class Invidious::Routes::Misc < Invidious::Routes::BaseRoute
def home(env)
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Misc
def self.home(env)
preferences = env.get("preferences").as(Preferences)
locale = LOCALES[preferences.locale]?
locale = preferences.locale
user = env.get? "user"
case preferences.default_home
@@ -17,7 +19,7 @@ class Invidious::Routes::Misc < Invidious::Routes::BaseRoute
end
when "Playlists"
if user
env.redirect "/view_all_playlists"
env.redirect "/feed/playlists"
else
env.redirect "/feed/popular"
end
@@ -26,13 +28,28 @@ class Invidious::Routes::Misc < Invidious::Routes::BaseRoute
end
end
def privacy(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.privacy(env)
locale = env.get("preferences").as(Preferences).locale
templated "privacy"
end
def licenses(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.licenses(env)
locale = env.get("preferences").as(Preferences).locale
rendered "licenses"
end
def self.cross_instance_redirect(env)
referer = get_referer(env)
instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
if instance_list.empty?
instance_url = "redirect.invidious.io"
else
# Sample returns an array
# Instances are packaged as {region, domain} in the instance list
instance_url = instance_list.sample(1)[0][1]
end
env.redirect "https://#{instance_url}#{referer}"
end
end

34
src/invidious/routes/notifications.cr Normal file
View File

@@ -0,0 +1,34 @@
module Invidious::Routes::Notifications
# /modify_notifications
# will "ding" all subscriptions.
# /modify_notifications?receive_all_updates=false&receive_no_updates=false
# will "unding" all subscriptions.
def self.modify(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
referer = get_referer(env, "/")
redirect = env.params.query["redirect"]?
redirect ||= "false"
redirect = redirect == "true"
if !user
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
end
end
user = user.as(User)
if redirect
env.redirect referer
else
env.response.content_type = "application/json"
"{}"
end
end
end

237
src/invidious/routes/playlists.cr
View File

@@ -1,31 +1,8 @@
class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
def index(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
{% skip_file if flag?(:api_only) %}
user = env.get? "user"
referer = get_referer(env)
return env.redirect "/" if user.nil?
user = user.as(User)
items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
items_created.map! do |item|
item.author = ""
item
end
items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
items_saved.map! do |item|
item.author = ""
item
end
templated "view_all_playlists"
end
def new(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
module Invidious::Routes::Playlists
def self.new(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -35,13 +12,13 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY)
templated "create_playlist"
end
def create(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.create(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -54,7 +31,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
@@ -69,17 +46,17 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
return error_template(400, "Invalid privacy setting.")
end
if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_template(400, "User cannot have more than 100 playlists.")
end
playlist = create_playlist(PG_DB, title, privacy, user)
playlist = create_playlist(title, privacy, user)
env.redirect "/playlist?list=#{playlist.id}"
end
def subscribe(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.subscribe(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
referer = get_referer(env)
@@ -89,14 +66,20 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
user = user.as(User)
playlist_id = env.params.query["list"]
playlist = get_playlist(PG_DB, playlist_id, locale)
subscribe_playlist(PG_DB, user, playlist)
begin
playlist = get_playlist(playlist_id)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}"
end
def delete_page(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.delete_page(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -108,18 +91,22 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
sid = sid.as(String)
plid = env.params.query["list"]?
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if !plid || plid.empty?
return error_template(400, "A playlist ID is required")
end
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY)
templated "delete_playlist"
end
def delete(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.delete(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -135,24 +122,23 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid)
PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid)
Invidious::Database::Playlists.delete(plid)
env.redirect "/view_all_playlists"
env.redirect "/feed/playlists"
end
def edit(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.edit(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -171,28 +157,31 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
page = env.params.query["page"]?.try &.to_i?
page ||= 1
begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
rescue ex
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
begin
videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale)
items = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
videos = [] of PlaylistVideo
items = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB)
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
# Pagination
page_nav_html = Frontend::Pagination.nav_numeric(locale,
base_url: "/playlist?list=#{playlist.id}",
current_page: page,
show_next: (items.size == 100)
)
templated "edit_playlist"
end
def update(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.update(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -208,12 +197,12 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@@ -230,13 +219,16 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
updated = playlist.updated
end
PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid)
Invidious::Database::Playlists.update(plid, title, privacy, description, updated)
env.redirect "/playlist?list=#{plid}"
end
def add_playlist_items_page(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.add_playlist_items_page(env)
prefs = env.get("preferences").as(Preferences)
locale = prefs.locale
region = env.params.query["region"]? || prefs.region
user = env.get? "user"
sid = env.get? "sid"
@@ -255,35 +247,32 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
page = env.params.query["page"]?.try &.to_i?
page ||= 1
begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
rescue ex
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email
return env.redirect referer
end
query = env.params.query["q"]?
if query
begin
search_query, count, items, operators = process_search_query(query, page, user, region: nil)
videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) }
rescue ex
videos = [] of SearchVideo
count = 0
end
else
videos = [] of SearchVideo
count = 0
begin
query = Invidious::Search::Query.new(env.params.query, :playlist, region)
items = query.process.select(SearchVideo).map(&.as(SearchVideo))
rescue ex
items = [] of SearchVideo
end
# Pagination
query_encoded = URI.encode_www_form(query.try &.text || "", space_to_plus: true)
page_nav_html = Frontend::Pagination.nav_numeric(locale,
base_url: "/add_playlist_items?list=#{playlist.id}&q=#{query_encoded}",
current_page: page,
show_next: (items.size >= 20)
)
env.set "add_playlist_items", plid
templated "add_playlist_items"
end
def playlist_ajax(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.playlist_ajax(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
sid = env.get? "sid"
@@ -306,7 +295,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
@@ -334,8 +323,10 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist)
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
if redirect
return error_template(400, ex)
@@ -344,28 +335,26 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
end
end
if !user.password
# TODO: Playlist stub, sync with YouTube for Google accounts
# playlist_ajax(playlist_id, action, env.request.headers)
end
email = user.email
case action
when "action_edit_playlist"
# TODO: Playlist stub
when "action_add_video"
if playlist.index.size >= 500
if playlist.index.size >= CONFIG.playlist_length_limit
if redirect
return error_template(400, "Playlist cannot have more than 500 videos")
return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
else
return error_json(400, "Playlist cannot have more than 500 videos")
return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
end
video_id = env.params.query["video_id"]
begin
video = get_video(video_id, PG_DB)
video = get_video(video_id)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
if redirect
return error_template(500, ex)
@@ -386,15 +375,12 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
index: Random::Secure.rand(0_i64..Int64::MAX),
})
video_array = playlist_video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array)
PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id)
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
when "action_remove_video"
index = env.params.query["set_video_id"]
PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index)
PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id)
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
when "action_move_video_before"
# TODO: Playlist stub
else
@@ -409,8 +395,8 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
end
end
def show(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.show(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get?("user").try &.as(User)
referer = get_referer(env)
@@ -428,13 +414,20 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
end
begin
playlist = get_playlist(PG_DB, plid, locale)
playlist = get_playlist(plid)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
page_count = (playlist.video_count / 100).to_i
page_count += 1 if (playlist.video_count % 100) > 0
if playlist.is_a? InvidiousPlaylist
page_count = (playlist.video_count / 100).to_i
page_count += 1 if (playlist.video_count % 100) > 0
else
page_count = (playlist.video_count / 200).to_i
page_count += 1 if (playlist.video_count % 200) > 0
end
if page > page_count
return env.redirect "/playlist?list=#{plid}&page=#{page_count}"
@@ -445,7 +438,11 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
end
begin
videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale)
if playlist.is_a? InvidiousPlaylist
items = get_playlist_videos(playlist, offset: (page - 1) * 100)
else
items = get_playlist_videos(playlist, offset: (page - 1) * 200)
end
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
end
@@ -454,11 +451,18 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
env.set "remove_playlist_items", plid
end
# Pagination
page_nav_html = Frontend::Pagination.nav_numeric(locale,
base_url: "/playlist?list=#{playlist.id}",
current_page: page,
show_next: (page_count != 1 && page < page_count)
)
templated "playlist"
end
def mix(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.mix(env)
locale = env.get("preferences").as(Preferences).locale
rdid = env.params.query["list"]?
if !rdid
@@ -476,4 +480,15 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
templated "mix"
end
# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
def self.watch_videos(env)
response = YT_POOL.client &.get(env.request.resource)
if url = response.headers["Location"]?
url = URI.parse(url).request_target
return env.redirect url
end
env.response.status_code = response.status_code
end
end

241
src/invidious/routes/preferences.cr
View File

@@ -1,16 +1,18 @@
class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
def show(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::PreferencesRoute
def self.show(env)
locale = env.get("preferences").as(Preferences).locale
referer = get_referer(env)
preferences = env.get("preferences").as(Preferences)
templated "preferences"
templated "user/preferences"
end
def update(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.update(env)
locale = env.get("preferences").as(Preferences).locale
referer = get_referer(env)
video_loop = env.params.body["video_loop"]?.try &.as(String)
@@ -25,6 +27,10 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
preload = env.params.body["preload"]?.try &.as(String)
preload ||= "off"
preload = preload == "on"
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -45,6 +51,10 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
local ||= "off"
local = local == "on"
watch_history = env.params.body["watch_history"]?.try &.as(String)
watch_history ||= "off"
watch_history = watch_history == "on"
speed = env.params.body["speed"]?.try &.as(String).to_f32?
speed ||= CONFIG.default_user_preferences.speed
@@ -60,6 +70,22 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
volume = env.params.body["volume"]?.try &.as(String).to_i?
volume ||= CONFIG.default_user_preferences.volume
extend_desc = env.params.body["extend_desc"]?.try &.as(String)
extend_desc ||= "off"
extend_desc = extend_desc == "on"
vr_mode = env.params.body["vr_mode"]?.try &.as(String)
vr_mode ||= "off"
vr_mode = vr_mode == "on"
save_player_pos = env.params.body["save_player_pos"]?.try &.as(String)
save_player_pos ||= "off"
save_player_pos = save_player_pos == "on"
show_nick = env.params.body["show_nick"]?.try &.as(String)
show_nick ||= "off"
show_nick = show_nick == "on"
comments = [] of String
2.times do |i|
comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i])
@@ -84,6 +110,12 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
end
end
automatic_instance_redirect = env.params.body["automatic_instance_redirect"]?.try &.as(String)
automatic_instance_redirect ||= "off"
automatic_instance_redirect = automatic_instance_redirect == "on"
region = env.params.body["region"]?.try &.as(String)
locale = env.params.body["locale"]?.try &.as(String)
locale ||= CONFIG.default_user_preferences.locale
@@ -112,39 +144,48 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
notifications_only ||= "off"
notifications_only = notifications_only == "on"
# Convert to JSON and back again to take advantage of converters used for compatability
# Convert to JSON and back again to take advantage of converters used for compatibility
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
autoplay: autoplay,
captions: captions,
comments: comments,
continue: continue,
continue_autoplay: continue_autoplay,
dark_mode: dark_mode,
latest_only: latest_only,
listen: listen,
local: local,
locale: locale,
max_results: max_results,
notifications_only: notifications_only,
player_style: player_style,
quality: quality,
quality_dash: quality_dash,
default_home: default_home,
feed_menu: feed_menu,
related_videos: related_videos,
sort: sort,
speed: speed,
thin_mode: thin_mode,
unseen_only: unseen_only,
video_loop: video_loop,
volume: volume,
}.to_json).to_json
annotations: annotations,
annotations_subscribed: annotations_subscribed,
preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
continue: continue,
continue_autoplay: continue_autoplay,
dark_mode: dark_mode,
latest_only: latest_only,
listen: listen,
local: local,
watch_history: watch_history,
locale: locale,
max_results: max_results,
notifications_only: notifications_only,
player_style: player_style,
quality: quality,
quality_dash: quality_dash,
default_home: default_home,
feed_menu: feed_menu,
automatic_instance_redirect: automatic_instance_redirect,
region: region,
related_videos: related_videos,
sort: sort,
speed: speed,
thin_mode: thin_mode,
unseen_only: unseen_only,
video_loop: video_loop,
volume: volume,
extend_desc: extend_desc,
vr_mode: vr_mode,
show_nick: show_nick,
save_player_pos: save_player_pos,
}.to_json)
if user = env.get? "user"
user = user.as(User)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email)
user.preferences = preferences
Invidious::Database::Users.update_preferences(user)
if CONFIG.admins.includes? user.email
CONFIG.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home
@@ -178,29 +219,19 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
File.write("config/config.yml", CONFIG.to_yaml)
end
else
if Kemal.config.ssl || CONFIG.https_only
secure = true
else
secure = false
end
if CONFIG.domain
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: preferences, expires: Time.utc + 2.years,
secure: secure, http_only: true)
else
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years,
secure: secure, http_only: true)
end
env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences)
end
env.redirect referer
end
def toggle_theme(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
def self.toggle_theme(env)
locale = env.get("preferences").as(Preferences).locale
referer = get_referer(env, unroll: false)
redirect = env.params.query["redirect"]?
@@ -209,18 +240,15 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
if user = env.get? "user"
user = user.as(User)
preferences = user.preferences
case preferences.dark_mode
case user.preferences.dark_mode
when "dark"
preferences.dark_mode = "light"
user.preferences.dark_mode = "light"
else
preferences.dark_mode = "dark"
user.preferences.dark_mode = "dark"
end
preferences = preferences.to_json
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email)
Invidious::Database::Users.update_preferences(user)
else
preferences = env.get("preferences").as(Preferences)
@@ -231,21 +259,7 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
preferences.dark_mode = "dark"
end
preferences = preferences.to_json
if Kemal.config.ssl || CONFIG.https_only
secure = true
else
secure = false
end
if CONFIG.domain
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: preferences, expires: Time.utc + 2.years,
secure: secure, http_only: true)
else
env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years,
secure: secure, http_only: true)
end
env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences)
end
if redirect
@@ -255,4 +269,87 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute
"{}"
end
end
def self.data_control(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
referer = get_referer(env)
if !user
return env.redirect referer
end
user = user.as(User)
templated "user/data_control"
end
def self.update_data_control(env)
locale = env.get("preferences").as(Preferences).locale
user = env.get? "user"
referer = get_referer(env)
if user
user = user.as(User)
# TODO: Find a way to prevent browser timeout
HTTP::FormData.parse(env.request) do |part|
body = part.body.gets_to_end
type = part.headers["Content-Type"]
next if body.empty?
# TODO: Unify into single import based on content-type
case part.name
when "import_invidious"
Invidious::User::Import.from_invidious(user, body)
when "import_youtube"
filename = part.filename || ""
success = Invidious::User::Import.from_youtube(user, body, filename, type)
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid subscription file uploaded")
)
end
when "import_youtube_pl"
filename = part.filename || ""
success = Invidious::User::Import.from_youtube_pl(user, body, filename, type)
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid playlist file uploaded")
)
end
when "import_youtube_wh"
filename = part.filename || ""
success = Invidious::User::Import.from_youtube_wh(user, body, filename, type)
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid watch history file uploaded")
)
end
when "import_freetube"
Invidious::User::Import.from_freetube(user, body)
when "import_newpipe_subscriptions"
Invidious::User::Import.from_newpipe_subs(user, body)
when "import_newpipe"
success = Invidious::User::Import.from_newpipe(user, body)
if !success
haltf(env, status_code: 415,
response: error_template(415, "Uploaded file is too large")
)
end
else nil # Ignore
end
end
end
env.redirect referer
end
end

Some files were not shown because too many files have changed in this diff Show More