From 4fa7aa65bdfa34f4aba07bcc0f9e7d6e068eca1e Mon Sep 17 00:00:00 2001 From: mooleshacat <43627985+mooleshacat@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:32:29 -0700 Subject: [PATCH 1/6] cherry picked commits for dev-token-updater --- README.md | 3 + src/invidious.cr | 4 ++ src/invidious/jobs/token_monitor.cr | 17 +++++ src/invidious/reloadpotoken.cr | 83 +++++++++++++++++++++++++ src/invidious/yt_backend/youtube_api.cr | 10 +-- 5 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 src/invidious/jobs/token_monitor.cr create mode 100644 src/invidious/reloadpotoken.cr diff --git a/README.md b/README.md index b139c5f6..5677b937 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,9 @@ ## Features +**Patches** +- token updater patch (mooleshacat) + **User features** - Lightweight - No ads diff --git a/src/invidious.cr b/src/invidious.cr index 63f2a9cc..0b420ddf 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -189,6 +189,10 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +ReloadPOToken.get_tokens #init + +Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new() + Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new Invidious::Jobs.start_all diff --git a/src/invidious/jobs/token_monitor.cr b/src/invidious/jobs/token_monitor.cr new file mode 100644 index 00000000..a94f7ed1 --- /dev/null +++ b/src/invidious/jobs/token_monitor.cr @@ -0,0 +1,17 @@ + +class Invidious::Jobs::MonitorCfgPotokensJob < Invidious::Jobs::BaseJob + include Invidious + def begin + loop do + + LOGGER.info("jobs: running MonitorCfgPotokens job") + + ReloadPOToken.get_tokens + + LOGGER.info("jobs: MonitorCfgPotokens: pot: " + ReloadPOToken.pot.as(String)) + LOGGER.info("jobs: MonitorCfgPotokens: vdata: " + ReloadPOToken.vdata.as(String)) + + sleep 15.seconds + end + end +end diff --git a/src/invidious/reloadpotoken.cr b/src/invidious/reloadpotoken.cr new file mode 100644 index 00000000..2b874ad8 --- /dev/null +++ b/src/invidious/reloadpotoken.cr @@ -0,0 +1,83 @@ +class ReloadPOToken + + @@instance = new + + def self.pot + @@pot + end + + def self.vdata + @@vdata + end + + def initialize + + @@pot = "error" + @@vdata = "error" + + end + + def self.get_tokens + + # Load config from file or YAML string env var + env_config_file = "INVIDIOUS_CONFIG_FILE" + env_config_yaml = "INVIDIOUS_CONFIG" + + config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" + config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) + + config = Config.from_yaml(config_yaml) + + + + + # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + {% for ivar in Config.instance_vars %} + {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} + + if ENV.has_key?({{env_id}}) + env_value = ENV.fetch({{env_id}}) + success = false + + # Use YAML converter if specified + {% ann = ivar.annotation(::YAML::Field) %} + {% if ann && ann[:converter] %} + config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) + success = true + + # Use regular YAML parser otherwise + {% else %} + {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} + # Sort types to avoid parsing nulls and numbers as strings + {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} + {{ivar_types}}.each do |ivar_type| + if !success + begin + config.{{ivar.id}} = ivar_type.from_yaml(env_value) + success = true + rescue + # nop + end + end + end + {% end %} + + # Exit on fail + if !success + puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) + exit(1) + end + end + {% end %} + + + @@pot = config.po_token + @@vdata = config.visitor_data + + end + + def self.get_instance + return @@instance + end + +end \ No newline at end of file diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index baa3cd92..be817d78 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -330,8 +330,8 @@ module YoutubeAPI client_context["client"]["platform"] = platform end - if CONFIG.visitor_data.is_a?(String) - client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String) + if ReloadPOToken.vdata.is_a?(String) + client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String) end return client_context @@ -492,7 +492,7 @@ module YoutubeAPI "contentPlaybackContext" => playback_ctx, }, "serviceIntegrityDimensions" => { - "poToken" => CONFIG.po_token, + "poToken" => ReloadPOToken.pot.as(String), }, } @@ -626,8 +626,8 @@ module YoutubeAPI headers["User-Agent"] = user_agent end - if CONFIG.visitor_data.is_a?(String) - headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String) + if ReloadPOToken.vdata.is_a?(String) + headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String) end # Logging From 61d9491ee60e356f7d826bd7e70e18cf6ba1472c Mon Sep 17 00:00:00 2001 From: mooleshacat <43627985+mooleshacat@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:52:25 -0400 Subject: [PATCH 2/6] few fixes --- src/invidious.cr | 4 +- src/invidious.cr~ | 248 +++++++ src/invidious/jobs/token_monitor.cr | 8 +- src/invidious/jobs/token_monitor.cr~ | 17 + .../{reloadpotoken.cr => reloadpotoken.cr~} | 0 src/invidious/reloadtoken.cr | 83 +++ src/invidious/yt_backend/youtube_api.cr | 10 +- src/invidious/yt_backend/youtube_api.cr~ | 685 ++++++++++++++++++ 8 files changed, 1043 insertions(+), 12 deletions(-) create mode 100644 src/invidious.cr~ create mode 100644 src/invidious/jobs/token_monitor.cr~ rename src/invidious/{reloadpotoken.cr => reloadpotoken.cr~} (100%) create mode 100644 src/invidious/reloadtoken.cr create mode 100644 src/invidious/yt_backend/youtube_api.cr~ diff --git a/src/invidious.cr b/src/invidious.cr index 0b420ddf..b2387158 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -189,9 +189,7 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new -ReloadPOToken.get_tokens #init - -Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new() +Invidious::Jobs.register Invidious::Jobs::MonitorCfgTokensJob.new() Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new diff --git a/src/invidious.cr~ b/src/invidious.cr~ new file mode 100644 index 00000000..0b420ddf --- /dev/null +++ b/src/invidious.cr~ @@ -0,0 +1,248 @@ +# "Invidious" (which is an alternative front-end to YouTube) +# Copyright (C) 2019 Omar Roth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +require "digest/md5" +require "file_utils" + +# Require kemal, kilt, then our own overrides +require "kemal" +require "kilt" +require "./ext/kemal_content_for.cr" +require "./ext/kemal_static_file_handler.cr" + +require "athena-negotiation" +require "openssl/hmac" +require "option_parser" +require "sqlite3" +require "xml" +require "yaml" +require "compress/zip" +require "protodec/utils" + +require "./invidious/database/*" +require "./invidious/database/migrations/*" +require "./invidious/http_server/*" +require "./invidious/helpers/*" +require "./invidious/yt_backend/*" +require "./invidious/frontend/*" +require "./invidious/videos/*" + +require "./invidious/jsonify/**" + +require "./invidious/*" +require "./invidious/comments/*" +require "./invidious/channels/*" +require "./invidious/user/*" +require "./invidious/search/*" +require "./invidious/routes/**" +require "./invidious/jobs/**" + +# Declare the base namespace for invidious +module Invidious +end + +# Simple alias to make code easier to read +alias IV = Invidious + +CONFIG = Config.load +HMAC_KEY = CONFIG.hmac_key + +PG_DB = DB.open CONFIG.database_url +ARCHIVE_URL = URI.parse("https://archive.org") +PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") +REDDIT_URL = URI.parse("https://www.reddit.com") +YT_URL = URI.parse("https://www.youtube.com") +HOST_URL = make_host_url(Kemal.config) + +CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" +TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} +MAX_ITEMS_PER_PAGE = 1500 + +REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"} +RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"} +HTTP_CHUNK_SIZE = 10485760 # ~10MB + +CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} +CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} +CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} + +# This is used to determine the `?v=` on the end of file URLs (for cache busting). We +# only need to expire modified assets, so we can use this to find the last commit that changes +# any assets +ASSET_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }} + +SOFTWARE = { + "name" => "invidious", + "version" => "#{CURRENT_VERSION}-#{CURRENT_COMMIT}", + "branch" => "#{CURRENT_BRANCH}", +} + +YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) + +# CLI +Kemal.config.extra_options do |parser| + parser.banner = "Usage: invidious [arguments]" + parser.on("-c THREADS", "--channel-threads=THREADS", "Number of threads for refreshing channels (default: #{CONFIG.channel_threads})") do |number| + begin + CONFIG.channel_threads = number.to_i + rescue ex + puts "THREADS must be integer" + exit + end + end + parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number| + begin + CONFIG.feed_threads = number.to_i + rescue ex + puts "THREADS must be integer" + exit + end + end + parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output| + CONFIG.output = output + end + parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| + CONFIG.log_level = LogLevel.parse(log_level) + end + parser.on("-v", "--version", "Print version") do + puts SOFTWARE.to_pretty_json + exit + end + parser.on("--migrate", "Run any migrations (beta, use at your own risk!!") do + Invidious::Database::Migrator.new(PG_DB).migrate + exit + end +end + +Kemal::CLI.new ARGV + +if CONFIG.output.upcase != "STDOUT" + FileUtils.mkdir_p(File.dirname(CONFIG.output)) +end +OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") +LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) + +# Check table integrity +Invidious::Database.check_integrity(CONFIG) + +{% if !flag?(:skip_videojs_download) %} + # Resolve player dependencies. This is done at compile time. + # + # Running the script by itself would show some colorful feedback while this doesn't. + # Perhaps we should just move the script to runtime in order to get that feedback? + + {% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %} + {% if flag?(:minified_player_dependencies) %} + {% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %} + {% else %} + {% puts run("../scripts/fetch-player-dependencies.cr").stringify %} + {% end %} + {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} +{% end %} + +# Misc + +DECRYPT_FUNCTION = + if sig_helper_address = CONFIG.signature_server.presence + IV::DecryptFunction.new(sig_helper_address) + else + nil + end + +# Start jobs + +if CONFIG.channel_threads > 0 + Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB) +end + +if CONFIG.feed_threads > 0 + Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) +end + +if CONFIG.statistics_enabled + Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) +end + +if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0) + Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, HMAC_KEY) +end + +if CONFIG.popular_enabled + Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) +end + +CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) + +Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new + +ReloadPOToken.get_tokens #init + +Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new() + +Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new + +Invidious::Jobs.start_all + +def popular_videos + Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get +end + +# Routing + +before_all do |env| + Invidious::Routes::BeforeAll.handle(env) +end + +Invidious::Routing.register_all + +error 404 do |env| + Invidious::Routes::ErrorRoutes.error_404(env) +end + +error 500 do |env, ex| + error_template(500, ex) +end + +static_headers do |response| + response.headers.add("Cache-Control", "max-age=2629800") +end + +# Init Kemal + +public_folder "assets" + +Kemal.config.powered_by_header = false +add_handler FilteredCompressHandler.new +add_handler APIHandler.new +add_handler AuthHandler.new +add_handler DenyFrame.new +add_context_storage_type(Array(String)) +add_context_storage_type(Preferences) +add_context_storage_type(Invidious::User) + +Kemal.config.logger = LOGGER +Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding +Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port +Kemal.config.app_name = "Invidious" + +# Use in kemal's production mode. +# Users can also set the KEMAL_ENV environmental variable for this to be set automatically. +{% if flag?(:release) || flag?(:production) %} + Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV") +{% end %} + +Kemal.run diff --git a/src/invidious/jobs/token_monitor.cr b/src/invidious/jobs/token_monitor.cr index a94f7ed1..e863acfe 100644 --- a/src/invidious/jobs/token_monitor.cr +++ b/src/invidious/jobs/token_monitor.cr @@ -1,15 +1,15 @@ -class Invidious::Jobs::MonitorCfgPotokensJob < Invidious::Jobs::BaseJob +class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob include Invidious def begin loop do - LOGGER.info("jobs: running MonitorCfgPotokens job") + LOGGER.info("jobs: running MonitorCfgTokensJob job") ReloadPOToken.get_tokens - LOGGER.info("jobs: MonitorCfgPotokens: pot: " + ReloadPOToken.pot.as(String)) - LOGGER.info("jobs: MonitorCfgPotokens: vdata: " + ReloadPOToken.vdata.as(String)) + LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadTokens.pot.as(String)) + LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadTokens.vdata.as(String)) sleep 15.seconds end diff --git a/src/invidious/jobs/token_monitor.cr~ b/src/invidious/jobs/token_monitor.cr~ new file mode 100644 index 00000000..d923dd7d --- /dev/null +++ b/src/invidious/jobs/token_monitor.cr~ @@ -0,0 +1,17 @@ + +class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob + include Invidious + def begin + loop do + + LOGGER.info("jobs: running MonitorCfgTokensJob job") + + ReloadPOToken.get_tokens + + LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadPOToken.pot.as(String)) + LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadPOToken.vdata.as(String)) + + sleep 15.seconds + end + end +end diff --git a/src/invidious/reloadpotoken.cr b/src/invidious/reloadpotoken.cr~ similarity index 100% rename from src/invidious/reloadpotoken.cr rename to src/invidious/reloadpotoken.cr~ diff --git a/src/invidious/reloadtoken.cr b/src/invidious/reloadtoken.cr new file mode 100644 index 00000000..8ff56f48 --- /dev/null +++ b/src/invidious/reloadtoken.cr @@ -0,0 +1,83 @@ +class ReloadToken + + @@instance = new + + def self.pot + @@pot + end + + def self.vdata + @@vdata + end + + def initialize + + @@pot = "error" + @@vdata = "error" + + end + + def self.get_tokens + + # Load config from file or YAML string env var + env_config_file = "INVIDIOUS_CONFIG_FILE" + env_config_yaml = "INVIDIOUS_CONFIG" + + config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" + config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) + + config = Config.from_yaml(config_yaml) + + + + + # Update config from env vars (upcased and prefixed with "INVIDIOUS_") + {% for ivar in Config.instance_vars %} + {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} + + if ENV.has_key?({{env_id}}) + env_value = ENV.fetch({{env_id}}) + success = false + + # Use YAML converter if specified + {% ann = ivar.annotation(::YAML::Field) %} + {% if ann && ann[:converter] %} + config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) + success = true + + # Use regular YAML parser otherwise + {% else %} + {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} + # Sort types to avoid parsing nulls and numbers as strings + {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} + {{ivar_types}}.each do |ivar_type| + if !success + begin + config.{{ivar.id}} = ivar_type.from_yaml(env_value) + success = true + rescue + # nop + end + end + end + {% end %} + + # Exit on fail + if !success + puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) + exit(1) + end + end + {% end %} + + + @@pot = config.po_token + @@vdata = config.visitor_data + + end + + def self.get_instance + return @@instance + end + +end \ No newline at end of file diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b717a67e..7862fd80 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -320,8 +320,8 @@ module YoutubeAPI client_context["client"]["platform"] = platform end - if ReloadPOToken.vdata.is_a?(String) - client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String) + if ReloadToken.vdata.is_a?(String) + client_context["client"]["visitorData"] = ReloadToken.vdata.as(String) end return client_context @@ -482,7 +482,7 @@ module YoutubeAPI "contentPlaybackContext" => playback_ctx, }, "serviceIntegrityDimensions" => { - "poToken" => ReloadPOToken.pot.as(String), + "poToken" => ReloadToken.pot.as(String), }, } @@ -616,8 +616,8 @@ module YoutubeAPI headers["User-Agent"] = user_agent end - if ReloadPOToken.vdata.is_a?(String) - headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String) + if ReloadToken.vdata.is_a?(String) + headers["X-Goog-Visitor-Id"] = ReloadToken.vdata.as(String) end # Logging diff --git a/src/invidious/yt_backend/youtube_api.cr~ b/src/invidious/yt_backend/youtube_api.cr~ new file mode 100644 index 00000000..b717a67e --- /dev/null +++ b/src/invidious/yt_backend/youtube_api.cr~ @@ -0,0 +1,685 @@ +# +# This file contains youtube API wrappers +# + +module YoutubeAPI + extend self + + # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history + private ANDROID_APP_VERSION = "19.32.34" + private ANDROID_VERSION = "12" + private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip" + private ANDROID_SDK_VERSION = 31_i64 + + private ANDROID_TS_APP_VERSION = "1.9" + private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" + + # For Apple device names, see https://gist.github.com/adamawolf/3048717 + # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, + # then go to the dedicated article of the major version you want. + private IOS_APP_VERSION = "19.32.8" + private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)" + private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build + + private WINDOWS_VERSION = "10.0" + + # Enumerate used to select one of the clients supported by the API + enum ClientType + Web + WebEmbeddedPlayer + WebMobile + WebScreenEmbed + + Android + AndroidEmbeddedPlayer + AndroidScreenEmbed + AndroidTestSuite + + IOS + IOSEmbedded + IOSMusic + + TvHtml5 + TvHtml5ScreenEmbed + end + + # List of hard-coded values used by the different clients + HARDCODED_CLIENTS = { + ClientType::Web => { + name: "WEB", + name_proto: "1", + version: "2.20240814.00.00", + screen: "WATCH_FULL_SCREEN", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", + }, + ClientType::WebEmbeddedPlayer => { + name: "WEB_EMBEDDED_PLAYER", + name_proto: "56", + version: "1.20240812.01.00", + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", + }, + ClientType::WebMobile => { + name: "MWEB", + name_proto: "2", + version: "2.20240813.02.00", + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", + }, + ClientType::WebScreenEmbed => { + name: "WEB", + name_proto: "1", + version: "2.20240814.00.00", + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", + }, + + # Android + + ClientType::Android => { + name: "ANDROID", + name_proto: "3", + version: ANDROID_APP_VERSION, + android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", + }, + ClientType::AndroidEmbeddedPlayer => { + name: "ANDROID_EMBEDDED_PLAYER", + name_proto: "55", + version: ANDROID_APP_VERSION, + }, + ClientType::AndroidScreenEmbed => { + name: "ANDROID", + name_proto: "3", + version: ANDROID_APP_VERSION, + screen: "EMBED", + android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", + }, + ClientType::AndroidTestSuite => { + name: "ANDROID_TESTSUITE", + name_proto: "30", + version: ANDROID_TS_APP_VERSION, + android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_TS_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", + }, + + # IOS + + ClientType::IOS => { + name: "IOS", + name_proto: "5", + version: IOS_APP_VERSION, + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", + }, + ClientType::IOSEmbedded => { + name: "IOS_MESSAGES_EXTENSION", + name_proto: "66", + version: IOS_APP_VERSION, + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", + }, + ClientType::IOSMusic => { + name: "IOS_MUSIC", + name_proto: "26", + version: "7.14", + user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)", + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", + }, + + # TV app + + ClientType::TvHtml5 => { + name: "TVHTML5", + name_proto: "7", + version: "7.20240813.07.00", + }, + ClientType::TvHtml5ScreenEmbed => { + name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", + name_proto: "85", + version: "2.0", + screen: "EMBED", + }, + } + + #################################################################### + # struct ClientConfig + # + # Data structure used to pass a client configuration to the different + # API endpoints handlers. + # + # Use case examples: + # + # ``` + # # Get Norwegian search results + # conf_1 = ClientConfig.new(region: "NO") + # YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1) + # + # # Use the Android client to request video streams URLs + # conf_2 = ClientConfig.new(client_type: ClientType::Android) + # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2) + # + # + struct ClientConfig + # Type of client to emulate. + # See `enum ClientType` and `HARDCODED_CLIENTS`. + property client_type : ClientType + + # Region to provide to youtube, e.g to alter search results + # (this is passed as the `gl` parameter). + property region : String | Nil + + # Initialization function + def initialize( + *, + @client_type = ClientType::Web, + @region = "US" + ) + end + + # Getter functions that provides easy access to hardcoded clients + # parameters (name/version strings and related API key) + def name : String + HARDCODED_CLIENTS[@client_type][:name] + end + + def name_proto : String + HARDCODED_CLIENTS[@client_type][:name_proto] + end + + # :ditto: + def version : String + HARDCODED_CLIENTS[@client_type][:version] + end + + # :ditto: + def screen : String + HARDCODED_CLIENTS[@client_type][:screen]? || "" + end + + def android_sdk_version : Int64? + HARDCODED_CLIENTS[@client_type][:android_sdk_version]? + end + + def user_agent : String? + HARDCODED_CLIENTS[@client_type][:user_agent]? + end + + def os_name : String? + HARDCODED_CLIENTS[@client_type][:os_name]? + end + + def device_make : String? + HARDCODED_CLIENTS[@client_type][:device_make]? + end + + def device_model : String? + HARDCODED_CLIENTS[@client_type][:device_model]? + end + + def os_version : String? + HARDCODED_CLIENTS[@client_type][:os_version]? + end + + def platform : String? + HARDCODED_CLIENTS[@client_type][:platform]? + end + + # Convert to string, for logging purposes + def to_s + return { + client_type: self.name, + region: @region, + }.to_s + end + end + + # Default client config, used if nothing is passed + DEFAULT_CLIENT_CONFIG = ClientConfig.new + + #################################################################### + # make_context(client_config) + # + # Return, as a Hash, the "context" data required to request the + # youtube API endpoints. + # + private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash + # Use the default client config if nil is passed + client_config ||= DEFAULT_CLIENT_CONFIG + + client_context = { + "client" => { + "hl" => "en", + "gl" => client_config.region || "US", # Can't be empty! + "clientName" => client_config.name, + "clientVersion" => client_config.version, + } of String => String | Int64, + } + + # Add some more context if it exists in the client definitions + if !client_config.screen.empty? + client_context["client"]["clientScreen"] = client_config.screen + end + + if client_config.screen == "EMBED" + client_context["thirdParty"] = { + "embedUrl" => "https://www.youtube.com/embed/#{video_id}", + } of String => String | Int64 + end + + if android_sdk_version = client_config.android_sdk_version + client_context["client"]["androidSdkVersion"] = android_sdk_version + end + + if device_make = client_config.device_make + client_context["client"]["deviceMake"] = device_make + end + + if device_model = client_config.device_model + client_context["client"]["deviceModel"] = device_model + end + + if os_name = client_config.os_name + client_context["client"]["osName"] = os_name + end + + if os_version = client_config.os_version + client_context["client"]["osVersion"] = os_version + end + + if platform = client_config.platform + client_context["client"]["platform"] = platform + end + + if ReloadPOToken.vdata.is_a?(String) + client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String) + end + + return client_context + end + + #################################################################### + # browse(continuation, client_config?) + # browse(browse_id, params, client_config?) + # + # Requests the youtubei/v1/browse endpoint with the required headers + # and POST data in order to get a JSON reply in english that can + # be easily parsed. + # + # Both forms can take an optional ClientConfig parameter (see + # `struct ClientConfig` above for more details). + # + # The requested data can either be: + # + # - A continuation token (ctoken). Depending on this token's + # contents, the returned data can be playlist videos, channel + # community tab content, channel info, ... + # + # - A playlist ID (parameters MUST be an empty string) + # + def browse(continuation : String, client_config : ClientConfig | Nil = nil) + # JSON Request data, required by the API + data = { + "context" => self.make_context(client_config), + "continuation" => continuation, + } + + return self._post_json("/youtubei/v1/browse", data, client_config) + end + + # :ditto: + def browse( + browse_id : String, + *, # Force the following parameters to be passed by name + params : String, + client_config : ClientConfig | Nil = nil + ) + # JSON Request data, required by the API + data = { + "browseId" => browse_id, + "context" => self.make_context(client_config), + } + + # Append the additional parameters if those were provided + # (this is required for channel info, playlist and community, e.g) + if params != "" + data["params"] = params + end + + return self._post_json("/youtubei/v1/browse", data, client_config) + end + + #################################################################### + # next(continuation, client_config?) + # next(data, client_config?) + # + # Requests the youtubei/v1/next endpoint with the required headers + # and POST data in order to get a JSON reply in english that can + # be easily parsed. + # + # Both forms can take an optional ClientConfig parameter (see + # `struct ClientConfig` above for more details). + # + # The requested data can be: + # + # - A continuation token (ctoken). Depending on this token's + # contents, the returned data can be videos comments, + # their replies, ... In this case, the string must be passed + # directly to the function. E.g: + # + # ``` + # YoutubeAPI::next("ABCDEFGH_abcdefgh==") + # ``` + # + # - Arbitrary parameters, in Hash form. See examples below for + # known examples of arbitrary data that can be passed to YouTube: + # + # ``` + # # Get the videos related to a specific video ID + # YoutubeAPI::next({"videoId" => "dQw4w9WgXcQ"}) + # + # # Get a playlist video's details + # YoutubeAPI::next({ + # "videoId" => "9bZkp7q19f0", + # "playlistId" => "PL_oFlvgqkrjUVQwiiE3F3k3voF4tjXeP0", + # }) + # ``` + # + def next(continuation : String, *, client_config : ClientConfig | Nil = nil) + # JSON Request data, required by the API + data = { + "context" => self.make_context(client_config), + "continuation" => continuation, + } + + return self._post_json("/youtubei/v1/next", data, client_config) + end + + # :ditto: + def next(data : Hash, *, client_config : ClientConfig | Nil = nil) + # JSON Request data, required by the API + data2 = data.merge({ + "context" => self.make_context(client_config), + }) + + return self._post_json("/youtubei/v1/next", data2, client_config) + end + + # Allow a NamedTuple to be passed, too. + def next(data : NamedTuple, *, client_config : ClientConfig | Nil = nil) + return self.next(data.to_h, client_config: client_config) + end + + #################################################################### + # player(video_id, params, client_config?) + # + # Requests the youtubei/v1/player endpoint with the required headers + # and POST data in order to get a JSON reply. + # + # The requested data is a video ID (`v=` parameter), with some + # additional parameters, formatted as a base64 string. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + def player( + video_id : String, + *, # Force the following parameters to be passed by name + params : String, + client_config : ClientConfig | Nil = nil + ) + # Playback context, separate because it can be different between clients + playback_ctx = { + "html5Preference" => "HTML5_PREF_WANTS", + "referer" => "https://www.youtube.com/watch?v=#{video_id}", + } of String => String | Int64 + + if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s } + if sts = DECRYPT_FUNCTION.try &.get_sts + playback_ctx["signatureTimestamp"] = sts.to_i64 + end + end + + # JSON Request data, required by the API + data = { + "contentCheckOk" => true, + "videoId" => video_id, + "context" => self.make_context(client_config, video_id), + "racyCheckOk" => true, + "user" => { + "lockedSafetyMode" => false, + }, + "playbackContext" => { + "contentPlaybackContext" => playback_ctx, + }, + "serviceIntegrityDimensions" => { + "poToken" => ReloadPOToken.pot.as(String), + }, + } + + # Append the additional parameters if those were provided + if params != "" + data["params"] = params + end + + return self._post_json("/youtubei/v1/player", data, client_config) + end + + #################################################################### + # resolve_url(url, client_config?) + # + # Requests the youtubei/v1/navigation/resolve_url endpoint with the + # required headers and POST data in order to get a JSON reply. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + # Output: + # + # ``` + # # Valid channel "brand URL" gives the related UCID and browse ID + # channel_a = YoutubeAPI.resolve_url("https://youtube.com/c/google") + # channel_a # => { + # "endpoint": { + # "browseEndpoint": { + # "params": "EgC4AQA%3D", + # "browseId":"UCK8sQmJBp8GCxrOtXWBpyEA" + # }, + # ... + # } + # } + # + # # Invalid URL returns throws an InfoException + # channel_b = YoutubeAPI.resolve_url("https://youtube.com/c/invalid") + # ``` + # + def resolve_url(url : String, client_config : ClientConfig | Nil = nil) + data = { + "context" => self.make_context(nil), + "url" => url, + } + + return self._post_json("/youtubei/v1/navigation/resolve_url", data, client_config) + end + + #################################################################### + # search(search_query, params, client_config?) + # + # Requests the youtubei/v1/search endpoint with the required headers + # and POST data in order to get a JSON reply. As the search results + # vary depending on the region, a region code can be specified in + # order to get non-US results. + # + # The requested data is a search string, with some additional + # parameters, formatted as a base64 string. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + def search( + search_query : String, + params : String, + client_config : ClientConfig | Nil = nil + ) + # JSON Request data, required by the API + data = { + "query" => search_query, + "context" => self.make_context(client_config), + "params" => params, + } + + return self._post_json("/youtubei/v1/search", data, client_config) + end + + #################################################################### + # get_transcript(params, client_config?) + # + # Requests the youtubei/v1/get_transcript endpoint with the required headers + # and POST data in order to get a JSON reply. + # + # The requested data is a specially encoded protobuf string that denotes the specific language requested. + # + # An optional ClientConfig parameter can be passed, too (see + # `struct ClientConfig` above for more details). + # + + def get_transcript( + params : String, + client_config : ClientConfig | Nil = nil + ) : Hash(String, JSON::Any) + data = { + "context" => self.make_context(client_config), + "params" => params, + } + + return self._post_json("/youtubei/v1/get_transcript", data, client_config) + end + + #################################################################### + # _post_json(endpoint, data, client_config?) + # + # Internal function that does the actual request to youtube servers + # and handles errors. + # + # The requested data is an endpoint (URL without the domain part) + # and the data as a Hash object. + # + def _post_json( + endpoint : String, + data : Hash, + client_config : ClientConfig | Nil + ) : Hash(String, JSON::Any) + # Use the default client config if nil is passed + client_config ||= DEFAULT_CLIENT_CONFIG + + # Query parameters + url = "#{endpoint}?prettyPrint=false" + + headers = HTTP::Headers{ + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", + "x-goog-api-format-version" => "2", + "x-youtube-client-name" => client_config.name_proto, + "x-youtube-client-version" => client_config.version, + } + + if user_agent = client_config.user_agent + headers["User-Agent"] = user_agent + end + + if ReloadPOToken.vdata.is_a?(String) + headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String) + end + + # Logging + LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") + LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") + LOGGER.trace("YoutubeAPI: POST data: #{data}") + + # Send the POST request + body = YT_POOL.client() do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + self._decompress(response.body_io, response.headers["Content-Encoding"]?) + end + end + + # Convert result to Hash + initial_data = JSON.parse(body).as_h + + # Error handling + if initial_data.has_key?("error") + code = initial_data["error"]["code"] + message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "") + + # Logging + LOGGER.error("YoutubeAPI: Got error #{code} when requesting #{endpoint}") + LOGGER.error("YoutubeAPI: #{message}") + LOGGER.info("YoutubeAPI: POST data was: #{data}") + + raise InfoException.new("Could not extract JSON. Youtube API returned \ + error #{code} with message:
\"#{message}\"") + end + + return initial_data + end + + #################################################################### + # _decompress(body_io, headers) + # + # Internal function that reads the Content-Encoding headers and + # decompresses the content accordingly. + # + # We decompress the body ourselves (when using HTTP::Client) because + # the auto-decompress feature is broken in the Crystal stdlib. + # + # Read more: + # - https://github.com/iv-org/invidious/issues/2612 + # - https://github.com/crystal-lang/crystal/issues/11354 + # + def _decompress(body_io : IO, encodings : String?) : String + if encodings + # Multiple encodings can be combined, and are listed in the order + # in which they were applied. E.g: "deflate, gzip" means that the + # content must be first "gunzipped", then "defated". + encodings.split(',').reverse.each do |enc| + case enc.strip(' ') + when "gzip" + body_io = Compress::Gzip::Reader.new(body_io, sync_close: true) + when "deflate" + body_io = Compress::Deflate::Reader.new(body_io, sync_close: true) + end + end + end + + return body_io.gets_to_end + end +end # End of module From 965ac6b09d7b3c5d159153b95fa55934ed8b1058 Mon Sep 17 00:00:00 2001 From: mooleshacat <43627985+mooleshacat@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:53:55 -0400 Subject: [PATCH 3/6] few fixes --- .gitignore | 1 + src/invidious.cr~ | 248 -------- src/invidious/jobs/token_monitor.cr~ | 17 - src/invidious/reloadpotoken.cr~ | 83 --- src/invidious/yt_backend/youtube_api.cr~ | 685 ----------------------- 5 files changed, 1 insertion(+), 1033 deletions(-) delete mode 100644 src/invidious.cr~ delete mode 100644 src/invidious/jobs/token_monitor.cr~ delete mode 100644 src/invidious/reloadpotoken.cr~ delete mode 100644 src/invidious/yt_backend/youtube_api.cr~ diff --git a/.gitignore b/.gitignore index 7a26e1a6..3dd4ab41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*~ /docs/ /dev/ /lib/ diff --git a/src/invidious.cr~ b/src/invidious.cr~ deleted file mode 100644 index 0b420ddf..00000000 --- a/src/invidious.cr~ +++ /dev/null @@ -1,248 +0,0 @@ -# "Invidious" (which is an alternative front-end to YouTube) -# Copyright (C) 2019 Omar Roth -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -require "digest/md5" -require "file_utils" - -# Require kemal, kilt, then our own overrides -require "kemal" -require "kilt" -require "./ext/kemal_content_for.cr" -require "./ext/kemal_static_file_handler.cr" - -require "athena-negotiation" -require "openssl/hmac" -require "option_parser" -require "sqlite3" -require "xml" -require "yaml" -require "compress/zip" -require "protodec/utils" - -require "./invidious/database/*" -require "./invidious/database/migrations/*" -require "./invidious/http_server/*" -require "./invidious/helpers/*" -require "./invidious/yt_backend/*" -require "./invidious/frontend/*" -require "./invidious/videos/*" - -require "./invidious/jsonify/**" - -require "./invidious/*" -require "./invidious/comments/*" -require "./invidious/channels/*" -require "./invidious/user/*" -require "./invidious/search/*" -require "./invidious/routes/**" -require "./invidious/jobs/**" - -# Declare the base namespace for invidious -module Invidious -end - -# Simple alias to make code easier to read -alias IV = Invidious - -CONFIG = Config.load -HMAC_KEY = CONFIG.hmac_key - -PG_DB = DB.open CONFIG.database_url -ARCHIVE_URL = URI.parse("https://archive.org") -PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -YT_URL = URI.parse("https://www.youtube.com") -HOST_URL = make_host_url(Kemal.config) - -CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" -TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} -MAX_ITEMS_PER_PAGE = 1500 - -REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"} -RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"} -HTTP_CHUNK_SIZE = 10485760 # ~10MB - -CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} -CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} -CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} - -# This is used to determine the `?v=` on the end of file URLs (for cache busting). We -# only need to expire modified assets, so we can use this to find the last commit that changes -# any assets -ASSET_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }} - -SOFTWARE = { - "name" => "invidious", - "version" => "#{CURRENT_VERSION}-#{CURRENT_COMMIT}", - "branch" => "#{CURRENT_BRANCH}", -} - -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) - -# CLI -Kemal.config.extra_options do |parser| - parser.banner = "Usage: invidious [arguments]" - parser.on("-c THREADS", "--channel-threads=THREADS", "Number of threads for refreshing channels (default: #{CONFIG.channel_threads})") do |number| - begin - CONFIG.channel_threads = number.to_i - rescue ex - puts "THREADS must be integer" - exit - end - end - parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number| - begin - CONFIG.feed_threads = number.to_i - rescue ex - puts "THREADS must be integer" - exit - end - end - parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output| - CONFIG.output = output - end - parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| - CONFIG.log_level = LogLevel.parse(log_level) - end - parser.on("-v", "--version", "Print version") do - puts SOFTWARE.to_pretty_json - exit - end - parser.on("--migrate", "Run any migrations (beta, use at your own risk!!") do - Invidious::Database::Migrator.new(PG_DB).migrate - exit - end -end - -Kemal::CLI.new ARGV - -if CONFIG.output.upcase != "STDOUT" - FileUtils.mkdir_p(File.dirname(CONFIG.output)) -end -OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") -LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) - -# Check table integrity -Invidious::Database.check_integrity(CONFIG) - -{% if !flag?(:skip_videojs_download) %} - # Resolve player dependencies. This is done at compile time. - # - # Running the script by itself would show some colorful feedback while this doesn't. - # Perhaps we should just move the script to runtime in order to get that feedback? - - {% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %} - {% if flag?(:minified_player_dependencies) %} - {% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %} - {% else %} - {% puts run("../scripts/fetch-player-dependencies.cr").stringify %} - {% end %} - {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} -{% end %} - -# Misc - -DECRYPT_FUNCTION = - if sig_helper_address = CONFIG.signature_server.presence - IV::DecryptFunction.new(sig_helper_address) - else - nil - end - -# Start jobs - -if CONFIG.channel_threads > 0 - Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB) -end - -if CONFIG.feed_threads > 0 - Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) -end - -if CONFIG.statistics_enabled - Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) -end - -if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0) - Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, HMAC_KEY) -end - -if CONFIG.popular_enabled - Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) -end - -CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) - -Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new - -ReloadPOToken.get_tokens #init - -Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new() - -Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new - -Invidious::Jobs.start_all - -def popular_videos - Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get -end - -# Routing - -before_all do |env| - Invidious::Routes::BeforeAll.handle(env) -end - -Invidious::Routing.register_all - -error 404 do |env| - Invidious::Routes::ErrorRoutes.error_404(env) -end - -error 500 do |env, ex| - error_template(500, ex) -end - -static_headers do |response| - response.headers.add("Cache-Control", "max-age=2629800") -end - -# Init Kemal - -public_folder "assets" - -Kemal.config.powered_by_header = false -add_handler FilteredCompressHandler.new -add_handler APIHandler.new -add_handler AuthHandler.new -add_handler DenyFrame.new -add_context_storage_type(Array(String)) -add_context_storage_type(Preferences) -add_context_storage_type(Invidious::User) - -Kemal.config.logger = LOGGER -Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding -Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port -Kemal.config.app_name = "Invidious" - -# Use in kemal's production mode. -# Users can also set the KEMAL_ENV environmental variable for this to be set automatically. -{% if flag?(:release) || flag?(:production) %} - Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV") -{% end %} - -Kemal.run diff --git a/src/invidious/jobs/token_monitor.cr~ b/src/invidious/jobs/token_monitor.cr~ deleted file mode 100644 index d923dd7d..00000000 --- a/src/invidious/jobs/token_monitor.cr~ +++ /dev/null @@ -1,17 +0,0 @@ - -class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob - include Invidious - def begin - loop do - - LOGGER.info("jobs: running MonitorCfgTokensJob job") - - ReloadPOToken.get_tokens - - LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadPOToken.pot.as(String)) - LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadPOToken.vdata.as(String)) - - sleep 15.seconds - end - end -end diff --git a/src/invidious/reloadpotoken.cr~ b/src/invidious/reloadpotoken.cr~ deleted file mode 100644 index 2b874ad8..00000000 --- a/src/invidious/reloadpotoken.cr~ +++ /dev/null @@ -1,83 +0,0 @@ -class ReloadPOToken - - @@instance = new - - def self.pot - @@pot - end - - def self.vdata - @@vdata - end - - def initialize - - @@pot = "error" - @@vdata = "error" - - end - - def self.get_tokens - - # Load config from file or YAML string env var - env_config_file = "INVIDIOUS_CONFIG_FILE" - env_config_yaml = "INVIDIOUS_CONFIG" - - config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml" - config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file) - - config = Config.from_yaml(config_yaml) - - - - - # Update config from env vars (upcased and prefixed with "INVIDIOUS_") - {% for ivar in Config.instance_vars %} - {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %} - - if ENV.has_key?({{env_id}}) - env_value = ENV.fetch({{env_id}}) - success = false - - # Use YAML converter if specified - {% ann = ivar.annotation(::YAML::Field) %} - {% if ann && ann[:converter] %} - config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0]) - success = true - - # Use regular YAML parser otherwise - {% else %} - {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %} - # Sort types to avoid parsing nulls and numbers as strings - {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %} - {{ivar_types}}.each do |ivar_type| - if !success - begin - config.{{ivar.id}} = ivar_type.from_yaml(env_value) - success = true - rescue - # nop - end - end - end - {% end %} - - # Exit on fail - if !success - puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}}) - exit(1) - end - end - {% end %} - - - @@pot = config.po_token - @@vdata = config.visitor_data - - end - - def self.get_instance - return @@instance - end - -end \ No newline at end of file diff --git a/src/invidious/yt_backend/youtube_api.cr~ b/src/invidious/yt_backend/youtube_api.cr~ deleted file mode 100644 index b717a67e..00000000 --- a/src/invidious/yt_backend/youtube_api.cr~ +++ /dev/null @@ -1,685 +0,0 @@ -# -# This file contains youtube API wrappers -# - -module YoutubeAPI - extend self - - # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history - private ANDROID_APP_VERSION = "19.32.34" - private ANDROID_VERSION = "12" - private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip" - private ANDROID_SDK_VERSION = 31_i64 - - private ANDROID_TS_APP_VERSION = "1.9" - private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" - - # For Apple device names, see https://gist.github.com/adamawolf/3048717 - # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, - # then go to the dedicated article of the major version you want. - private IOS_APP_VERSION = "19.32.8" - private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)" - private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build - - private WINDOWS_VERSION = "10.0" - - # Enumerate used to select one of the clients supported by the API - enum ClientType - Web - WebEmbeddedPlayer - WebMobile - WebScreenEmbed - - Android - AndroidEmbeddedPlayer - AndroidScreenEmbed - AndroidTestSuite - - IOS - IOSEmbedded - IOSMusic - - TvHtml5 - TvHtml5ScreenEmbed - end - - # List of hard-coded values used by the different clients - HARDCODED_CLIENTS = { - ClientType::Web => { - name: "WEB", - name_proto: "1", - version: "2.20240814.00.00", - screen: "WATCH_FULL_SCREEN", - os_name: "Windows", - os_version: WINDOWS_VERSION, - platform: "DESKTOP", - }, - ClientType::WebEmbeddedPlayer => { - name: "WEB_EMBEDDED_PLAYER", - name_proto: "56", - version: "1.20240812.01.00", - screen: "EMBED", - os_name: "Windows", - os_version: WINDOWS_VERSION, - platform: "DESKTOP", - }, - ClientType::WebMobile => { - name: "MWEB", - name_proto: "2", - version: "2.20240813.02.00", - os_name: "Android", - os_version: ANDROID_VERSION, - platform: "MOBILE", - }, - ClientType::WebScreenEmbed => { - name: "WEB", - name_proto: "1", - version: "2.20240814.00.00", - screen: "EMBED", - os_name: "Windows", - os_version: WINDOWS_VERSION, - platform: "DESKTOP", - }, - - # Android - - ClientType::Android => { - name: "ANDROID", - name_proto: "3", - version: ANDROID_APP_VERSION, - android_sdk_version: ANDROID_SDK_VERSION, - user_agent: ANDROID_USER_AGENT, - os_name: "Android", - os_version: ANDROID_VERSION, - platform: "MOBILE", - }, - ClientType::AndroidEmbeddedPlayer => { - name: "ANDROID_EMBEDDED_PLAYER", - name_proto: "55", - version: ANDROID_APP_VERSION, - }, - ClientType::AndroidScreenEmbed => { - name: "ANDROID", - name_proto: "3", - version: ANDROID_APP_VERSION, - screen: "EMBED", - android_sdk_version: ANDROID_SDK_VERSION, - user_agent: ANDROID_USER_AGENT, - os_name: "Android", - os_version: ANDROID_VERSION, - platform: "MOBILE", - }, - ClientType::AndroidTestSuite => { - name: "ANDROID_TESTSUITE", - name_proto: "30", - version: ANDROID_TS_APP_VERSION, - android_sdk_version: ANDROID_SDK_VERSION, - user_agent: ANDROID_TS_USER_AGENT, - os_name: "Android", - os_version: ANDROID_VERSION, - platform: "MOBILE", - }, - - # IOS - - ClientType::IOS => { - name: "IOS", - name_proto: "5", - version: IOS_APP_VERSION, - user_agent: IOS_USER_AGENT, - device_make: "Apple", - device_model: "iPhone14,5", - os_name: "iPhone", - os_version: IOS_VERSION, - platform: "MOBILE", - }, - ClientType::IOSEmbedded => { - name: "IOS_MESSAGES_EXTENSION", - name_proto: "66", - version: IOS_APP_VERSION, - user_agent: IOS_USER_AGENT, - device_make: "Apple", - device_model: "iPhone14,5", - os_name: "iPhone", - os_version: IOS_VERSION, - platform: "MOBILE", - }, - ClientType::IOSMusic => { - name: "IOS_MUSIC", - name_proto: "26", - version: "7.14", - user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)", - device_make: "Apple", - device_model: "iPhone14,5", - os_name: "iPhone", - os_version: IOS_VERSION, - platform: "MOBILE", - }, - - # TV app - - ClientType::TvHtml5 => { - name: "TVHTML5", - name_proto: "7", - version: "7.20240813.07.00", - }, - ClientType::TvHtml5ScreenEmbed => { - name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", - name_proto: "85", - version: "2.0", - screen: "EMBED", - }, - } - - #################################################################### - # struct ClientConfig - # - # Data structure used to pass a client configuration to the different - # API endpoints handlers. - # - # Use case examples: - # - # ``` - # # Get Norwegian search results - # conf_1 = ClientConfig.new(region: "NO") - # YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1) - # - # # Use the Android client to request video streams URLs - # conf_2 = ClientConfig.new(client_type: ClientType::Android) - # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2) - # - # - struct ClientConfig - # Type of client to emulate. - # See `enum ClientType` and `HARDCODED_CLIENTS`. - property client_type : ClientType - - # Region to provide to youtube, e.g to alter search results - # (this is passed as the `gl` parameter). - property region : String | Nil - - # Initialization function - def initialize( - *, - @client_type = ClientType::Web, - @region = "US" - ) - end - - # Getter functions that provides easy access to hardcoded clients - # parameters (name/version strings and related API key) - def name : String - HARDCODED_CLIENTS[@client_type][:name] - end - - def name_proto : String - HARDCODED_CLIENTS[@client_type][:name_proto] - end - - # :ditto: - def version : String - HARDCODED_CLIENTS[@client_type][:version] - end - - # :ditto: - def screen : String - HARDCODED_CLIENTS[@client_type][:screen]? || "" - end - - def android_sdk_version : Int64? - HARDCODED_CLIENTS[@client_type][:android_sdk_version]? - end - - def user_agent : String? - HARDCODED_CLIENTS[@client_type][:user_agent]? - end - - def os_name : String? - HARDCODED_CLIENTS[@client_type][:os_name]? - end - - def device_make : String? - HARDCODED_CLIENTS[@client_type][:device_make]? - end - - def device_model : String? - HARDCODED_CLIENTS[@client_type][:device_model]? - end - - def os_version : String? - HARDCODED_CLIENTS[@client_type][:os_version]? - end - - def platform : String? - HARDCODED_CLIENTS[@client_type][:platform]? - end - - # Convert to string, for logging purposes - def to_s - return { - client_type: self.name, - region: @region, - }.to_s - end - end - - # Default client config, used if nothing is passed - DEFAULT_CLIENT_CONFIG = ClientConfig.new - - #################################################################### - # make_context(client_config) - # - # Return, as a Hash, the "context" data required to request the - # youtube API endpoints. - # - private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash - # Use the default client config if nil is passed - client_config ||= DEFAULT_CLIENT_CONFIG - - client_context = { - "client" => { - "hl" => "en", - "gl" => client_config.region || "US", # Can't be empty! - "clientName" => client_config.name, - "clientVersion" => client_config.version, - } of String => String | Int64, - } - - # Add some more context if it exists in the client definitions - if !client_config.screen.empty? - client_context["client"]["clientScreen"] = client_config.screen - end - - if client_config.screen == "EMBED" - client_context["thirdParty"] = { - "embedUrl" => "https://www.youtube.com/embed/#{video_id}", - } of String => String | Int64 - end - - if android_sdk_version = client_config.android_sdk_version - client_context["client"]["androidSdkVersion"] = android_sdk_version - end - - if device_make = client_config.device_make - client_context["client"]["deviceMake"] = device_make - end - - if device_model = client_config.device_model - client_context["client"]["deviceModel"] = device_model - end - - if os_name = client_config.os_name - client_context["client"]["osName"] = os_name - end - - if os_version = client_config.os_version - client_context["client"]["osVersion"] = os_version - end - - if platform = client_config.platform - client_context["client"]["platform"] = platform - end - - if ReloadPOToken.vdata.is_a?(String) - client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String) - end - - return client_context - end - - #################################################################### - # browse(continuation, client_config?) - # browse(browse_id, params, client_config?) - # - # Requests the youtubei/v1/browse endpoint with the required headers - # and POST data in order to get a JSON reply in english that can - # be easily parsed. - # - # Both forms can take an optional ClientConfig parameter (see - # `struct ClientConfig` above for more details). - # - # The requested data can either be: - # - # - A continuation token (ctoken). Depending on this token's - # contents, the returned data can be playlist videos, channel - # community tab content, channel info, ... - # - # - A playlist ID (parameters MUST be an empty string) - # - def browse(continuation : String, client_config : ClientConfig | Nil = nil) - # JSON Request data, required by the API - data = { - "context" => self.make_context(client_config), - "continuation" => continuation, - } - - return self._post_json("/youtubei/v1/browse", data, client_config) - end - - # :ditto: - def browse( - browse_id : String, - *, # Force the following parameters to be passed by name - params : String, - client_config : ClientConfig | Nil = nil - ) - # JSON Request data, required by the API - data = { - "browseId" => browse_id, - "context" => self.make_context(client_config), - } - - # Append the additional parameters if those were provided - # (this is required for channel info, playlist and community, e.g) - if params != "" - data["params"] = params - end - - return self._post_json("/youtubei/v1/browse", data, client_config) - end - - #################################################################### - # next(continuation, client_config?) - # next(data, client_config?) - # - # Requests the youtubei/v1/next endpoint with the required headers - # and POST data in order to get a JSON reply in english that can - # be easily parsed. - # - # Both forms can take an optional ClientConfig parameter (see - # `struct ClientConfig` above for more details). - # - # The requested data can be: - # - # - A continuation token (ctoken). Depending on this token's - # contents, the returned data can be videos comments, - # their replies, ... In this case, the string must be passed - # directly to the function. E.g: - # - # ``` - # YoutubeAPI::next("ABCDEFGH_abcdefgh==") - # ``` - # - # - Arbitrary parameters, in Hash form. See examples below for - # known examples of arbitrary data that can be passed to YouTube: - # - # ``` - # # Get the videos related to a specific video ID - # YoutubeAPI::next({"videoId" => "dQw4w9WgXcQ"}) - # - # # Get a playlist video's details - # YoutubeAPI::next({ - # "videoId" => "9bZkp7q19f0", - # "playlistId" => "PL_oFlvgqkrjUVQwiiE3F3k3voF4tjXeP0", - # }) - # ``` - # - def next(continuation : String, *, client_config : ClientConfig | Nil = nil) - # JSON Request data, required by the API - data = { - "context" => self.make_context(client_config), - "continuation" => continuation, - } - - return self._post_json("/youtubei/v1/next", data, client_config) - end - - # :ditto: - def next(data : Hash, *, client_config : ClientConfig | Nil = nil) - # JSON Request data, required by the API - data2 = data.merge({ - "context" => self.make_context(client_config), - }) - - return self._post_json("/youtubei/v1/next", data2, client_config) - end - - # Allow a NamedTuple to be passed, too. - def next(data : NamedTuple, *, client_config : ClientConfig | Nil = nil) - return self.next(data.to_h, client_config: client_config) - end - - #################################################################### - # player(video_id, params, client_config?) - # - # Requests the youtubei/v1/player endpoint with the required headers - # and POST data in order to get a JSON reply. - # - # The requested data is a video ID (`v=` parameter), with some - # additional parameters, formatted as a base64 string. - # - # An optional ClientConfig parameter can be passed, too (see - # `struct ClientConfig` above for more details). - # - def player( - video_id : String, - *, # Force the following parameters to be passed by name - params : String, - client_config : ClientConfig | Nil = nil - ) - # Playback context, separate because it can be different between clients - playback_ctx = { - "html5Preference" => "HTML5_PREF_WANTS", - "referer" => "https://www.youtube.com/watch?v=#{video_id}", - } of String => String | Int64 - - if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s } - if sts = DECRYPT_FUNCTION.try &.get_sts - playback_ctx["signatureTimestamp"] = sts.to_i64 - end - end - - # JSON Request data, required by the API - data = { - "contentCheckOk" => true, - "videoId" => video_id, - "context" => self.make_context(client_config, video_id), - "racyCheckOk" => true, - "user" => { - "lockedSafetyMode" => false, - }, - "playbackContext" => { - "contentPlaybackContext" => playback_ctx, - }, - "serviceIntegrityDimensions" => { - "poToken" => ReloadPOToken.pot.as(String), - }, - } - - # Append the additional parameters if those were provided - if params != "" - data["params"] = params - end - - return self._post_json("/youtubei/v1/player", data, client_config) - end - - #################################################################### - # resolve_url(url, client_config?) - # - # Requests the youtubei/v1/navigation/resolve_url endpoint with the - # required headers and POST data in order to get a JSON reply. - # - # An optional ClientConfig parameter can be passed, too (see - # `struct ClientConfig` above for more details). - # - # Output: - # - # ``` - # # Valid channel "brand URL" gives the related UCID and browse ID - # channel_a = YoutubeAPI.resolve_url("https://youtube.com/c/google") - # channel_a # => { - # "endpoint": { - # "browseEndpoint": { - # "params": "EgC4AQA%3D", - # "browseId":"UCK8sQmJBp8GCxrOtXWBpyEA" - # }, - # ... - # } - # } - # - # # Invalid URL returns throws an InfoException - # channel_b = YoutubeAPI.resolve_url("https://youtube.com/c/invalid") - # ``` - # - def resolve_url(url : String, client_config : ClientConfig | Nil = nil) - data = { - "context" => self.make_context(nil), - "url" => url, - } - - return self._post_json("/youtubei/v1/navigation/resolve_url", data, client_config) - end - - #################################################################### - # search(search_query, params, client_config?) - # - # Requests the youtubei/v1/search endpoint with the required headers - # and POST data in order to get a JSON reply. As the search results - # vary depending on the region, a region code can be specified in - # order to get non-US results. - # - # The requested data is a search string, with some additional - # parameters, formatted as a base64 string. - # - # An optional ClientConfig parameter can be passed, too (see - # `struct ClientConfig` above for more details). - # - def search( - search_query : String, - params : String, - client_config : ClientConfig | Nil = nil - ) - # JSON Request data, required by the API - data = { - "query" => search_query, - "context" => self.make_context(client_config), - "params" => params, - } - - return self._post_json("/youtubei/v1/search", data, client_config) - end - - #################################################################### - # get_transcript(params, client_config?) - # - # Requests the youtubei/v1/get_transcript endpoint with the required headers - # and POST data in order to get a JSON reply. - # - # The requested data is a specially encoded protobuf string that denotes the specific language requested. - # - # An optional ClientConfig parameter can be passed, too (see - # `struct ClientConfig` above for more details). - # - - def get_transcript( - params : String, - client_config : ClientConfig | Nil = nil - ) : Hash(String, JSON::Any) - data = { - "context" => self.make_context(client_config), - "params" => params, - } - - return self._post_json("/youtubei/v1/get_transcript", data, client_config) - end - - #################################################################### - # _post_json(endpoint, data, client_config?) - # - # Internal function that does the actual request to youtube servers - # and handles errors. - # - # The requested data is an endpoint (URL without the domain part) - # and the data as a Hash object. - # - def _post_json( - endpoint : String, - data : Hash, - client_config : ClientConfig | Nil - ) : Hash(String, JSON::Any) - # Use the default client config if nil is passed - client_config ||= DEFAULT_CLIENT_CONFIG - - # Query parameters - url = "#{endpoint}?prettyPrint=false" - - headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", - "x-goog-api-format-version" => "2", - "x-youtube-client-name" => client_config.name_proto, - "x-youtube-client-version" => client_config.version, - } - - if user_agent = client_config.user_agent - headers["User-Agent"] = user_agent - end - - if ReloadPOToken.vdata.is_a?(String) - headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String) - end - - # Logging - LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") - LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") - LOGGER.trace("YoutubeAPI: POST data: #{data}") - - # Send the POST request - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - self._decompress(response.body_io, response.headers["Content-Encoding"]?) - end - end - - # Convert result to Hash - initial_data = JSON.parse(body).as_h - - # Error handling - if initial_data.has_key?("error") - code = initial_data["error"]["code"] - message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "") - - # Logging - LOGGER.error("YoutubeAPI: Got error #{code} when requesting #{endpoint}") - LOGGER.error("YoutubeAPI: #{message}") - LOGGER.info("YoutubeAPI: POST data was: #{data}") - - raise InfoException.new("Could not extract JSON. Youtube API returned \ - error #{code} with message:
\"#{message}\"") - end - - return initial_data - end - - #################################################################### - # _decompress(body_io, headers) - # - # Internal function that reads the Content-Encoding headers and - # decompresses the content accordingly. - # - # We decompress the body ourselves (when using HTTP::Client) because - # the auto-decompress feature is broken in the Crystal stdlib. - # - # Read more: - # - https://github.com/iv-org/invidious/issues/2612 - # - https://github.com/crystal-lang/crystal/issues/11354 - # - def _decompress(body_io : IO, encodings : String?) : String - if encodings - # Multiple encodings can be combined, and are listed in the order - # in which they were applied. E.g: "deflate, gzip" means that the - # content must be first "gunzipped", then "defated". - encodings.split(',').reverse.each do |enc| - case enc.strip(' ') - when "gzip" - body_io = Compress::Gzip::Reader.new(body_io, sync_close: true) - when "deflate" - body_io = Compress::Deflate::Reader.new(body_io, sync_close: true) - end - end - end - - return body_io.gets_to_end - end -end # End of module From b761ed3dcd8c96e61013eec75277397f12ae5056 Mon Sep 17 00:00:00 2001 From: mooleshacat <43627985+mooleshacat@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:54:52 -0400 Subject: [PATCH 4/6] another fix --- src/invidious/jobs/token_monitor.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/jobs/token_monitor.cr b/src/invidious/jobs/token_monitor.cr index e863acfe..6ed3e52d 100644 --- a/src/invidious/jobs/token_monitor.cr +++ b/src/invidious/jobs/token_monitor.cr @@ -6,7 +6,7 @@ class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob LOGGER.info("jobs: running MonitorCfgTokensJob job") - ReloadPOToken.get_tokens + ReloadToken.get_tokens LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadTokens.pot.as(String)) LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadTokens.vdata.as(String)) From 2513aef39cb28268ccf44f98430e1f5e5390cd02 Mon Sep 17 00:00:00 2001 From: mooleshacat <43627985+mooleshacat@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:00:11 -0400 Subject: [PATCH 5/6] another fix --- src/invidious/jobs/token_monitor.cr | 2 +- src/invidious/reloadtoken.cr | 2 +- src/invidious/yt_backend/youtube_api.cr | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/invidious/jobs/token_monitor.cr b/src/invidious/jobs/token_monitor.cr index 6ed3e52d..a26b8e61 100644 --- a/src/invidious/jobs/token_monitor.cr +++ b/src/invidious/jobs/token_monitor.cr @@ -6,7 +6,7 @@ class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob LOGGER.info("jobs: running MonitorCfgTokensJob job") - ReloadToken.get_tokens + ReloadTokens.get_tokens LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadTokens.pot.as(String)) LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadTokens.vdata.as(String)) diff --git a/src/invidious/reloadtoken.cr b/src/invidious/reloadtoken.cr index 8ff56f48..f9428184 100644 --- a/src/invidious/reloadtoken.cr +++ b/src/invidious/reloadtoken.cr @@ -1,4 +1,4 @@ -class ReloadToken +class ReloadTokens @@instance = new diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 7862fd80..1b640bc2 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -320,8 +320,8 @@ module YoutubeAPI client_context["client"]["platform"] = platform end - if ReloadToken.vdata.is_a?(String) - client_context["client"]["visitorData"] = ReloadToken.vdata.as(String) + if ReloadTokens.vdata.is_a?(String) + client_context["client"]["visitorData"] = ReloadTokens.vdata.as(String) end return client_context @@ -482,7 +482,7 @@ module YoutubeAPI "contentPlaybackContext" => playback_ctx, }, "serviceIntegrityDimensions" => { - "poToken" => ReloadToken.pot.as(String), + "poToken" => ReloadTokens.pot.as(String), }, } @@ -616,8 +616,8 @@ module YoutubeAPI headers["User-Agent"] = user_agent end - if ReloadToken.vdata.is_a?(String) - headers["X-Goog-Visitor-Id"] = ReloadToken.vdata.as(String) + if ReloadTokens.vdata.is_a?(String) + headers["X-Goog-Visitor-Id"] = ReloadTokens.vdata.as(String) end # Logging From 0f2c57b358383b0a7bade319f1cf119f97aaa619 Mon Sep 17 00:00:00 2001 From: mooleshacat <43627985+mooleshacat@users.noreply.github.com> Date: Wed, 16 Oct 2024 01:50:56 -0400 Subject: [PATCH 6/6] cherry picked commits --- config/config.example.yml | 11 ++++++ shard.lock | 8 +++-- shard.yml | 3 ++ src/invidious.cr | 1 + src/invidious/config.cr | 11 ++++++ .../helpers/crystal_class_overrides.cr | 34 +++++++++++++++++++ src/invidious/yt_backend/connection_pool.cr | 19 ++++++++++- 7 files changed, 84 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index e9eebfde..759b81e0 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -173,6 +173,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/shard.lock b/shard.lock index 397bd8bc..50e64c64 100644 --- a/shard.lock +++ b/shard.lock @@ -10,7 +10,7 @@ shards: backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.3 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index 367f7c73..14c2a84e 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.3 development_dependencies: spectator: diff --git a/src/invidious.cr b/src/invidious.cr index b2387158..826d6768 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index a097b7f1..c1766fbb 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -55,6 +55,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -129,6 +138,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index fec3f62c..3040d7a0 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.12.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..70b15f26 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,16 @@ struct YoutubeConnectionPool def client(&) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -77,3 +81,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end