From 965ac6b09d7b3c5d159153b95fa55934ed8b1058 Mon Sep 17 00:00:00 2001
From: mooleshacat <43627985+mooleshacat@users.noreply.github.com>
Date: Tue, 15 Oct 2024 17:53:55 -0400
Subject: [PATCH] few fixes
---
.gitignore | 1 +
src/invidious.cr~ | 248 --------
src/invidious/jobs/token_monitor.cr~ | 17 -
src/invidious/reloadpotoken.cr~ | 83 ---
src/invidious/yt_backend/youtube_api.cr~ | 685 -----------------------
5 files changed, 1 insertion(+), 1033 deletions(-)
delete mode 100644 src/invidious.cr~
delete mode 100644 src/invidious/jobs/token_monitor.cr~
delete mode 100644 src/invidious/reloadpotoken.cr~
delete mode 100644 src/invidious/yt_backend/youtube_api.cr~
diff --git a/.gitignore b/.gitignore
index 7a26e1a6..3dd4ab41 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+*~
/docs/
/dev/
/lib/
diff --git a/src/invidious.cr~ b/src/invidious.cr~
deleted file mode 100644
index 0b420ddf..00000000
--- a/src/invidious.cr~
+++ /dev/null
@@ -1,248 +0,0 @@
-# "Invidious" (which is an alternative front-end to YouTube)
-# Copyright (C) 2019 Omar Roth
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see .
-
-require "digest/md5"
-require "file_utils"
-
-# Require kemal, kilt, then our own overrides
-require "kemal"
-require "kilt"
-require "./ext/kemal_content_for.cr"
-require "./ext/kemal_static_file_handler.cr"
-
-require "athena-negotiation"
-require "openssl/hmac"
-require "option_parser"
-require "sqlite3"
-require "xml"
-require "yaml"
-require "compress/zip"
-require "protodec/utils"
-
-require "./invidious/database/*"
-require "./invidious/database/migrations/*"
-require "./invidious/http_server/*"
-require "./invidious/helpers/*"
-require "./invidious/yt_backend/*"
-require "./invidious/frontend/*"
-require "./invidious/videos/*"
-
-require "./invidious/jsonify/**"
-
-require "./invidious/*"
-require "./invidious/comments/*"
-require "./invidious/channels/*"
-require "./invidious/user/*"
-require "./invidious/search/*"
-require "./invidious/routes/**"
-require "./invidious/jobs/**"
-
-# Declare the base namespace for invidious
-module Invidious
-end
-
-# Simple alias to make code easier to read
-alias IV = Invidious
-
-CONFIG = Config.load
-HMAC_KEY = CONFIG.hmac_key
-
-PG_DB = DB.open CONFIG.database_url
-ARCHIVE_URL = URI.parse("https://archive.org")
-PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
-REDDIT_URL = URI.parse("https://www.reddit.com")
-YT_URL = URI.parse("https://www.youtube.com")
-HOST_URL = make_host_url(Kemal.config)
-
-CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
-TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
-MAX_ITEMS_PER_PAGE = 1500
-
-REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
-RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
-HTTP_CHUNK_SIZE = 10485760 # ~10MB
-
-CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
-CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
-CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
-
-# This is used to determine the `?v=` on the end of file URLs (for cache busting). We
-# only need to expire modified assets, so we can use this to find the last commit that changes
-# any assets
-ASSET_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}
-
-SOFTWARE = {
- "name" => "invidious",
- "version" => "#{CURRENT_VERSION}-#{CURRENT_COMMIT}",
- "branch" => "#{CURRENT_BRANCH}",
-}
-
-YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
-
-# CLI
-Kemal.config.extra_options do |parser|
- parser.banner = "Usage: invidious [arguments]"
- parser.on("-c THREADS", "--channel-threads=THREADS", "Number of threads for refreshing channels (default: #{CONFIG.channel_threads})") do |number|
- begin
- CONFIG.channel_threads = number.to_i
- rescue ex
- puts "THREADS must be integer"
- exit
- end
- end
- parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
- begin
- CONFIG.feed_threads = number.to_i
- rescue ex
- puts "THREADS must be integer"
- exit
- end
- end
- parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
- CONFIG.output = output
- end
- parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level|
- CONFIG.log_level = LogLevel.parse(log_level)
- end
- parser.on("-v", "--version", "Print version") do
- puts SOFTWARE.to_pretty_json
- exit
- end
- parser.on("--migrate", "Run any migrations (beta, use at your own risk!!") do
- Invidious::Database::Migrator.new(PG_DB).migrate
- exit
- end
-end
-
-Kemal::CLI.new ARGV
-
-if CONFIG.output.upcase != "STDOUT"
- FileUtils.mkdir_p(File.dirname(CONFIG.output))
-end
-OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a")
-LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
-
-# Check table integrity
-Invidious::Database.check_integrity(CONFIG)
-
-{% if !flag?(:skip_videojs_download) %}
- # Resolve player dependencies. This is done at compile time.
- #
- # Running the script by itself would show some colorful feedback while this doesn't.
- # Perhaps we should just move the script to runtime in order to get that feedback?
-
- {% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %}
- {% if flag?(:minified_player_dependencies) %}
- {% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %}
- {% else %}
- {% puts run("../scripts/fetch-player-dependencies.cr").stringify %}
- {% end %}
- {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}
-{% end %}
-
-# Misc
-
-DECRYPT_FUNCTION =
- if sig_helper_address = CONFIG.signature_server.presence
- IV::DecryptFunction.new(sig_helper_address)
- else
- nil
- end
-
-# Start jobs
-
-if CONFIG.channel_threads > 0
- Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
-end
-
-if CONFIG.feed_threads > 0
- Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
-end
-
-if CONFIG.statistics_enabled
- Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE)
-end
-
-if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0)
- Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, HMAC_KEY)
-end
-
-if CONFIG.popular_enabled
- Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
-end
-
-CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
-Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
-
-Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
-
-ReloadPOToken.get_tokens #init
-
-Invidious::Jobs.register Invidious::Jobs::MonitorCfgPotokensJob.new()
-
-Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
-
-Invidious::Jobs.start_all
-
-def popular_videos
- Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
-end
-
-# Routing
-
-before_all do |env|
- Invidious::Routes::BeforeAll.handle(env)
-end
-
-Invidious::Routing.register_all
-
-error 404 do |env|
- Invidious::Routes::ErrorRoutes.error_404(env)
-end
-
-error 500 do |env, ex|
- error_template(500, ex)
-end
-
-static_headers do |response|
- response.headers.add("Cache-Control", "max-age=2629800")
-end
-
-# Init Kemal
-
-public_folder "assets"
-
-Kemal.config.powered_by_header = false
-add_handler FilteredCompressHandler.new
-add_handler APIHandler.new
-add_handler AuthHandler.new
-add_handler DenyFrame.new
-add_context_storage_type(Array(String))
-add_context_storage_type(Preferences)
-add_context_storage_type(Invidious::User)
-
-Kemal.config.logger = LOGGER
-Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding
-Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port
-Kemal.config.app_name = "Invidious"
-
-# Use in kemal's production mode.
-# Users can also set the KEMAL_ENV environmental variable for this to be set automatically.
-{% if flag?(:release) || flag?(:production) %}
- Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV")
-{% end %}
-
-Kemal.run
diff --git a/src/invidious/jobs/token_monitor.cr~ b/src/invidious/jobs/token_monitor.cr~
deleted file mode 100644
index d923dd7d..00000000
--- a/src/invidious/jobs/token_monitor.cr~
+++ /dev/null
@@ -1,17 +0,0 @@
-
-class Invidious::Jobs::MonitorCfgTokensJob < Invidious::Jobs::BaseJob
- include Invidious
- def begin
- loop do
-
- LOGGER.info("jobs: running MonitorCfgTokensJob job")
-
- ReloadPOToken.get_tokens
-
- LOGGER.info("jobs: MonitorCfgTokensJob: pot: " + ReloadPOToken.pot.as(String))
- LOGGER.info("jobs: MonitorCfgTokensJob: vdata: " + ReloadPOToken.vdata.as(String))
-
- sleep 15.seconds
- end
- end
-end
diff --git a/src/invidious/reloadpotoken.cr~ b/src/invidious/reloadpotoken.cr~
deleted file mode 100644
index 2b874ad8..00000000
--- a/src/invidious/reloadpotoken.cr~
+++ /dev/null
@@ -1,83 +0,0 @@
-class ReloadPOToken
-
- @@instance = new
-
- def self.pot
- @@pot
- end
-
- def self.vdata
- @@vdata
- end
-
- def initialize
-
- @@pot = "error"
- @@vdata = "error"
-
- end
-
- def self.get_tokens
-
- # Load config from file or YAML string env var
- env_config_file = "INVIDIOUS_CONFIG_FILE"
- env_config_yaml = "INVIDIOUS_CONFIG"
-
- config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
- config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
-
- config = Config.from_yaml(config_yaml)
-
-
-
-
- # Update config from env vars (upcased and prefixed with "INVIDIOUS_")
- {% for ivar in Config.instance_vars %}
- {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
-
- if ENV.has_key?({{env_id}})
- env_value = ENV.fetch({{env_id}})
- success = false
-
- # Use YAML converter if specified
- {% ann = ivar.annotation(::YAML::Field) %}
- {% if ann && ann[:converter] %}
- config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
- success = true
-
- # Use regular YAML parser otherwise
- {% else %}
- {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %}
- # Sort types to avoid parsing nulls and numbers as strings
- {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %}
- {{ivar_types}}.each do |ivar_type|
- if !success
- begin
- config.{{ivar.id}} = ivar_type.from_yaml(env_value)
- success = true
- rescue
- # nop
- end
- end
- end
- {% end %}
-
- # Exit on fail
- if !success
- puts %(Config.{{ivar.id}} failed to parse #{env_value} as {{ivar.type}})
- exit(1)
- end
- end
- {% end %}
-
-
- @@pot = config.po_token
- @@vdata = config.visitor_data
-
- end
-
- def self.get_instance
- return @@instance
- end
-
-end
\ No newline at end of file
diff --git a/src/invidious/yt_backend/youtube_api.cr~ b/src/invidious/yt_backend/youtube_api.cr~
deleted file mode 100644
index b717a67e..00000000
--- a/src/invidious/yt_backend/youtube_api.cr~
+++ /dev/null
@@ -1,685 +0,0 @@
-#
-# This file contains youtube API wrappers
-#
-
-module YoutubeAPI
- extend self
-
- # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
- private ANDROID_APP_VERSION = "19.32.34"
- private ANDROID_VERSION = "12"
- private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip"
- private ANDROID_SDK_VERSION = 31_i64
-
- private ANDROID_TS_APP_VERSION = "1.9"
- private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip"
-
- # For Apple device names, see https://gist.github.com/adamawolf/3048717
- # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
- # then go to the dedicated article of the major version you want.
- private IOS_APP_VERSION = "19.32.8"
- private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)"
- private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build
-
- private WINDOWS_VERSION = "10.0"
-
- # Enumerate used to select one of the clients supported by the API
- enum ClientType
- Web
- WebEmbeddedPlayer
- WebMobile
- WebScreenEmbed
-
- Android
- AndroidEmbeddedPlayer
- AndroidScreenEmbed
- AndroidTestSuite
-
- IOS
- IOSEmbedded
- IOSMusic
-
- TvHtml5
- TvHtml5ScreenEmbed
- end
-
- # List of hard-coded values used by the different clients
- HARDCODED_CLIENTS = {
- ClientType::Web => {
- name: "WEB",
- name_proto: "1",
- version: "2.20240814.00.00",
- screen: "WATCH_FULL_SCREEN",
- os_name: "Windows",
- os_version: WINDOWS_VERSION,
- platform: "DESKTOP",
- },
- ClientType::WebEmbeddedPlayer => {
- name: "WEB_EMBEDDED_PLAYER",
- name_proto: "56",
- version: "1.20240812.01.00",
- screen: "EMBED",
- os_name: "Windows",
- os_version: WINDOWS_VERSION,
- platform: "DESKTOP",
- },
- ClientType::WebMobile => {
- name: "MWEB",
- name_proto: "2",
- version: "2.20240813.02.00",
- os_name: "Android",
- os_version: ANDROID_VERSION,
- platform: "MOBILE",
- },
- ClientType::WebScreenEmbed => {
- name: "WEB",
- name_proto: "1",
- version: "2.20240814.00.00",
- screen: "EMBED",
- os_name: "Windows",
- os_version: WINDOWS_VERSION,
- platform: "DESKTOP",
- },
-
- # Android
-
- ClientType::Android => {
- name: "ANDROID",
- name_proto: "3",
- version: ANDROID_APP_VERSION,
- android_sdk_version: ANDROID_SDK_VERSION,
- user_agent: ANDROID_USER_AGENT,
- os_name: "Android",
- os_version: ANDROID_VERSION,
- platform: "MOBILE",
- },
- ClientType::AndroidEmbeddedPlayer => {
- name: "ANDROID_EMBEDDED_PLAYER",
- name_proto: "55",
- version: ANDROID_APP_VERSION,
- },
- ClientType::AndroidScreenEmbed => {
- name: "ANDROID",
- name_proto: "3",
- version: ANDROID_APP_VERSION,
- screen: "EMBED",
- android_sdk_version: ANDROID_SDK_VERSION,
- user_agent: ANDROID_USER_AGENT,
- os_name: "Android",
- os_version: ANDROID_VERSION,
- platform: "MOBILE",
- },
- ClientType::AndroidTestSuite => {
- name: "ANDROID_TESTSUITE",
- name_proto: "30",
- version: ANDROID_TS_APP_VERSION,
- android_sdk_version: ANDROID_SDK_VERSION,
- user_agent: ANDROID_TS_USER_AGENT,
- os_name: "Android",
- os_version: ANDROID_VERSION,
- platform: "MOBILE",
- },
-
- # IOS
-
- ClientType::IOS => {
- name: "IOS",
- name_proto: "5",
- version: IOS_APP_VERSION,
- user_agent: IOS_USER_AGENT,
- device_make: "Apple",
- device_model: "iPhone14,5",
- os_name: "iPhone",
- os_version: IOS_VERSION,
- platform: "MOBILE",
- },
- ClientType::IOSEmbedded => {
- name: "IOS_MESSAGES_EXTENSION",
- name_proto: "66",
- version: IOS_APP_VERSION,
- user_agent: IOS_USER_AGENT,
- device_make: "Apple",
- device_model: "iPhone14,5",
- os_name: "iPhone",
- os_version: IOS_VERSION,
- platform: "MOBILE",
- },
- ClientType::IOSMusic => {
- name: "IOS_MUSIC",
- name_proto: "26",
- version: "7.14",
- user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)",
- device_make: "Apple",
- device_model: "iPhone14,5",
- os_name: "iPhone",
- os_version: IOS_VERSION,
- platform: "MOBILE",
- },
-
- # TV app
-
- ClientType::TvHtml5 => {
- name: "TVHTML5",
- name_proto: "7",
- version: "7.20240813.07.00",
- },
- ClientType::TvHtml5ScreenEmbed => {
- name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
- name_proto: "85",
- version: "2.0",
- screen: "EMBED",
- },
- }
-
- ####################################################################
- # struct ClientConfig
- #
- # Data structure used to pass a client configuration to the different
- # API endpoints handlers.
- #
- # Use case examples:
- #
- # ```
- # # Get Norwegian search results
- # conf_1 = ClientConfig.new(region: "NO")
- # YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1)
- #
- # # Use the Android client to request video streams URLs
- # conf_2 = ClientConfig.new(client_type: ClientType::Android)
- # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2)
- #
- #
- struct ClientConfig
- # Type of client to emulate.
- # See `enum ClientType` and `HARDCODED_CLIENTS`.
- property client_type : ClientType
-
- # Region to provide to youtube, e.g to alter search results
- # (this is passed as the `gl` parameter).
- property region : String | Nil
-
- # Initialization function
- def initialize(
- *,
- @client_type = ClientType::Web,
- @region = "US"
- )
- end
-
- # Getter functions that provides easy access to hardcoded clients
- # parameters (name/version strings and related API key)
- def name : String
- HARDCODED_CLIENTS[@client_type][:name]
- end
-
- def name_proto : String
- HARDCODED_CLIENTS[@client_type][:name_proto]
- end
-
- # :ditto:
- def version : String
- HARDCODED_CLIENTS[@client_type][:version]
- end
-
- # :ditto:
- def screen : String
- HARDCODED_CLIENTS[@client_type][:screen]? || ""
- end
-
- def android_sdk_version : Int64?
- HARDCODED_CLIENTS[@client_type][:android_sdk_version]?
- end
-
- def user_agent : String?
- HARDCODED_CLIENTS[@client_type][:user_agent]?
- end
-
- def os_name : String?
- HARDCODED_CLIENTS[@client_type][:os_name]?
- end
-
- def device_make : String?
- HARDCODED_CLIENTS[@client_type][:device_make]?
- end
-
- def device_model : String?
- HARDCODED_CLIENTS[@client_type][:device_model]?
- end
-
- def os_version : String?
- HARDCODED_CLIENTS[@client_type][:os_version]?
- end
-
- def platform : String?
- HARDCODED_CLIENTS[@client_type][:platform]?
- end
-
- # Convert to string, for logging purposes
- def to_s
- return {
- client_type: self.name,
- region: @region,
- }.to_s
- end
- end
-
- # Default client config, used if nothing is passed
- DEFAULT_CLIENT_CONFIG = ClientConfig.new
-
- ####################################################################
- # make_context(client_config)
- #
- # Return, as a Hash, the "context" data required to request the
- # youtube API endpoints.
- #
- private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash
- # Use the default client config if nil is passed
- client_config ||= DEFAULT_CLIENT_CONFIG
-
- client_context = {
- "client" => {
- "hl" => "en",
- "gl" => client_config.region || "US", # Can't be empty!
- "clientName" => client_config.name,
- "clientVersion" => client_config.version,
- } of String => String | Int64,
- }
-
- # Add some more context if it exists in the client definitions
- if !client_config.screen.empty?
- client_context["client"]["clientScreen"] = client_config.screen
- end
-
- if client_config.screen == "EMBED"
- client_context["thirdParty"] = {
- "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
- } of String => String | Int64
- end
-
- if android_sdk_version = client_config.android_sdk_version
- client_context["client"]["androidSdkVersion"] = android_sdk_version
- end
-
- if device_make = client_config.device_make
- client_context["client"]["deviceMake"] = device_make
- end
-
- if device_model = client_config.device_model
- client_context["client"]["deviceModel"] = device_model
- end
-
- if os_name = client_config.os_name
- client_context["client"]["osName"] = os_name
- end
-
- if os_version = client_config.os_version
- client_context["client"]["osVersion"] = os_version
- end
-
- if platform = client_config.platform
- client_context["client"]["platform"] = platform
- end
-
- if ReloadPOToken.vdata.is_a?(String)
- client_context["client"]["visitorData"] = ReloadPOToken.vdata.as(String)
- end
-
- return client_context
- end
-
- ####################################################################
- # browse(continuation, client_config?)
- # browse(browse_id, params, client_config?)
- #
- # Requests the youtubei/v1/browse endpoint with the required headers
- # and POST data in order to get a JSON reply in english that can
- # be easily parsed.
- #
- # Both forms can take an optional ClientConfig parameter (see
- # `struct ClientConfig` above for more details).
- #
- # The requested data can either be:
- #
- # - A continuation token (ctoken). Depending on this token's
- # contents, the returned data can be playlist videos, channel
- # community tab content, channel info, ...
- #
- # - A playlist ID (parameters MUST be an empty string)
- #
- def browse(continuation : String, client_config : ClientConfig | Nil = nil)
- # JSON Request data, required by the API
- data = {
- "context" => self.make_context(client_config),
- "continuation" => continuation,
- }
-
- return self._post_json("/youtubei/v1/browse", data, client_config)
- end
-
- # :ditto:
- def browse(
- browse_id : String,
- *, # Force the following parameters to be passed by name
- params : String,
- client_config : ClientConfig | Nil = nil
- )
- # JSON Request data, required by the API
- data = {
- "browseId" => browse_id,
- "context" => self.make_context(client_config),
- }
-
- # Append the additional parameters if those were provided
- # (this is required for channel info, playlist and community, e.g)
- if params != ""
- data["params"] = params
- end
-
- return self._post_json("/youtubei/v1/browse", data, client_config)
- end
-
- ####################################################################
- # next(continuation, client_config?)
- # next(data, client_config?)
- #
- # Requests the youtubei/v1/next endpoint with the required headers
- # and POST data in order to get a JSON reply in english that can
- # be easily parsed.
- #
- # Both forms can take an optional ClientConfig parameter (see
- # `struct ClientConfig` above for more details).
- #
- # The requested data can be:
- #
- # - A continuation token (ctoken). Depending on this token's
- # contents, the returned data can be videos comments,
- # their replies, ... In this case, the string must be passed
- # directly to the function. E.g:
- #
- # ```
- # YoutubeAPI::next("ABCDEFGH_abcdefgh==")
- # ```
- #
- # - Arbitrary parameters, in Hash form. See examples below for
- # known examples of arbitrary data that can be passed to YouTube:
- #
- # ```
- # # Get the videos related to a specific video ID
- # YoutubeAPI::next({"videoId" => "dQw4w9WgXcQ"})
- #
- # # Get a playlist video's details
- # YoutubeAPI::next({
- # "videoId" => "9bZkp7q19f0",
- # "playlistId" => "PL_oFlvgqkrjUVQwiiE3F3k3voF4tjXeP0",
- # })
- # ```
- #
- def next(continuation : String, *, client_config : ClientConfig | Nil = nil)
- # JSON Request data, required by the API
- data = {
- "context" => self.make_context(client_config),
- "continuation" => continuation,
- }
-
- return self._post_json("/youtubei/v1/next", data, client_config)
- end
-
- # :ditto:
- def next(data : Hash, *, client_config : ClientConfig | Nil = nil)
- # JSON Request data, required by the API
- data2 = data.merge({
- "context" => self.make_context(client_config),
- })
-
- return self._post_json("/youtubei/v1/next", data2, client_config)
- end
-
- # Allow a NamedTuple to be passed, too.
- def next(data : NamedTuple, *, client_config : ClientConfig | Nil = nil)
- return self.next(data.to_h, client_config: client_config)
- end
-
- ####################################################################
- # player(video_id, params, client_config?)
- #
- # Requests the youtubei/v1/player endpoint with the required headers
- # and POST data in order to get a JSON reply.
- #
- # The requested data is a video ID (`v=` parameter), with some
- # additional parameters, formatted as a base64 string.
- #
- # An optional ClientConfig parameter can be passed, too (see
- # `struct ClientConfig` above for more details).
- #
- def player(
- video_id : String,
- *, # Force the following parameters to be passed by name
- params : String,
- client_config : ClientConfig | Nil = nil
- )
- # Playback context, separate because it can be different between clients
- playback_ctx = {
- "html5Preference" => "HTML5_PREF_WANTS",
- "referer" => "https://www.youtube.com/watch?v=#{video_id}",
- } of String => String | Int64
-
- if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s }
- if sts = DECRYPT_FUNCTION.try &.get_sts
- playback_ctx["signatureTimestamp"] = sts.to_i64
- end
- end
-
- # JSON Request data, required by the API
- data = {
- "contentCheckOk" => true,
- "videoId" => video_id,
- "context" => self.make_context(client_config, video_id),
- "racyCheckOk" => true,
- "user" => {
- "lockedSafetyMode" => false,
- },
- "playbackContext" => {
- "contentPlaybackContext" => playback_ctx,
- },
- "serviceIntegrityDimensions" => {
- "poToken" => ReloadPOToken.pot.as(String),
- },
- }
-
- # Append the additional parameters if those were provided
- if params != ""
- data["params"] = params
- end
-
- return self._post_json("/youtubei/v1/player", data, client_config)
- end
-
- ####################################################################
- # resolve_url(url, client_config?)
- #
- # Requests the youtubei/v1/navigation/resolve_url endpoint with the
- # required headers and POST data in order to get a JSON reply.
- #
- # An optional ClientConfig parameter can be passed, too (see
- # `struct ClientConfig` above for more details).
- #
- # Output:
- #
- # ```
- # # Valid channel "brand URL" gives the related UCID and browse ID
- # channel_a = YoutubeAPI.resolve_url("https://youtube.com/c/google")
- # channel_a # => {
- # "endpoint": {
- # "browseEndpoint": {
- # "params": "EgC4AQA%3D",
- # "browseId":"UCK8sQmJBp8GCxrOtXWBpyEA"
- # },
- # ...
- # }
- # }
- #
- # # Invalid URL returns throws an InfoException
- # channel_b = YoutubeAPI.resolve_url("https://youtube.com/c/invalid")
- # ```
- #
- def resolve_url(url : String, client_config : ClientConfig | Nil = nil)
- data = {
- "context" => self.make_context(nil),
- "url" => url,
- }
-
- return self._post_json("/youtubei/v1/navigation/resolve_url", data, client_config)
- end
-
- ####################################################################
- # search(search_query, params, client_config?)
- #
- # Requests the youtubei/v1/search endpoint with the required headers
- # and POST data in order to get a JSON reply. As the search results
- # vary depending on the region, a region code can be specified in
- # order to get non-US results.
- #
- # The requested data is a search string, with some additional
- # parameters, formatted as a base64 string.
- #
- # An optional ClientConfig parameter can be passed, too (see
- # `struct ClientConfig` above for more details).
- #
- def search(
- search_query : String,
- params : String,
- client_config : ClientConfig | Nil = nil
- )
- # JSON Request data, required by the API
- data = {
- "query" => search_query,
- "context" => self.make_context(client_config),
- "params" => params,
- }
-
- return self._post_json("/youtubei/v1/search", data, client_config)
- end
-
- ####################################################################
- # get_transcript(params, client_config?)
- #
- # Requests the youtubei/v1/get_transcript endpoint with the required headers
- # and POST data in order to get a JSON reply.
- #
- # The requested data is a specially encoded protobuf string that denotes the specific language requested.
- #
- # An optional ClientConfig parameter can be passed, too (see
- # `struct ClientConfig` above for more details).
- #
-
- def get_transcript(
- params : String,
- client_config : ClientConfig | Nil = nil
- ) : Hash(String, JSON::Any)
- data = {
- "context" => self.make_context(client_config),
- "params" => params,
- }
-
- return self._post_json("/youtubei/v1/get_transcript", data, client_config)
- end
-
- ####################################################################
- # _post_json(endpoint, data, client_config?)
- #
- # Internal function that does the actual request to youtube servers
- # and handles errors.
- #
- # The requested data is an endpoint (URL without the domain part)
- # and the data as a Hash object.
- #
- def _post_json(
- endpoint : String,
- data : Hash,
- client_config : ClientConfig | Nil
- ) : Hash(String, JSON::Any)
- # Use the default client config if nil is passed
- client_config ||= DEFAULT_CLIENT_CONFIG
-
- # Query parameters
- url = "#{endpoint}?prettyPrint=false"
-
- headers = HTTP::Headers{
- "Content-Type" => "application/json; charset=UTF-8",
- "Accept-Encoding" => "gzip, deflate",
- "x-goog-api-format-version" => "2",
- "x-youtube-client-name" => client_config.name_proto,
- "x-youtube-client-version" => client_config.version,
- }
-
- if user_agent = client_config.user_agent
- headers["User-Agent"] = user_agent
- end
-
- if ReloadPOToken.vdata.is_a?(String)
- headers["X-Goog-Visitor-Id"] = ReloadPOToken.vdata.as(String)
- end
-
- # Logging
- LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"")
- LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")
- LOGGER.trace("YoutubeAPI: POST data: #{data}")
-
- # Send the POST request
- body = YT_POOL.client() do |client|
- client.post(url, headers: headers, body: data.to_json) do |response|
- self._decompress(response.body_io, response.headers["Content-Encoding"]?)
- end
- end
-
- # Convert result to Hash
- initial_data = JSON.parse(body).as_h
-
- # Error handling
- if initial_data.has_key?("error")
- code = initial_data["error"]["code"]
- message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "")
-
- # Logging
- LOGGER.error("YoutubeAPI: Got error #{code} when requesting #{endpoint}")
- LOGGER.error("YoutubeAPI: #{message}")
- LOGGER.info("YoutubeAPI: POST data was: #{data}")
-
- raise InfoException.new("Could not extract JSON. Youtube API returned \
- error #{code} with message:
\"#{message}\"")
- end
-
- return initial_data
- end
-
- ####################################################################
- # _decompress(body_io, headers)
- #
- # Internal function that reads the Content-Encoding headers and
- # decompresses the content accordingly.
- #
- # We decompress the body ourselves (when using HTTP::Client) because
- # the auto-decompress feature is broken in the Crystal stdlib.
- #
- # Read more:
- # - https://github.com/iv-org/invidious/issues/2612
- # - https://github.com/crystal-lang/crystal/issues/11354
- #
- def _decompress(body_io : IO, encodings : String?) : String
- if encodings
- # Multiple encodings can be combined, and are listed in the order
- # in which they were applied. E.g: "deflate, gzip" means that the
- # content must be first "gunzipped", then "defated".
- encodings.split(',').reverse.each do |enc|
- case enc.strip(' ')
- when "gzip"
- body_io = Compress::Gzip::Reader.new(body_io, sync_close: true)
- when "deflate"
- body_io = Compress::Deflate::Reader.new(body_io, sync_close: true)
- end
- end
- end
-
- return body_io.gets_to_end
- end
-end # End of module