From b17ee5d4f736571200b8b88aeb71a6276a8aa30a Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 20 Jul 2021 11:42:45 +0200 Subject: [PATCH] Put youtube API functions under the YoutubeAPI namespace --- src/invidious/channels/playlists.cr | 2 +- src/invidious/channels/videos.cr | 2 +- src/invidious/helpers/youtube_api.cr | 216 ++++++++++++++------------- src/invidious/playlists.cr | 6 +- src/invidious/search.cr | 4 +- src/invidious/trending.cr | 2 +- 6 files changed, 119 insertions(+), 113 deletions(-) diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index 222ec2b1..393b055e 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -1,6 +1,6 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation - response_json = request_youtube_api_browse(continuation) + response_json = YoutubeAPI.browse(continuation) continuationItems = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index cc291e9e..2c43bf0b 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -61,7 +61,7 @@ def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) - return request_youtube_api_browse(continuation) + return YoutubeAPI.browse(continuation) end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/helpers/youtube_api.cr index 734fddcd..d134a60a 100644 --- a/src/invidious/helpers/youtube_api.cr +++ b/src/invidious/helpers/youtube_api.cr @@ -2,120 +2,126 @@ # This file contains youtube API wrappers # -# Hard-coded constants required by the API -HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" -HARDCODED_CLIENT_VERS = "2.20210330.08.00" +module YoutubeAPI + extend self -#################################################################### -# make_youtube_api_context(region) -# -# Return, as a Hash, the "context" data required to request the -# youtube API endpoints. -# -def make_youtube_api_context(region : String | Nil) : Hash - return { - "client" => { - "hl" => "en", - "gl" => region || "US", # Can't be empty! - "clientName" => "WEB", - "clientVersion" => HARDCODED_CLIENT_VERS, - }, - } -end + # Hard-coded constants required by the API + HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" + HARDCODED_CLIENT_VERS = "2.20210330.08.00" -#################################################################### -# request_youtube_api_browse(continuation) -# request_youtube_api_browse(browse_id, params, region) -# -# Requests the youtubei/v1/browse endpoint with the required headers -# and POST data in order to get a JSON reply in english that can -# be easily parsed. -# -# The region can be provided, default is US. -# -# The requested data can either be: -# -# - A continuation token (ctoken). Depending on this token's -# contents, the returned data can be comments, playlist videos, -# search results, channel community tab, ... -# -# - A playlist ID (parameters MUST be an empty string) -# -def request_youtube_api_browse(continuation : String) - # JSON Request data, required by the API - data = { - "context" => make_youtube_api_context("US"), - "continuation" => continuation, - } - - return _youtube_api_post_json("/youtubei/v1/browse", data) -end - -def request_youtube_api_browse(browse_id : String, params : String, region : String = "US") - # JSON Request data, required by the API - data = { - "browseId" => browse_id, - "context" => make_youtube_api_context(region), - } - - # Append the additionnal parameters if those were provided - # (this is required for channel info, playlist and community, e.g) - if params != "" - data["params"] = params + #################################################################### + # make_context(region) + # + # Return, as a Hash, the "context" data required to request the + # youtube API endpoints. + # + private def make_context(region : String | Nil) : Hash + return { + "client" => { + "hl" => "en", + "gl" => region || "US", # Can't be empty! + "clientName" => "WEB", + "clientVersion" => HARDCODED_CLIENT_VERS, + }, + } end - return _youtube_api_post_json("/youtubei/v1/browse", data) -end + #################################################################### + # browse(continuation) + # browse(browse_id, params) + # browse(browse_id, params, region) + # + # Requests the youtubei/v1/browse endpoint with the required headers + # and POST data in order to get a JSON reply in english that can + # be easily parsed. + # + # A region can be provided, default is US. + # + # The requested data can either be: + # + # - A continuation token (ctoken). Depending on this token's + # contents, the returned data can be comments, playlist videos, + # search results, channel community tab, ... + # + # - A playlist ID (parameters MUST be an empty string) + # + def browse(continuation : String) + # JSON Request data, required by the API + data = { + "context" => self.make_context("US"), + "continuation" => continuation, + } -#################################################################### -# request_youtube_api_search(search_query, params, region) -# -# Requests the youtubei/v1/search endpoint with the required headers -# and POST data in order to get a JSON reply. As the search results -# vary depending on the region, a region code can be specified in -# order to get non-US results. -# -# The requested data is a search string, with some additional -# paramters, formatted as a base64 string. -# -def request_youtube_api_search(search_query : String, params : String, region = nil) - # JSON Request data, required by the API - data = { - "query" => search_query, - "context" => make_youtube_api_context(region), - "params" => params, - } + return self._post_json("/youtubei/v1/browse", data) + end - return _youtube_api_post_json("/youtubei/v1/search", data) -end + # :ditto: + def browse(browse_id : String, *, params : String, region : String = "US") + # JSON Request data, required by the API + data = { + "browseId" => browse_id, + "context" => self.make_context(region), + } -#################################################################### -# _youtube_api_post_json(endpoint, data) -# -# Internal function that does the actual request to youtube servers -# and handles errors. -# -# The requested data is an endpoint (URL without the domain part) -# and the data as a Hash object. -# -def _youtube_api_post_json(endpoint, data) - # Send the POST request and parse result - response = YT_POOL.client &.post( - "#{endpoint}?key=#{HARDCODED_API_KEY}", - headers: HTTP::Headers{"content-type" => "application/json; charset=UTF-8"}, - body: data.to_json - ) + # Append the additionnal parameters if those were provided + # (this is required for channel info, playlist and community, e.g) + if params != "" + data["params"] = params + end - initial_data = JSON.parse(response.body).as_h + return self._post_json("/youtubei/v1/browse", data) + end - # Error handling - if initial_data.has_key?("error") - code = initial_data["error"]["code"] - message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "") + #################################################################### + # search(search_query, params, region) + # + # Requests the youtubei/v1/search endpoint with the required headers + # and POST data in order to get a JSON reply. As the search results + # vary depending on the region, a region code can be specified in + # order to get non-US results. + # + # The requested data is a search string, with some additional + # paramters, formatted as a base64 string. + # + def search(search_query : String, params : String, region = nil) + # JSON Request data, required by the API + data = { + "query" => search_query, + "context" => self.make_context(region), + "params" => params, + } - raise InfoException.new("Could not extract JSON. Youtube API returned \ + return self._post_json("/youtubei/v1/search", data) + end + + #################################################################### + # _post_json(endpoint, data) + # + # Internal function that does the actual request to youtube servers + # and handles errors. + # + # The requested data is an endpoint (URL without the domain part) + # and the data as a Hash object. + # + def _post_json(endpoint, data) : Hash(String, JSON::Any) + # Send the POST request and parse result + response = YT_POOL.client &.post( + "#{endpoint}?key=#{HARDCODED_API_KEY}", + headers: HTTP::Headers{"content-type" => "application/json; charset=UTF-8"}, + body: data.to_json + ) + + initial_data = JSON.parse(response.body).as_h + + # Error handling + if initial_data.has_key?("error") + code = initial_data["error"]["code"] + message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "") + + raise InfoException.new("Could not extract JSON. Youtube API returned \ error #{code} with message:
\"#{message}\"") - end + end - return initial_data -end + return initial_data + end +end # End of module diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index fe7f82f3..f56cc2ea 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -361,7 +361,7 @@ def fetch_playlist(plid, locale) plid = "UU#{plid.lchop("UC")}" end - initial_data = request_youtube_api_browse("VL" + plid, params: "") + initial_data = YoutubeAPI.browse("VL" + plid, params: "") playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]? raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer @@ -442,9 +442,9 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) offset = (offset / 100).to_i64 * 100_i64 ctoken = produce_playlist_continuation(playlist.id, offset) - initial_data = request_youtube_api_browse(ctoken) + initial_data = YoutubeAPI.browse(ctoken) else - initial_data = request_youtube_api_browse("VL" + playlist.id, params: "") + initial_data = YoutubeAPI.browse("VL" + playlist.id, params: "") end return extract_playlist_videos(initial_data) diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 662173a0..5e9bd202 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -244,7 +244,7 @@ def channel_search(query, page, channel) end continuation = produce_channel_search_continuation(ucid, query, page) - response_json = request_youtube_api_browse(continuation) + response_json = YoutubeAPI.browse(continuation) continuationItems = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] @@ -263,7 +263,7 @@ end def search(query, search_params = produce_search_params(content_type: "all"), region = nil) return 0, [] of SearchItem if query.empty? - initial_data = request_youtube_api_search(query, search_params, region) + initial_data = YoutubeAPI.search(query, search_params, region) items = extract_items(initial_data) return items.size, items diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 2ab1e7ba..829875cb 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -14,7 +14,7 @@ def fetch_trending(trending_type, region, locale) params = "" end - initial_data = request_youtube_api_browse("FEtrending", params: params, region: region) + initial_data = YoutubeAPI.browse("FEtrending", params: params, region: region) trending = extract_videos(initial_data) return {trending, plid}