mirrors/nadeko_invidious
1
0
Fork 0
mirror of https://git.nadeko.net/Fijxu/invidious.git synced 2025-12-19 19:38:51 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #1608 from saltycrys/add-subscription-traces

Browse Source 
Add Subscription Traces
This commit is contained in:
Perflyst
2020-12-31 11:30:04 +01:00
committed by GitHub
parent 82c8f3b556 c4ef055248
commit c89632d2a8
7 changed files with 72 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/invidious/channels.cr
View File

@@ -144,7 +144,7 @@ class ChannelRedirect < Exception
end
end
def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
def get_batch_channels(channels, db, logger, refresh = false, pull_all_videos = true, max_threads = 10)
finished_channel = Channel(String | Nil).new
spawn do
@@ -160,7 +160,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
active_threads += 1
spawn do
begin
get_channel(ucid, db, refresh, pull_all_videos)
get_channel(ucid, db, logger, refresh, pull_all_videos)
finished_channel.send(ucid)
rescue ex
finished_channel.send(nil)
@@ -181,10 +181,10 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
return final
end
def get_channel(id, db, refresh = true, pull_all_videos = true)
def get_channel(id, db, logger, refresh = true, pull_all_videos = true)
if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
if refresh && Time.utc - channel.updated > 10.minutes
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel = fetch_channel(id, db, logger, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
args = arg_array(channel_array)
@@ -192,7 +192,7 @@ def get_channel(id, db, refresh = true, pull_all_videos = true)
ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array)
end
else
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel = fetch_channel(id, db, logger, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
args = arg_array(channel_array)
@@ -202,8 +202,12 @@ def get_channel(id, db, refresh = true, pull_all_videos = true)
return channel
end
def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
def fetch_channel(ucid, db, logger, pull_all_videos = true, locale = nil)
logger.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}")
logger.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
logger.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse_html(rss)
author = rss.xpath_node(%q(//feed/title))
@@ -219,14 +223,19 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
auto_generated = true
end
logger.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
page = 1
logger.trace("fetch_channel: #{ucid} : Downloading channel videos page")
response = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = [] of SearchVideo
begin
initial_data = JSON.parse(response.body).as_a.find &.["response"]?
raise InfoException.new("Could not extract channel JSON") if !initial_data
logger.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data")
videos = extract_videos(initial_data.as_h, author, ucid)
rescue ex
if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
@@ -236,6 +245,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
raise ex
end
logger.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//feed/entry").each do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content
@@ -269,6 +279,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
views: views,
})
logger.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
@@ -276,8 +288,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool)
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert
if was_insert
logger.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
db.exec("UPDATE users SET notifications = array_append(notifications, $1), \
feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid)
else
logger.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
end
if pull_all_videos