diff --git a/src/utils/youtube/fetchLatestUploads.ts b/src/utils/youtube/fetchLatestUploads.ts index 4aef1d7..b6f1ac7 100644 --- a/src/utils/youtube/fetchLatestUploads.ts +++ b/src/utils/youtube/fetchLatestUploads.ts @@ -1,4 +1,4 @@ -import type { YouTubeVideoContentDetailsResponse } from "../../types/youtube"; +import type { YouTubePlaylistResponse } from "../../types/youtube"; import { Platform } from "../../types/types.d"; import { @@ -13,79 +13,89 @@ import { import { discordGetAllGuildsTrackingChannel } from "../../db/discord"; import getChannelDetails from "./getChannelDetails"; -import getSinglePlaylistAndReturnVideoData, { - PlaylistType, -} from "./getSinglePlaylistAndReturnVideoData"; - -/** - * Parse a full ISO 8601 duration string into total seconds. - * Supports formats like "PT1H2M3S", "P1DT2H3M4S", "PT30S", etc. - * Note: YouTube typically returns PT-prefixed durations, but very long - * streams/videos could include a day component (P1DT...). - */ -function parseISO8601Duration(duration: string): number { - const match = duration.match( - /^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/, - ); - - if (!match) return 0; - const days = parseInt(match[1] || "0", 10); - const hours = parseInt(match[2] || "0", 10); - const minutes = parseInt(match[3] || "0", 10); - const seconds = parseInt(match[4] || "0", 10); +import { PlaylistType } from "./getSinglePlaylistAndReturnVideoData"; - return days * 86400 + hours * 3600 + minutes * 60 + seconds; -} +export const updates = new Map< + string, + { + channelInfo: Awaited>; + discordGuildsToUpdate: (typeof dbGuildYouTubeSubscriptionsTable.$inferSelect)[]; + } +>(); -/** - * Fetch the duration (in seconds) of a video using the YouTube Videos API. - * Returns 0 if the duration cannot be determined. - */ -async function fetchVideoDuration(videoId: string): Promise { +async function fetchUnseenUploadVideoIds( + channelId: string, + latestKnownVideoId: string, +): Promise { + const uploadPlaylistId = `UU${channelId.slice(2)}`; + const unseenVideoIds: string[] = []; const res = await fetch( - `https://youtube.googleapis.com/youtube/v3/videos?part=contentDetails&id=${videoId}&key=${env.youtubeApiKey}`, + `https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=50&playlistId=${uploadPlaylistId}&key=${env.youtubeApiKey}`, ); if (!res.ok) { - console.error("Error fetching video duration:", res.statusText); + console.error( + "Error fetching upload playlist items in fetchLatestUploads:", + res.statusText, + ); + return unseenVideoIds; + } + + const data = (await res.json()) as YouTubePlaylistResponse; - return 0; + if (!data.items || data.items.length === 0) { + return unseenVideoIds; } - const data = - (await res.json()) as unknown as YouTubeVideoContentDetailsResponse; + for (const item of data.items) { + const unseenVideoId = item?.snippet?.resourceId?.videoId; - if (!data.items || data.items.length === 0) return 0; + if (!unseenVideoId) { + continue; + } + + if (unseenVideoId === latestKnownVideoId) { + break; + } - const durationFirstItem = data.items[0]; + unseenVideoIds.push(unseenVideoId); + } + + return unseenVideoIds; +} + +async function fetchPlaylistVideoIds( + channelId: string, + playlistType: PlaylistType.Short | PlaylistType.Stream, +): Promise> { + const playlistPrefix = + playlistType === PlaylistType.Short ? "UUSH" : "UULV"; + const playlistId = `${playlistPrefix}${channelId.slice(2)}`; + const res = await fetch( + `https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=50&playlistId=${playlistId}&key=${env.youtubeApiKey}`, + ); - if ( - !durationFirstItem.contentDetails || - !durationFirstItem.contentDetails.duration - ) { + if (!res.ok) { console.error( - "Duration not found in video details for video ID:", - videoId, + `Error fetching ${playlistType} playlist items in fetchLatestUploads:`, + res.statusText, ); - - return 0; + return new Set(); } - const duration = durationFirstItem?.contentDetails?.duration; + const data = (await res.json()) as YouTubePlaylistResponse; - if (typeof duration !== "string") return 0; + if (!data.items || data.items.length === 0) { + return new Set(); + } - return parseISO8601Duration(duration); + return new Set( + data.items + .map((item) => item?.snippet?.resourceId?.videoId) + .filter((videoId): videoId is string => typeof videoId === "string"), + ); } -export const updates = new Map< - string, - { - channelInfo: Awaited>; - discordGuildsToUpdate: (typeof dbGuildYouTubeSubscriptionsTable.$inferSelect)[]; - } ->(); - export default async function fetchLatestUploads() { console.log("Fetching latest uploads..."); @@ -139,7 +149,7 @@ export default async function fetchLatestUploads() { // TODO: Upload time (https://github.com/GalvinPython/feedr/issues/136) for (const playlist of data.items) { const channelId = playlist.snippet.channelId; - const videoId = + const latestVideoId = playlist.snippet.thumbnails.default.url.split("/")[4]; if (!channelDict[channelId]) { @@ -150,103 +160,30 @@ export default async function fetchLatestUploads() { continue; } - const requiresUpdate = - channelDict[channelId].latestAllId !== videoId; + const latestKnownVideoId = channelDict[channelId].latestAllId; + const requiresUpdate = latestKnownVideoId !== latestVideoId; if (requiresUpdate) { console.log( "Channel ID:", channelId, "Video ID:", - videoId, + latestVideoId, "Requires update?", requiresUpdate, ); - // Use duration-based detection to reduce API quota usage - // and avoid UULF which is currently lagging. - // YouTube Shorts are currently limited to 3 minutes (180s). - // Videos at exactly 180s could still be shorts, so we use - // a strict greater-than check. - const durationSeconds = await fetchVideoDuration(videoId); - const SHORTS_DURATION = 180; - - let contentType: PlaylistType | null = null; - - if (durationSeconds > SHORTS_DURATION) { - // Over the shorts limit: cannot be a short, check only if it's a stream - const streamVideoId = - await getSinglePlaylistAndReturnVideoData( - channelId, - PlaylistType.Stream, - ); + let videosToProcess = [latestVideoId]; - if (videoId === streamVideoId.videoId) { - contentType = PlaylistType.Stream; - } else { - // Not a stream and over 3 min; must be a regular video - contentType = PlaylistType.Video; - } - } else { - // Under 3 minutes: could be a short or a video, check UUSH and UULV - const [shortVideoId, streamVideoId] = await Promise.all([ - getSinglePlaylistAndReturnVideoData( - channelId, - PlaylistType.Short, - ), - getSinglePlaylistAndReturnVideoData( - channelId, - PlaylistType.Stream, - ), - ]); - - if (videoId === shortVideoId.videoId) { - contentType = PlaylistType.Short; - } else if (videoId === streamVideoId.videoId) { - contentType = PlaylistType.Stream; - } else { - // Not in shorts or streams playlist → regular video - contentType = PlaylistType.Video; - } - } - - console.log( - "Determined content type:", - contentType, - `(duration: ${durationSeconds}s)`, - ); - - if (contentType) { - console.log( - `Updating ${contentType} video ID for channel`, - channelId, - "to", - videoId, - ); - } else { - console.error( - "No valid video ID found for channel", + if (latestKnownVideoId) { + const unseenVideoIds = await fetchUnseenUploadVideoIds( channelId, - "with video ID", - videoId, - ); - continue; - } - - const updateSuccess = await youtubeUpdateVideoId( - channelId, - videoId, - contentType, - - // Temporarily using current date for update time - new Date(), - ); - - if (!updateSuccess.success) { - console.error( - "Error updating video ID in fetchLatestUploads", + latestKnownVideoId, ); - return; + if (unseenVideoIds.length > 0) { + // Process oldest to newest so notifications preserve upload order. + videosToProcess = unseenVideoIds.reverse(); + } } const discordGuildsToUpdate = @@ -264,34 +201,69 @@ export default async function fetchLatestUploads() { } const channelInfo = await getChannelDetails(channelId); + const [shortVideoIds, streamVideoIds] = await Promise.all([ + fetchPlaylistVideoIds(channelId, PlaylistType.Short), + fetchPlaylistVideoIds(channelId, PlaylistType.Stream), + ]); + + for (const videoId of videosToProcess) { + let contentType = PlaylistType.Video; - console.info(`Filtered guilds for channel ID ${channelId}:`, { - count: discordGuildsToUpdate.data.filter( - ( - guild, - ): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect => - "youtubeChannelId" in guild && - "trackVideos" in guild && - "trackShorts" in guild && - "trackStreams" in guild, - ).length, - }); - - updates.set(videoId, { - channelInfo, - discordGuildsToUpdate: discordGuildsToUpdate.data.filter( - ( - guild, - ): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect => - "youtubeChannelId" in guild && - ((contentType === PlaylistType.Video && - guild.trackVideos) || - (contentType === PlaylistType.Short && - guild.trackShorts) || - (contentType === PlaylistType.Stream && - guild.trackStreams)), - ), - }); + if (shortVideoIds.has(videoId)) { + contentType = PlaylistType.Short; + } else if (streamVideoIds.has(videoId)) { + contentType = PlaylistType.Stream; + } else { + contentType = PlaylistType.Video; + } + + console.log("Determined content type:", contentType); + + const updateSuccess = await youtubeUpdateVideoId( + channelId, + videoId, + contentType, + + // Temporarily using current date for update time + new Date(), + ); + + if (!updateSuccess.success) { + console.error( + "Error updating video ID in fetchLatestUploads", + ); + + return; + } + + console.info(`Filtered guilds for channel ID ${channelId}:`, { + count: discordGuildsToUpdate.data.filter( + ( + guild, + ): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect => + "youtubeChannelId" in guild && + "trackVideos" in guild && + "trackShorts" in guild && + "trackStreams" in guild, + ).length, + }); + + updates.set(videoId, { + channelInfo, + discordGuildsToUpdate: discordGuildsToUpdate.data.filter( + ( + guild, + ): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect => + "youtubeChannelId" in guild && + ((contentType === PlaylistType.Video && + guild.trackVideos) || + (contentType === PlaylistType.Short && + guild.trackShorts) || + (contentType === PlaylistType.Stream && + guild.trackStreams)), + ), + }); + } } } }