Skip to content

Commit

Permalink
Merge dev changes for v5.3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@Inrixia
Inrixia committed Mar 28, 2022
2 parents 06300ee + 2c7dc55 commit d3582b2
Show file tree
Hide file tree
Showing 15 changed files with 994 additions and 1,761 deletions.
2,399 changes: 825 additions & 1,574 deletions package-lock.json
View file

Large diffs are not rendered by default.

27 changes: 13 additions & 14 deletions package.json
View file
@@ -1,6 +1,6 @@
{
"name": "floatplane-plex-downloader",
"version": "5.2.0",
"version": "5.3.0",
"private": true,
"scripts": {
"prep": "npm install && npm run build",
Expand All @@ -15,19 +15,18 @@
"dependencies": {
"@ctrl/plex": "^1.5.3",
"@inrixia/db": "^1.8.0",
"@inrixia/helpers": "^1.22.3",
"@inrixia/helpers": "^1.23.1",
"chalk": "^4.1.2",
"ffbinaries": "^1.1.4",
"floatplane": "^3.1.8",
"got": "^11.8.2",
"html-to-text": "^8.0.0",
"multi-progress-bars": "^4.0.1",
"floatplane": "^3.2.0",
"html-to-text": "^8.1.0",
"multi-progress-bars": "^4.2.1",
"process.argv": "^0.6.0",
"prompts": "^2.4.1",
"prompts": "^2.4.2",
"sanitize-filename": "^1.6.3",
"semver": "^7.3.5",
"tough-cookie": "^4.0.0",
"tough-cookie-file-store": "^2.0.2"
"tough-cookie-file-store": "^2.0.3"
},
"pkg": {
"scripts": "./float.js",
Expand All @@ -41,13 +40,13 @@
"@types/html-to-text": "^8.0.1",
"@types/multi-progress": "^2.0.3",
"@types/prompts": "^2.0.14",
"@types/semver": "^7.3.8",
"@types/semver": "^7.3.9",
"@types/tough-cookie-file-store": "^2.0.1",
"@typescript-eslint/eslint-plugin": "^4.31.0",
"@typescript-eslint/parser": "^4.31.0",
"eslint": "^7.32.0",
"@typescript-eslint/eslint-plugin": "^5.9.0",
"@typescript-eslint/parser": "^5.9.0",
"eslint": "^8.6.0",
"eslint-plugin-prettier": "^4.0.0",
"pkg": "^5.3.1",
"typescript": "^4.4.2"
"pkg": "^5.5.1",
"typescript": "^4.5.4"
}
}
15 changes: 8 additions & 7 deletions src/Downloader.ts
View file
Expand Up @@ -71,12 +71,13 @@ export default class Downloader {
);
// (videos remaining * avg time to download a video)
const totalVideos = this.videoQueue.length + this.videosProcessed + this.videosProcessing;
const whitespace = ' ';
const processed = `Processed: ${ye(this.videosProcessed)}/${ye(totalVideos)}${whitespace}`;
const downloaded = `Total Downloaded: ${cy(downloadedMB.toFixed(2))}/${cy(totalMB.toFixed(2) + 'MB')}${whitespace}`;
const speed = `Download Speed: ${gr(((downloadSpeed / 1024000) * 8).toFixed(2) + 'Mb/s')}${whitespace}`;
process.stdout.write(' ');
process.stdout.write(`\n${processed}\n${downloaded}\n${speed}\n\n\n`);
const processed = `Processed: ${ye(this.videosProcessed)}/${ye(totalVideos)}`;
const downloaded = `Total Downloaded: ${cy(downloadedMB.toFixed(2))}/${cy(totalMB.toFixed(2) + 'MB')}`;
const speed = `Download Speed: ${gr(((downloadSpeed / 1024000) * 8).toFixed(2) + 'Mb/s')}`;
this.mpb?.setFooter({
message: `${processed} ${downloaded} ${speed}`,
pattern: '',
});
}

/**
Expand Down Expand Up @@ -124,7 +125,7 @@ export default class Downloader {

await Promise.all(
downloadRequests.map(async (request, i) => {
request.on('downloadProgress', (downloadProgress) => {
request.on('downloadProgress', (downloadProgress: { total: number; transferred: number; percent: number }) => {
const timeElapsed = (Date.now() - startTime) / 1000;

totalBytes[i] = downloadProgress.total;
Expand Down
23 changes: 3 additions & 20 deletions src/float.ts
View file
Expand Up @@ -4,7 +4,6 @@ import { settings, fetchFFMPEG } from './lib/helpers';
import { MyPlexAccount } from '@ctrl/plex';
import { fApi } from './lib/FloatplaneAPI';
import { loginFloatplane } from './logins';
import { deleteOldVideos } from './lib/deleteOldVideos';
import Downloader from './Downloader';
import { gt, diff } from 'semver';
import { resolve } from 'path';
Expand All @@ -17,15 +16,9 @@ import type Subscription from './lib/Subscription';
*/
const fetchNewVideos = async (subscriptions: Array<Subscription>, videoProcessor: Downloader) => {
for (const subscription of subscriptions) {
await Promise.all(
videoProcessor.processVideos(
await subscription.fetchNewVideos(
settings.floatplane.videosToSearch,
settings.extras.stripSubchannelPrefix,
settings.daysToKeepVideos !== -1 ? Date.now() - settings.daysToKeepVideos * 24 * 60 * 60 * 1000 : false
)
)
);
await subscription.deleteOldVideos();
console.log();
await Promise.all(videoProcessor.processVideos(await subscription.fetchNewVideos(settings.floatplane.videosToSearch, settings.extras.stripSubchannelPrefix)));
}

if (settings.plex.enabled) {
Expand Down Expand Up @@ -59,16 +52,6 @@ const fetchNewVideos = async (subscriptions: Array<Subscription>, videoProcessor
if (settings.runQuickstartPrompts) await quickStart();
settings.runQuickstartPrompts = false;

if (settings.daysToKeepVideos !== -1) {
const rootVideoFolder = resolve(settings.filePathFormatting.split('%')[0]);
process.stdout.write(
chalk`Checking for files older than {cyanBright ${settings.daysToKeepVideos}} days in {yellow ${rootVideoFolder}} for {redBright deletion}...`
);
const deleted = await deleteOldVideos(rootVideoFolder, settings.daysToKeepVideos);
if (deleted === 0) console.log(' No files found for deletion.\n');
else console.log(chalk` Deleted {redBright ${deleted}} files.\n`);
}

// Get Plex details if not saved
await validatePlexSettings();

Expand Down
67 changes: 55 additions & 12 deletions src/lib/Channel.ts
View file
@@ -1,3 +1,6 @@
import fs from 'fs/promises';
import chalk from 'chalk';

import db from '@inrixia/db';
import Video from './Video';

Expand All @@ -6,22 +9,25 @@ import type { ChannelOptions } from './types';
import type Subscription from './Subscription';

// e = episodeNo, d = downloaded, s = filesize in bytes, f = file
export type VideoDBEntry = { episodeNo: number; expectedSize?: number };
export type VideoDBEntry = { episodeNo: number; expectedSize?: number; filePath?: string; releaseDate: number };
export type ChannelDB = {
videos: { [key: string]: VideoDBEntry };
nextEpisodeNo: number;
};

export default class Channel {
public title: ChannelOptions['title'];
public identifiers: ChannelOptions['identifiers'];
public skip: ChannelOptions['skip'];
public readonly title: ChannelOptions['title'];
public readonly identifiers: ChannelOptions['identifiers'];
public readonly skip: ChannelOptions['skip'];
public readonly daysToKeepVideos: ChannelOptions['daysToKeepVideos'];

public readonly ignoreBeforeTimestamp: number;

public consoleColor: ChannelOptions['consoleColor'];
public readonly consoleColor: ChannelOptions['consoleColor'];

public subscription: Subscription;

public _db: ChannelDB;
private readonly _db: ChannelDB;
/**
* Returns a channel built from a subscription.
* @param {ChannelOptions} channel
Expand All @@ -34,6 +40,10 @@ export default class Channel {
this.skip = channel.skip;
this.consoleColor = channel.consoleColor;

if (channel.daysToKeepVideos === undefined) channel.daysToKeepVideos = -1;
this.daysToKeepVideos = channel.daysToKeepVideos;
this.ignoreBeforeTimestamp = Date.now() - this.daysToKeepVideos * 24 * 60 * 60 * 1000;

const databaseFilePath = `./db/channels/${subscription.creatorId}/${channel.title}.json`;
try {
this._db = db<ChannelDB>(databaseFilePath, { template: { videos: {}, nextEpisodeNo: 1 } });
Expand All @@ -42,17 +52,50 @@ export default class Channel {
}
}

public deleteOldVideos = async () => {
if (this.daysToKeepVideos !== -1) {
process.stdout.write(
chalk`Checking for videos older than {cyanBright ${this.daysToKeepVideos}} days in channel {yellow ${this.title}} for {redBright deletion}...`
);
let deletedFiles = 0;
let deletedVideos = 0;
for (const video of Object.values(this._db.videos)) {
if (video.releaseDate === undefined || video.filePath === undefined) continue;
if (video.releaseDate < this.ignoreBeforeTimestamp) {
deletedVideos++;
const deletionResults = await Promise.allSettled([
fs.rm(`${video.filePath}.mp4`),
fs.rm(`${video.filePath}.partial`),
fs.rm(`${video.filePath}.nfo`),
fs.rm(`${video.filePath}.png`),
]);
for (const result of deletionResults) {
if (result.status === 'fulfilled') deletedFiles++;
}
}
}
if (deletedFiles === 0) console.log(' No files found for deletion.');
else console.log(chalk` Deleted {redBright ${deletedVideos}} videos, {redBright ${deletedFiles}} files.`);
}
};

public lookupVideoDB = (guid: string): VideoDBEntry => this._db.videos[guid];

public markVideoCompleted(guid: string, releaseDate: string): void {
public markVideoCompleted(guid: string, releaseDate: number): void {
// Redundant check but worth keeping
if (this._db.videos[guid] === undefined) throw new Error(`Cannot mark unknown video ${guid} as completed. Video does not exist in channel database.`);
if (this.lookupVideoDB(guid) === undefined) throw new Error(`Cannot mark unknown video ${guid} as completed. Video does not exist in channel database.`);
this.subscription.updateLastSeenVideo({ guid, releaseDate });
}

public addVideo(video: BlogPost): Video {
// Set the episode number
this._db.videos[video.guid] ??= { episodeNo: this._db.nextEpisodeNo++ };
return new Video(video, this);
public addVideo(video: BlogPost): Video | null {
const releaseDate = new Date(video.releaseDate).getTime();
if (this.daysToKeepVideos !== -1 && releaseDate < this.ignoreBeforeTimestamp) return null;

// Set db info, have to instigate the db first before setting filepath
this._db.videos[video.guid] ??= { episodeNo: this._db.nextEpisodeNo++, releaseDate, filePath: '' };
const videoInstance = new Video(video, this);
this._db.videos[video.guid].filePath = videoInstance.filePath;

return videoInstance;
}
}
27 changes: 16 additions & 11 deletions src/lib/Subscription.ts
View file
Expand Up @@ -8,18 +8,17 @@ import type Video from './Video';

type LastSeenVideo = {
guid: BlogPost['guid'];
releaseDate: BlogPost['releaseDate'];
releaseDate: number;
};
type SubscriptionDB = {
lastSeenVideo: LastSeenVideo;
videos: BlogPost[];
};

export default class Subscription {
public channels: Channel[];
public defaultChannel: Channel;

public creatorId: string;
public readonly creatorId: string;

private _db: SubscriptionDB;
constructor(subscription: SubscriptionSettings) {
Expand All @@ -31,7 +30,7 @@ export default class Subscription {
// Load/Create database
const databaseFilePath = `./db/subscriptions/${subscription.creatorId}.json`;
try {
this._db = db<SubscriptionDB>(databaseFilePath, { template: { lastSeenVideo: { guid: '', releaseDate: '' }, videos: [] } });
this._db = db<SubscriptionDB>(databaseFilePath, { template: { lastSeenVideo: { guid: '', releaseDate: 0 } } });
} catch {
throw new Error(`Cannot load Subscription database file ${databaseFilePath}! Please delete the file or fix it!`);
}
Expand All @@ -42,15 +41,18 @@ export default class Subscription {
}

public updateLastSeenVideo(videoSeen: LastSeenVideo): void {
if (this.lastSeenVideo.releaseDate === '' || new Date(videoSeen.releaseDate) > new Date(this.lastSeenVideo.releaseDate)) this._db.lastSeenVideo = videoSeen;
if (videoSeen.releaseDate > this.lastSeenVideo.releaseDate) this._db.lastSeenVideo = videoSeen;
}

public deleteOldVideos = async () => {
for (const channel of this.channels) await channel.deleteOldVideos();
};

/**
* @param {fApiVideo} video
*/
public addVideo(video: BlogPost, overrideSkip: true, stripSubchannelPrefix?: boolean): ReturnType<Channel['addVideo']>;
public addVideo(video: BlogPost, overrideSkip?: false, stripSubchannelPrefix?: boolean): ReturnType<Channel['addVideo']> | null;

public addVideo(video: BlogPost, overrideSkip = false, stripSubchannelPrefix = true): ReturnType<Channel['addVideo']> | null {
for (const channel of this.channels) {
// Check if the video belongs to this channel
Expand Down Expand Up @@ -79,7 +81,7 @@ export default class Subscription {
return this.defaultChannel.addVideo(video);
}

public async fetchNewVideos(videosToSearch = 20, stripSubchannelPrefix: boolean, ignoreBeforeTimestamp: number | false): Promise<Array<Video>> {
public async fetchNewVideos(videosToSearch = 20, stripSubchannelPrefix: boolean): Promise<Array<Video>> {
const coloredTitle = `${this.defaultChannel.consoleColor || '\u001b[38;5;208m'}${this.defaultChannel.title}\u001b[0m`;

const videos = [];
Expand All @@ -88,10 +90,14 @@ export default class Subscription {

for await (const video of fApi.creator.blogPostsIterable(this.creatorId, { type: 'video' })) {
if (video.guid === this.lastSeenVideo.guid) {
if (!(await this.addVideo(video, true, stripSubchannelPrefix).isDownloaded())) this.lastSeenVideo.guid = '';
else break;
// If we have found the last seen video, check if its downloaded.
// If it is then break here and return the videos we have found.
// Otherwise continue to fetch new videos up to the videosToSearch limit to ensure partially or non downloaded videos are returned.
const channelVideo = this.addVideo(video, true, stripSubchannelPrefix);
if (channelVideo === null || channelVideo.isDownloaded()) break;
}
if (this.lastSeenVideo.guid === '' && videos.length >= videosToSearch) break;
// Stop searching if we have looked through videosToSearch
if (videos.length >= videosToSearch) break;
videos.push(video);
process.stdout.write(`\r> Fetching latest videos from [${coloredTitle}]... Fetched ${videos.length} videos!`);
}
Expand All @@ -102,7 +108,6 @@ export default class Subscription {
.sort((a, b) => +new Date(a.releaseDate) - +new Date(b.releaseDate))
.map((video) => this.addVideo(video, false, stripSubchannelPrefix))) {
if (video === null || (await video.isMuxed())) continue;
if (ignoreBeforeTimestamp !== false && new Date(video.releaseDate).getTime() < ignoreBeforeTimestamp) continue;
incompleteVideos.push(video);
}
process.stdout.write(` Skipped ${videos.length - incompleteVideos.length}.\n`);
Expand Down

0 comments on commit d3582b2

Please sign in to comment.