Skip to content

Commit

Permalink
Added support for videos with multiple files
Browse files Browse the repository at this point in the history
  • Loading branch information
@Inrixia
Inrixia committed Nov 6, 2021
1 parent 5768934 commit dce7ad0
Show file tree
Hide file tree
Showing 2 changed files with 132 additions and 88 deletions.
71 changes: 45 additions & 26 deletions src/downloader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export default class Downloader {
private videoQueue: Array<{ video: Video; res: promiseFunction }> = [];
private videosProcessing: number = 0;
private videosProcessed: number = 0;
private downloadStats: { [key: string]: { totalMB: number; downloadedMB: number; downloadSpeed: number } } = {};
private summaryStats: { [key: string]: { totalMB: number; downloadedMB: number; downloadSpeed: number } } = {};

private runQueue: boolean = false;

Expand Down Expand Up @@ -49,7 +49,7 @@ export default class Downloader {
if (videos.length !== 0) {
console.log(`> Processing ${videos.length} videos...`);
if (args.headless !== true) this.mpb = new MultiProgressBars({ initMessage: '', anchor: 'top' });
this.downloadStats = {};
this.summaryStats = {};
this.videosProcessed = 0;
}
const processingPromises = videos.reverse().map((video) => new Promise<void>((res) => this.videoQueue.push({ video, res })));
Expand All @@ -59,8 +59,8 @@ export default class Downloader {
}

private updateSummaryBar(): void {
if (this.downloadStats === undefined) return;
const { totalMB, downloadedMB, downloadSpeed } = Object.values(this.downloadStats).reduce(
if (this.summaryStats === undefined) return;
const { totalMB, downloadedMB, downloadSpeed } = Object.values(this.summaryStats).reduce(
(summary, stats) => {
for (const key in stats) {
summary[key as keyof typeof stats] += stats[key as keyof typeof stats];
Expand Down Expand Up @@ -101,7 +101,7 @@ export default class Downloader {
);
} else formattedTitle = `${video.channel.title} - ${video.title}`.slice(0, 32);

if (this.downloadStats !== undefined) while (formattedTitle in this.downloadStats) formattedTitle = `.${formattedTitle}`.slice(0, 32);
if (this.summaryStats !== undefined) while (formattedTitle in this.summaryStats) formattedTitle = `.${formattedTitle}`.slice(0, 32);

if (args.headless === true) console.log(`${formattedTitle} - Downloading...`);
else {
Expand All @@ -116,27 +116,46 @@ export default class Downloader {
// If the video is already downloaded then just mux its metadata
if (!(await video.isMuxed()) && !(await video.isDownloaded())) {
const startTime = Date.now();
const downloadRequest = await video.download(settings.floatplane.videoResolution as string, allowRangeQuery);
downloadRequest.on('downloadProgress', (downloadProgress) => {
const totalMB = downloadProgress.total / 1024000;
const downloadedMB = downloadProgress.transferred / 1024000;
const timeElapsed = (Date.now() - startTime) / 1000;
const downloadSpeed = downloadProgress.transferred / timeElapsed;
const downloadETA = downloadProgress.total / downloadSpeed - timeElapsed; // Round to 4 decimals
this.updateBar(formattedTitle, {
percentage: downloadProgress.percent,
message: `${reset}${cy(downloadedMB.toFixed(2))}/${cy(totalMB.toFixed(2) + 'MB')} ${gr(((downloadSpeed / 1024000) * 8).toFixed(2) + 'Mb/s')} ETA: ${bl(
Math.floor(downloadETA / 60) + 'm ' + (Math.floor(downloadETA) % 60) + 's'
)}`,
});
this.downloadStats[formattedTitle] = { totalMB, downloadedMB, downloadSpeed };
if (args.headless !== true) this.updateSummaryBar();
});
await new Promise((res, rej) => {
downloadRequest.on('end', res);
downloadRequest.on('error', rej);
});
this.downloadStats[formattedTitle].downloadSpeed = 0;
const downloadRequests = await video.download(settings.floatplane.videoResolution as string, allowRangeQuery);

const totalBytes: number[] = [];
const downloadedBytes: number[] = [];
const percentage: number[] = [];

await Promise.all(
downloadRequests.map(async (request, i) => {
request.on('downloadProgress', (downloadProgress) => {
const timeElapsed = (Date.now() - startTime) / 1000;

totalBytes[i] = downloadProgress.total;
downloadedBytes[i] = downloadProgress.transferred;
percentage[i] = downloadProgress.percent;

// Sum the stats for multi part video downloads
const total = totalBytes.reduce((sum, b) => sum + b, 0);
const transferred = downloadedBytes.reduce((sum, b) => sum + b, 0);

const totalMB = total / 1024000;
const downloadedMB = transferred / 1024000;
const downloadSpeed = transferred / timeElapsed;
const downloadETA = total / downloadSpeed - timeElapsed; // Round to 4 decimals

this.updateBar(formattedTitle, {
percentage: percentage.reduce((sum, b) => sum + b, 0) / percentage.length,
message: `${reset}${cy(downloadedMB.toFixed(2))}/${cy(totalMB.toFixed(2) + 'MB')} ${gr(((downloadSpeed / 1024000) * 8).toFixed(2) + 'Mb/s')} ETA: ${bl(
Math.floor(downloadETA / 60) + 'm ' + (Math.floor(downloadETA) % 60) + 's'
)}`,
});
this.summaryStats[formattedTitle] = { totalMB, downloadedMB, downloadSpeed };
if (args.headless !== true) this.updateSummaryBar();
});
await new Promise((res, rej) => {
request.on('end', res);
request.on('error', rej);
});
})
);
this.summaryStats[formattedTitle].downloadSpeed = 0;
}
if (!(await video.isMuxed())) {
this.updateBar(formattedTitle, {
Expand Down
149 changes: 87 additions & 62 deletions src/lib/Video.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export default class Video {
this.channel = channel;

this.guid = video.guid;
this.videoAttachments = video.videoAttachments;
this.videoAttachments = video.attachmentOrder.filter((a) => video.videoAttachments.includes(a));
this.title = video.title;
this.description = video.text;
this.releaseDate = new Date(video.releaseDate);
Expand Down Expand Up @@ -73,6 +73,11 @@ export default class Video {
return `${this.folderPath}/${sanitize(this.fullPath.split('/').slice(-1)[0])}`;
}

/**
* Get the suffix for a video file if there are multiple videoAttachments for this video
*/
private multiPartSuffix = (attachmentIndex: string | number): string => `${this.videoAttachments.length !== 1 ? ` - Part ${+attachmentIndex + 1}` : ''}`;

get expectedSize(): number | undefined {
return this.channel.lookupVideoDB(this.guid).expectedSize;
}
Expand All @@ -82,13 +87,17 @@ export default class Video {

static getFileBytes = async (path: string): Promise<number> => (await fs.stat(path).catch(() => ({ size: -1 }))).size;

public downloadedBytes = async (): Promise<number> => Video.getFileBytes(`${this.filePath}.partial`);
public isDownloaded = async (): Promise<boolean> => (await this.isMuxed()) || (await this.downloadedBytes()) === this.expectedSize;

public muxedBytes = async (): Promise<number> => Video.getFileBytes(`${this.filePath}.mp4`);
public isMuxed = async (): Promise<boolean> => (await this.muxedBytes()) === this.expectedSize;
public fileBytes = async (extension: string): Promise<number> => {
let bytes = 0;
for (const i in this.videoAttachments) {
bytes += await Video.getFileBytes(`${this.filePath}${this.multiPartSuffix(i)}.${extension}`);
}
return bytes;
};
public isDownloaded = async (): Promise<boolean> => (await this.isMuxed()) || (await this.fileBytes('partial')) === this.expectedSize;
public isMuxed = async (): Promise<boolean> => (await this.fileBytes('mp4')) === this.expectedSize;

public async download(quality: string, allowRangeQuery = true): Promise<Request> {
public async download(quality: string, allowRangeQuery = true): Promise<Request[]> {
if (await this.isDownloaded()) throw new Error(`Attempting to download "${this.title}" video already downloaded!`);

// Make sure the folder for the video exists
Expand Down Expand Up @@ -140,85 +149,101 @@ export default class Video {
await fs.writeFile(`${this.filePath}.nfo`, nfo, 'utf8');
}

// Handle download resumption if video was partially downloaded
let writeStreamOptions, requestOptions, downloadedBytes;
if (allowRangeQuery && this.expectedSize !== undefined && (downloadedBytes = await this.downloadedBytes()) !== -1) {
[writeStreamOptions, requestOptions] = [{ start: downloadedBytes, flags: 'r+' }, { headers: { range: `bytes=${downloadedBytes}-${this.expectedSize}` } }];
// Download resumption is not currently supported for multi part videos...
if (this.videoAttachments.length === 1) {
// Handle download resumption if video was partially downloaded
if (allowRangeQuery && this.expectedSize !== undefined && (downloadedBytes = await this.fileBytes('partial')) !== -1) {
[writeStreamOptions, requestOptions] = [{ start: downloadedBytes, flags: 'r+' }, { headers: { range: `bytes=${downloadedBytes}-${this.expectedSize}` } }];
}
}

// Send download request video, assume the first video attached is the actual video as most will not have more than one video
const cdnInfo = await fApi.cdn.delivery('download', this.videoAttachments[0]);
let downloadRequests = [];
for (const i in this.videoAttachments) {
// Send download request video, assume the first video attached is the actual video as most will not have more than one video
const cdnInfo = await fApi.cdn.delivery('download', this.videoAttachments[i]);

// Pick a random edge to download off, eventual even distribution
const downloadEdge = cdnInfo.edges[Math.floor(Math.random() * cdnInfo.edges.length)];
if (settings.floatplane.downloadEdge !== '') downloadEdge.hostname = settings.floatplane.downloadEdge;
// Pick a random edge to download off, eventual even distribution
const downloadEdge = cdnInfo.edges[Math.floor(Math.random() * cdnInfo.edges.length)];
if (settings.floatplane.downloadEdge !== '') downloadEdge.hostname = settings.floatplane.downloadEdge;

// Convert the qualities into an array of resolutions and sorts them smallest to largest
const availableQualities = cdnInfo.resource.data.qualityLevels.map((quality) => quality.name).sort((a, b) => +b - +a);
// Convert the qualities into an array of resolutions and sorts them smallest to largest
const availableQualities = cdnInfo.resource.data.qualityLevels.map((quality) => quality.name).sort((a, b) => +b - +a);

// Set the quality to use based on whats given in the settings.json or the highest available
const downloadQuality = availableQualities.includes(quality) ? quality : availableQualities[availableQualities.length - 1];
// Set the quality to use based on whats given in the settings.json or the highest available
const downloadQuality = availableQualities.includes(quality) ? quality : availableQualities[availableQualities.length - 1];

const downloadRequest = fApi.got.stream(
`https://${downloadEdge.hostname}${cdnInfo.resource.uri.replace('{qualityLevels}', downloadQuality).replace('{token}', cdnInfo.resource.data.token)}`,
requestOptions
);
// Pipe the download to the file once response starts
downloadRequest.pipe(createWriteStream(`${this.filePath}.partial`, writeStreamOptions));
// Set the videos expectedSize once we know how big it should be for download validation.
if (this.expectedSize === undefined) downloadRequest.once('downloadProgress', (progress) => (this.expectedSize = progress.total));
const downloadRequest = fApi.got.stream(
`https://${downloadEdge.hostname}${cdnInfo.resource.uri.replace('{qualityLevels}', downloadQuality).replace('{token}', cdnInfo.resource.data.token)}`,
requestOptions
);
// Pipe the download to the file once response starts
downloadRequest.pipe(createWriteStream(`${this.filePath}${this.multiPartSuffix(i)}.partial`, writeStreamOptions));
// Set the videos expectedSize once we know how big it should be for download validation.
if (this.expectedSize === undefined) downloadRequest.once('downloadProgress', (progress) => (this.expectedSize = progress.total));
downloadRequests.push(downloadRequest);
}

return downloadRequest;
return downloadRequests;
}

public async markCompleted(): Promise<void> {
if (!(await this.isMuxed()))
throw new Error(
`Cannot mark ${this.title} as completed as video file size is not correct. Expected: ${this.expectedSize} bytes, Got: ${await this.muxedBytes()} bytes...`
`Cannot mark ${this.title} as completed as video file size is not correct. Expected: ${this.expectedSize} bytes, Got: ${await this.fileBytes(
'mp4'
)} bytes...`
);
return this.channel.markVideoCompleted(this.guid, this.releaseDate.toString());
}

public async muxffmpegMetadata(): Promise<void> {
if (!this.isDownloaded())
throw new Error(
`Cannot mux ffmpeg metadata for ${this.title} as its not downloaded. Expected: ${this.expectedSize}, Got: ${await this.downloadedBytes()} bytes...`
`Cannot mux ffmpeg metadata for ${this.title} as its not downloaded. Expected: ${this.expectedSize}, Got: ${await this.fileBytes('partial')} bytes...`
);
await new Promise((resolve, reject) =>
execFile(
args.headless === true ? './ffmpeg' : './db/ffmpeg',
[
'-i',
`${this.filePath}.partial`,
'-metadata',
`title=${this.title}`,
'-metadata',
`AUTHOR=${this.channel.title}`,
'-metadata',
`YEAR=${this.releaseDate}`,
'-metadata',
`date=${this.releaseDate.getFullYear().toString() + nPad(this.releaseDate.getMonth() + 1) + nPad(this.releaseDate.getDate())}`,
'-metadata',
`description=${htmlToText(this.description)}`,
'-metadata',
`synopsis=${htmlToText(this.description)}`,
'-c:a',
'copy',
'-c:v',
'copy',
`${this.filePath}.mp4`,
],
(error, stdout) => {
if (error !== null) reject(error);
else resolve(stdout);
}
await Promise.all(
this.videoAttachments.map(
(a, i) =>
new Promise((resolve, reject) =>
execFile(
args.headless === true ? './ffmpeg' : './db/ffmpeg',
[
'-i',
`${this.filePath}${this.multiPartSuffix(i)}.partial`,
'-metadata',
`title=${this.title}${this.multiPartSuffix(i)}`,
'-metadata',
`AUTHOR=${this.channel.title}`,
'-metadata',
`YEAR=${this.releaseDate}`,
'-metadata',
`date=${this.releaseDate.getFullYear().toString() + nPad(this.releaseDate.getMonth() + 1) + nPad(this.releaseDate.getDate())}`,
'-metadata',
`description=${htmlToText(this.description)}`,
'-metadata',
`synopsis=${htmlToText(this.description)}`,
'-c:a',
'copy',
'-c:v',
'copy',
`${this.filePath}${this.multiPartSuffix(i)}.mp4`,
],
(error, stdout) => {
if (error !== null) reject(error);
else resolve(stdout);
}
)
)
)
);
this.expectedSize = await this.muxedBytes();
this.expectedSize = await this.fileBytes('mp4');
await this.markCompleted();
await fs.unlink(`${this.filePath}.partial`);
// Set the files update time to when the video was released
await fs.utimes(`${this.filePath}.mp4`, new Date(), this.releaseDate);
for (const i in this.videoAttachments) {
await fs.unlink(`${this.filePath}${this.multiPartSuffix(i)}.partial`);
// Set the files update time to when the video was released
await fs.utimes(`${this.filePath}${this.multiPartSuffix(i)}.mp4`, new Date(), this.releaseDate);
}
}

public async postProcessingCommand(): Promise<void> {
Expand Down

0 comments on commit dce7ad0

Please sign in to comment.