384 changes: 117 additions & 267 deletions packages/core/upload/server/services/upload.js

Large diffs are not rendered by default.

50 changes: 50 additions & 0 deletions packages/core/upload/server/utils/file.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
'use strict';

const os = require('os');
const path = require('path');
const fse = require('fs-extra');
const crypto = require('crypto');
const { fromStream } = require('file-type');
const { nameToSlug } = require('@strapi/utils');

/**
* Get the file type from a file stream.
* @param {File} file
* @returns {Promise<string>}
*/
const getFileType = async (file) => {
const fileType = await fromStream(file.getStream());
return fileType?.ext;
};

/**
* Generate a random file name based on the original file name.
*/
const generateFileName = (name) => {
const baseName = nameToSlug(name, { separator: '_', lowercase: false });
const randomSuffix = crypto.randomBytes(5).toString('hex');
return `${baseName}_${randomSuffix}`;
};

/**
* Creates a temporary directory and deletes it after the callback is executed.
* @param {*} callback
* @returns
*/
async function withTempDirectory(callback) {
const folderPath = path.join(os.tmpdir(), 'strapi-upload-');
const folder = await fse.mkdtemp(folderPath);

try {
const res = await callback(folder);
return res;
} finally {
await fse.remove(folder);
}
}

module.exports = {
getFileType,
generateFileName,
withTempDirectory,
};
53 changes: 53 additions & 0 deletions packages/core/upload/server/utils/media-builder/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
'use strict';

const { mapAsync, reduceAsync } = require('@strapi/utils');

const MediaBuilder = () => {
const transformations = new Map();

return {
transformOn(key, fileTypes, transforms) {
transformations.set(key, { fileTypes, transforms });
return this;
},

deleteTransform(key) {
transformations.delete(key);
return this;
},

async transform(file) {
// Get all transformations for the given file extension
const _transformations = Array.from(transformations.values())
.filter(({ fileTypes }) => fileTypes.includes(file.type))
.flatMap(({ transforms }) => transforms);

return reduceAsync(
_transformations,
(files, transformation) => transformFiles(files, transformation),
[file]
);
},

groupByFormats(transformedFiles, srcFile) {
// Merge files into one
const file = transformedFiles.find((file) => !file.format) || srcFile;
const formattedFiles = transformedFiles.filter((file) => !!file.format);

// Add formatted files into the original file
file.formats = {};
for (const formattedFile of formattedFiles) {
file.formats[formattedFile.format] = formattedFile;
}
return file;
},
};
};

const transformFiles = async (files, transformation) => {
const transformedFiles = await mapAsync(files, transformation);
// Some transformations might return multiple files (e.g. resize)
return transformedFiles.flat();
};

module.exports = MediaBuilder;
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
'use strict';

const sharp = require('sharp');

const autoRotate = async (file) => {
return {
...file,
getStream: () => file.getStream().pipe(sharp().rotate()),
};
};

module.exports = autoRotate;
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
'use strict';

const sharp = require('sharp');

const DEFAULT_BREAKPOINTS = { large: 1000, medium: 750, small: 500 };
const getBreakpoints = () => strapi.config.get('plugin.upload.breakpoints', DEFAULT_BREAKPOINTS);

/**
* Resize image to fit within the specified dimensions,
* but only if the image is larger. If the image is smaller, it is not resized.
*/
const calculateInlineResizing = (size, { width, height }) => {
// No need to resize if the image is smaller
if (size > width && size > height) {
return undefined;
}

let newWidth = size;
let newHeight = size;

// Adjust the newWidth and height to maintain aspect ratio
if (width > height) {
newHeight = Math.round((height / width) * size);
} else {
newWidth = Math.round((width / height) * size);
}

return { width: newWidth, height: newHeight };
};

const breakpoints = async (file) => {
// Only resize original image and not other responsive formats (e.g thumbnail)
if (file.format) {
return [file];
}

const breakpoints = getBreakpoints();
const files = [file];

for (const [format, breakpoint] of Object.entries(breakpoints)) {
const resize = calculateInlineResizing(breakpoint, file);

if (resize) {
files.push({
...file,
name: `${format}_${file.name}`,
hash: `${format}_${file.hash}`,
format,
width: resize.width,
height: resize.height,
getStream: () => file.getStream().pipe(sharp().resize(resize.width, resize.height)),
});
}
}

return files;
};

module.exports = breakpoints;
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
'use strict';

const sharp = require('sharp');
const autoRotate = require('./auto-rotate');
const breakpoints = require('./breakpoints');
const metadata = require('./metadata');
const optimize = require('./optimize');
const thumbnail = require('./thumbnail');

// set all necessary sharp options here to reduce ram usage
sharp.concurrency(1);
sharp.cache(false);
// set VIPS_DISC_THRESHOLD to 50 megs
// VIPS_DISC_THRESHOLD=50m

module.exports = {
autoRotate,
breakpoints,
metadata,
optimize,
thumbnail,
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
'use strict';

const sharp = require('sharp');

const getMetadata = (file) =>
new Promise((resolve, reject) => {
const pipeline = sharp();
pipeline.metadata().then(resolve).catch(reject);
file.getStream().pipe(pipeline);
});

const metadata = async (file) => {
const metadata = await getMetadata(file);

return {
...file,
width: metadata.width,
height: metadata.height,
};
};

module.exports = metadata;
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
'use strict';

const sharp = require('sharp');

const optimize = async (file) => {
return {
...file,
getStream() {
const stream = file.getStream();
if (sharp()[file?.type]) {
stream.pipe(sharp()[file.type]({ quality: 80 }));
}
return stream;
},
};
};

module.exports = optimize;
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
'use strict';

const sharp = require('sharp');

const THUMBNAIL_SIZE = { width: 245, height: 156 };

const calculateInsideResizing = (srcSize, destSize) => {
// No need to resize if the image is smaller than the thumbnail size
if (srcSize.width < destSize.width && srcSize.height < destSize.height) {
return;
}

const srcAspectRatio = srcSize.width / srcSize.height;
const destAspectRatio = destSize.width / destSize.height;

let width = destSize.width;
let height = destSize.height;

if (srcAspectRatio > destAspectRatio) {
height = Math.round(destSize.width / srcAspectRatio);
} else {
width = Math.round(destSize.height * srcAspectRatio);
}

return { width, height };
};

const thumbnail = async (file) => {
// Only resize original image and not other responsive formats (e.g breakpoints)
if (file.format) {
return [file];
}

const files = [file];

const resize = calculateInsideResizing(file, THUMBNAIL_SIZE);

if (resize) {
files.push({
...file,
name: `thumbnail_${file.name}`,
hash: `thumbnail_${file.hash}`,
format: 'thumbnail',
width: resize.width,
height: resize.height,
getStream: () => file.getStream().pipe(sharp().resize(resize.width, resize.height)),
});
}

return files;
};

module.exports = thumbnail;
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
'use strict';

const { ThrottleGroup } = require('stream-throttle');

const THROTTLE_MB_MULTIPLIER = 1024 * 1024;

const tg = new ThrottleGroup({
// Rate in bytes per second e.g 10240 = 10KB/s.
rate: 100 * THROTTLE_MB_MULTIPLIER, // 100 MB/s
});

const throttle = async (file) => {
return {
...file,
getStream: () => file.getStream().pipe(tg.throttle()),
};
};

module.exports = throttle;
63 changes: 61 additions & 2 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -8472,6 +8472,7 @@ __metadata:
byte-size: 7.0.1
cropperjs: 1.5.12
date-fns: 2.30.0
file-type: ^16.5.4
formik: 2.4.0
fs-extra: 10.0.0
immer: 9.0.19
Expand All @@ -8492,6 +8493,7 @@ __metadata:
react-router-dom: 5.3.4
react-select: 5.7.0
sharp: 0.32.0
stream-throttle: ^0.1.3
styled-components: 5.3.3
yup: ^0.32.9
peerDependencies:
Expand Down Expand Up @@ -14173,7 +14175,7 @@ __metadata:
languageName: node
linkType: hard

"commander@npm:^2.19.0, commander@npm:^2.20.0, commander@npm:^2.20.3":
"commander@npm:^2.19.0, commander@npm:^2.2.0, commander@npm:^2.20.0, commander@npm:^2.20.3":
version: 2.20.3
resolution: "commander@npm:2.20.3"
checksum: ab8c07884e42c3a8dbc5dd9592c606176c7eb5c1ca5ff274bcf907039b2c41de3626f684ea75ccf4d361ba004bbaff1f577d5384c155f3871e456bdf27becf9e
Expand Down Expand Up @@ -17723,6 +17725,17 @@ __metadata:
languageName: node
linkType: hard

"file-type@npm:^16.5.4":
version: 16.5.4
resolution: "file-type@npm:16.5.4"
dependencies:
readable-web-to-node-stream: ^3.0.0
strtok3: ^6.2.4
token-types: ^4.1.1
checksum: d983c0f36491c57fcb6cc70fcb02c36d6b53f312a15053263e1924e28ca8314adf0db32170801ad777f09432c32155f31715ceaee66310947731588120d7ec27
languageName: node
linkType: hard

"file-type@npm:^17.1.6":
version: 17.1.6
resolution: "file-type@npm:17.1.6"
Expand Down Expand Up @@ -23323,6 +23336,13 @@ __metadata:
languageName: node
linkType: hard

"limiter@npm:^1.0.5":
version: 1.1.5
resolution: "limiter@npm:1.1.5"
checksum: 2d51d3a8bef131aada820b76530f8223380a0079aa0fffdfd3ec47ac2f65763225cb4c62a2f22347f4898c5eeb248edfec991c4a4f5b608dfca0aaa37ac48071
languageName: node
linkType: hard

"lines-and-columns@npm:^1.1.6":
version: 1.2.4
resolution: "lines-and-columns@npm:1.2.4"
Expand Down Expand Up @@ -26948,6 +26968,13 @@ __metadata:
languageName: node
linkType: hard

"peek-readable@npm:^4.1.0":
version: 4.1.0
resolution: "peek-readable@npm:4.1.0"
checksum: 02c673f9bc816f8e4e74a054c097225ad38d457d745b775e2b96faf404a54473b2f62f5bcd496f5ebc28696708bcc5e95bed409856f4bef5ed62eae9b4ac0dab
languageName: node
linkType: hard

"peek-readable@npm:^5.0.0":
version: 5.0.0
resolution: "peek-readable@npm:5.0.0"
Expand Down Expand Up @@ -28633,7 +28660,7 @@ __metadata:
languageName: node
linkType: hard

"readable-web-to-node-stream@npm:^3.0.2":
"readable-web-to-node-stream@npm:^3.0.0, readable-web-to-node-stream@npm:^3.0.2":
version: 3.0.2
resolution: "readable-web-to-node-stream@npm:3.0.2"
dependencies:
Expand Down Expand Up @@ -30755,6 +30782,18 @@ __metadata:
languageName: node
linkType: hard

"stream-throttle@npm:^0.1.3":
version: 0.1.3
resolution: "stream-throttle@npm:0.1.3"
dependencies:
commander: ^2.2.0
limiter: ^1.0.5
bin:
throttleproxy: ./bin/throttleproxy.js
checksum: 93d870b37266e61753c2d0c1227cf4c7bef3562b0d018291b4ccc1fe7063041a04ec165f2dcfe6f1b9dfb749fecb58abd34377b10cd793277eff3a652695831b
languageName: node
linkType: hard

"streamsearch@npm:0.1.2":
version: 0.1.2
resolution: "streamsearch@npm:0.1.2"
Expand Down Expand Up @@ -31095,6 +31134,16 @@ __metadata:
languageName: node
linkType: hard

"strtok3@npm:^6.2.4":
version: 6.3.0
resolution: "strtok3@npm:6.3.0"
dependencies:
"@tokenizer/token": ^0.3.0
peek-readable: ^4.1.0
checksum: 90732cff3f325aef7c47c511f609b593e0873ec77b5081810071cde941344e6a0ee3ccb0cae1a9f5b4e12c81a2546fd6b322fabcdfbd1dd08362c2ce5291334a
languageName: node
linkType: hard

"strtok3@npm:^7.0.0-alpha.9":
version: 7.0.0
resolution: "strtok3@npm:7.0.0"
Expand Down Expand Up @@ -31894,6 +31943,16 @@ __metadata:
languageName: node
linkType: hard

"token-types@npm:^4.1.1":
version: 4.2.1
resolution: "token-types@npm:4.2.1"
dependencies:
"@tokenizer/token": ^0.3.0
ieee754: ^1.2.1
checksum: cce256766b33e0f08ceffefa2198fb4961a417866d00780e58625999ab5c0699821407053e64eadc41b00bbb6c0d0c4d02fbd2199940d8a3ccb71e1b148ab9a2
languageName: node
linkType: hard

"token-types@npm:^5.0.0-alpha.2":
version: 5.0.1
resolution: "token-types@npm:5.0.1"
Expand Down