diff --git a/bin/web.js b/bin/web.js index ec43af9..98f1faa 100644 --- a/bin/web.js +++ b/bin/web.js @@ -23,6 +23,8 @@ const Express = require('express') const multer = require('multer') // body parsing middleware const bodyParser = require('body-parser') +// UUID Generator +const { v4 } = require('uuid') // Redis to store buckets and Kraken async requests IDs const redis = require('redis') @@ -219,10 +221,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // setup multer for file upload const localUpload = multer({ storage: multer.memoryStorage(), - limits: { - // maximum 2mb - fileSize: 2000000 - } + limits: { fileSize: 2000000 } }) const baseS3Options = { @@ -231,181 +230,114 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = } app.post(urls.upload, (req, res) => { - const { storeId } = req - const { bucket, host } = spaces[0] - logger.log(`${storeId} Uploading...`) - // unique object key - let key = '@v3/' + logger.log(`[storage-api] Upload start...`,) + // Validate file localUpload.single('file')(req, res, (err) => { if (err) { - const usrMsg = { + sendError(res, 400, 3001, err.message, { en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' - } - sendError(res, 400, 3001, err.message, usrMsg) + }) } else { + + // Retrieve request parameters + let key = '@v3/' + const { bucket, host } = spaces[0] + + // Retrieve local image data let dir = req.query.directory if (typeof dir === 'string' && dir.charAt(0) === '/') { - // remove first char to not duplicate first bar - // normalize, then remove empty paths dir = dir.substr(1).replace(/[^\w-/]/g, '').replace('//', '') if (dir.length) { key += dir.toLowerCase() + '/' } } - // keep filename + + // Keep filename const filename = req.file.originalname.replace(/[^\w-.]/g, '').toLowerCase() - key += `${Date.now()}-${filename}` - const { mimetype } = req.file - - runMethod('putObject', { - ...baseS3Options, - ContentType: mimetype, - Key: `${storeId}/${key}`, - Body: req.file.buffer - }) - // S3 Response - .then(() => { - logger.log(`${storeId} ${key} Uploaded to ${bucket}`) - // zoom uploaded - const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` - const uri = mountUri(key) - const picture = { zoom: { url: uri } } - const pictureBytes = {} - // resize/optimize image - let i = -1 - let transformedImageBody = null - - const respond = () => { - logger.log(`${storeId} ${key} ${bucket} All optimizations done`) - res.json({ bucket, key, uri, picture }) + key += `${v4()}-${filename}` + + // Aux methods + const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${req.storeId}/${key}` + let picture = {} + const respond = (suc = true) => { + if (suc) { + logger.log(`${req.storeId} ${key} ${bucket} All optimizations done`) + res.json({ bucket, key, uri: picture['zoom'] ? picture['zoom'].url : '' , picture }) + } else { + res.status(500).json({ message: { + en_us: 'An error ocourred whlie uploading your file.', + pt_br: 'Ocorreu um erro ao persistir seu arquivo' + }}) } + } - const callback = err => { - if (!err) { - // next image size - i++ - if (i < pictureOptims.length) { - let newKey - const { label, size, webp } = pictureOptims[i] - newKey = `imgs/${label}/${key}` - - const imageBuffer = i === 0 ? req.file.buffer : transformedImageBody - const imageBase64 = imageBuffer - ? `data:${mimetype};base64,${imageBuffer.toString('base64')}` - : null - // free memory - transformedImageBody = req.file = null - - setTimeout(() => { - - // Retrieve url - let originUrl - if (picture[label] && webp) { - originUrl = picture[label].url - } else { - originUrl = uri - } - - // Transform image updated to cloudflare - const transformImg = (isRetry = false) => { - - cloudflare(imageBase64 || originUrl, label === 'small' ? 'w90' : label, (err, data) => { - if (!err && data) { - const { id, url, imageBody } = data - return new Promise(resolve => { - // Cloudinary keeps image as webp, so we using webp as default - const contentType = 'image/webp' - const fileFormat = 'webp' - if (imageBody || id) { - const s3Options = { - ...baseS3Options, - ContentType: contentType, - Key: `${storeId}/${newKey}.${fileFormat}` - } - if (imageBody) { - transformedImageBody = imageBody - // PUT new image on S3 bucket - return runMethod('putObject', { ...s3Options, Body: imageBody }) - .then(() => resolve(mountUri(newKey))) - .catch((err) => { - logger.error(err) - resolve(url) - }) - } - // async handle with callback URL - redisClient.setex(genRedisKey(id, true), 600, JSON.stringify(s3Options)) - return resolve(mountUri(newKey)) - } - resolve(url) - - }).then(url => { - if (url && (!picture[label] || pictureBytes[label] > bytes)) { - picture[label] = { url, size } - pictureBytes[label] = bytes - } - callback() - }) - } - - if ( - err && - typeof err.message === 'string' && - (err.message.indexOf('504 Gateway Timeout') > -1 || err.message.indexOf('503 Service Unavailable') > -1) - ) { - if (!isRetry) { - return setTimeout(() => transformImg(true), 1000) - } else { - return respond() - } - } - callback(err) - }) - - } - - // Transofrm image - transformImg() - - }, imageBase64 ? 50 : 300) - } else { - setTimeout(() => { - // all done - respond() - }, 50) - } + // Upload to cloudiflare + cloudflare(req.file, (err, data) => { + + // No errors by the way + if (!err && data) { + + // Retrieve converted images + const { convertedImages } = data + let s3attempts = 0 + + // Map converted images and upload to S3 + convertedImages.forEach(({ id, label, imageBody }) => { + + const newKey = `imgs/${label}/${key}` + + // Upload image to s3 + return new Promise((resolve) => { + + // Define s3 options + const contentType = label === 'zoom' ? 'image/jpeg' : 'image/webp' + const fileFormat = label === 'zoom' ? 'jpg' : 'webp' + const s3Options = { + ...baseS3Options, + ContentType: contentType, + Key: `${req.storeId}/${newKey}.${fileFormat}` + } + + // Put s3 object + if (imageBody) { + return runMethod('putObject', { ...s3Options, Body: imageBody }) + .then(() => resolve(mountUri(newKey))) + .catch((err) => { + logger.error(err) + resolve(key) + }) + } + + // async handle with callback URL + redisClient.setex(genRedisKey(id, true), 600, JSON.stringify(s3Options)) + return resolve(mountUri(newKey)) + }) + .then((url) => { + s3attempts++ + if (url && (!picture[label])) { + picture[label] = { url: mountUri(url), size: label } + // pictureBytes[label] = bytes + } + if (s3attempts === 4 && Object.keys(picture).length === 4) { + return respond() + } else if (s3attempts === 4) { + return sendError(res, 500, 3002, 'Internal server error', { + en_us: 'An error ocourred whlie uploading your file.', + pt_br: 'Ocorreu um erro ao persistir seu arquivo' + }) + } + }) + }) - } else if (uri && typeof err.message === 'string' && err.message.indexOf('cloud_name') > -1) { - // image uploaded but not transformed - respond() - logger.error(err) - } else { - // respond with error - const usrMsg = { - en_us: 'Error while handling image, the file may be protected or corrupted', - pt_br: 'Erro ao manipular a imagem, o arquivo pode estar protegido ou corrompido' - } - sendError(res, 415, uri, err.message, usrMsg) - } + //return respond() } - switch (mimetype) { - case 'image/jpeg': - case 'image/png': - callback() - break - default: - respond() - } - }) - // CDN Upload error - .catch((err) => { - const usrMsg = { - en_us: 'This file cannot be uploaded to CDN', - pt_br: 'O arquivo não pôde ser carregado para o CDN' + // Sadness and sorrow + if (err) { + return respond(false) } - sendError(res, 400, 3002, err.message, usrMsg) }) } }) @@ -481,6 +413,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = }) app.listen(port, () => { + console.log('App is running') logger.log('Storage API running with Express on port ' + port) }) } diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index bfdd667..a7e69a4 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -15,51 +15,75 @@ module.exports = (auth) => { }) // Function to Compress image - return function (url, size = 'normal', __callback = () => {}) { - + return function (imageFile, __callback = () => {}) { // API Payload - let options = new FormData() - options.append('file', url) + try { + + const fData = new FormData() + fData.append('file', imageFile.buffer, { filename: imageFile.filename }) - // Force timeout with 20s - let callbackSent = false - const callback = (err, data) => { - if (!callbackSent) { - callbackSent = true - __callback(err, data) - } - if (timer) { - clearTimeout(timer) + // Force timeout with 20s + let callbackSent = false + const callback = (err, data) => { + if (!callbackSent) { + callbackSent = true + __callback(err, data) + } + if (timer) { + clearTimeout(timer) + } } - } - // Verify if responded in 20s - const timer = setTimeout(() => { - callback(new Error('Cloudflare optimization timed out')) - logger.log(`Cloudflare timed out`) - }, 20000) + // Verify if responded in 30s + const timer = setTimeout(() => { + callback(new Error('Cloudflare optimization timed out')) + logger.log(`Cloudflare timed out`) + }, 30000) + + // Upload image to cloudflare + cloudflareClient({ + method: 'POST', + url: '/images/v1', + headers: { ...fData.getHeaders() }, + data: fData + }).then((response) => { + // Retrieve Upload data + const id = response.data.result.id + const variants = response.data.result.variants + const fetchedImages = [] + let processed = 0 - // Upload image to cloudflare - cloudflareClient({ - method: 'POST', - url: '/images/v1', - headers: { ...options.getHeaders() }, - data: options - }).then((response) => { - const id = response.data.result.id - const variants = response.data.result.variants - const url = variants.find(v => v.endsWith(`/${size}`)) || variants[0] - if (url) { - download(url, { 'Accept': 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { - if (err) logger.error(err) - callback(err, { - ...response.data.result, - url: normalImage, - imageBody - }) - setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + // Download the correct variations + variants.forEach(vari => { + if (vari.includes('normal') || vari.includes('zoom') || vari.includes('big') || vari.includes('w90')) { + // Retrieve image type + const labels = vari.split('/') + const label = labels[labels.length - 1] + + // Download image, zoom must be jpeg + download(vari, { 'Accept': label === 'zoom' ? 'image/jpeg,image/*,*/*;q=0.8' : 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { + processed++ + if (err) logger.error(err) + fetchedImages.push({ id, imageBody, label }) + + // IF all images was fetched, then proceed + if (fetchedImages.length === 4 && processed === 4) { + callback(err, { convertedImages: fetchedImages }) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + } else if (processed === 4) { + callback(err, { convertedImages: fetchedImages, error: true }) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + } + }) + } }) - } - }).catch((err) => callback(err)) + }).catch((err) => { + callback(err) + }) + + } catch (err) { + __callback(err, null) + } + } } diff --git a/lib/Logger.js b/lib/Logger.js index 89298fc..0e9ed93 100644 --- a/lib/Logger.js +++ b/lib/Logger.js @@ -40,8 +40,10 @@ function middleware (out, desc, conn, _obj, callback) { const fs = require('fs') // log files -const output = fs.createWriteStream('/var/log/nodejs/storage.out') -const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') +// const output = fs.createWriteStream('/var/log/nodejs/storage.out') +// const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') +const output = fs.createWriteStream('/home/gmazurco/storage.out') +const outputErrors = fs.createWriteStream('/home/gmazurco/storage.error') // declares logger with Console class of global console const logger = new console.Console(output, outputErrors) diff --git a/main.js b/main.js index 7a1fbef..2c5e823 100644 --- a/main.js +++ b/main.js @@ -23,10 +23,10 @@ process.on('uncaughtException', (err) => { msg += '\n' } - let fs = require('fs') - fs.appendFile('/var/log/nodejs/_stderr', msg, () => { - process.exit(1) - }) + // let fs = require('fs') + // fs.appendFile('/var/log/nodejs/_stderr', msg, () => { + // process.exit(1) + // }) }) // web application diff --git a/package-lock.json b/package-lock.json index be54ca7..890ca5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -671,6 +671,13 @@ "url": "0.10.3", "uuid": "3.3.2", "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } } }, "axios": { @@ -5404,9 +5411,9 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, "v8-compile-cache": { "version": "2.3.0", diff --git a/package.json b/package.json index 4ad9dba..702a0a4 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "express": "^4.17.1", "form-data": "^4.0.0", "multer": "^1.4.2", - "redis": "^3.1.2" + "redis": "^3.1.2", + "uuid": "^8.3.2" }, "devDependencies": { "@commitlint/cli": "^13.1.0",