From daaa14431eb0e64db78ba43cf712c72443eb5d12 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Thu, 27 Jan 2022 18:09:01 -0300 Subject: [PATCH 01/19] feat: creating cloudflare adapter --- README.md | 2 +- bin/web.js | 6 + lib/Cloudflare.js | 67 ++++ package-lock.json | 841 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 5 + 5 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 lib/Cloudflare.js diff --git a/README.md b/README.md index 1c454ef..6689e19 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Edit `config.json` placing correct values for your environment, after that, start app with node: ```bash -node ./main.js + node ./main.js ``` # Web server diff --git a/bin/web.js b/bin/web.js index 0a8ae63..2a710f0 100644 --- a/bin/web.js +++ b/bin/web.js @@ -8,6 +8,8 @@ const auth = require('./../lib/Auth') const Aws = require('./../lib/Aws') // Cloudinary API abstraction const Cloudinary = require('./../lib/Cloudinary') +// Cloudflare API abstraxtion +const CloudFlare = require('./../lib/Cloudflare') // download image from Kraken temporary CDN const download = require('./../lib/Download') @@ -39,6 +41,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = baseUri, doSpace, cloudinaryAuth, + cloudflareAuth, cdnHost, pictureSizes } = JSON.parse(data) @@ -102,6 +105,9 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // setup Cloudinary client const cloudinary = Cloudinary(cloudinaryAuth) + // setup Cloudflare client + const cloudflare = CloudFlare(cloudflareAuth) + const sendError = (res, status, code, devMsg, usrMsg) => { if (!devMsg) { devMsg = 'Unknow error' diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js new file mode 100644 index 0000000..e065d15 --- /dev/null +++ b/lib/Cloudflare.js @@ -0,0 +1,67 @@ +'use strict' + +// Dependencies +const axios = require('axios').default +const FormData = require('Form-data') +const download = require('./Download') + +// Cloudflare module +module.exports = (auth) => { + + // Setup axios client with api key + const cloudinaryClient = axios.create({ + baseURL: `https://api.cloudflare.com/client/v4/accounts/${auth.accountId}`, + headers: { Authorization: `Bearer ${auth.apiKey}` } + }) + + // Function to Compress image + return function (url, size, webpCompression, __callback) { + + // API Payload + let options = new FormData() + options.append('file', 'my_file') + + // Log cloudflare image upload + logger.log(`[Cloudflare] Image upload:`) + + // Force timeout with 20s + let callbackSent = false + const callback = (err, data) => { + if (!callbackSent) { + callbackSent = true + __callback(err, data) + } + if (timer) { + clearTimeout(timer) + } + } + + // Verify if responded in 20s + const timer = setTimeout(() => { + callback(new Error('Cloudflare optimization timed out')) + logger.log(`Cloudflare timed out`) + }, 20000) + + // Upload image to cloudflare + cloudinaryClient({ + method: 'POST', + url: '/images/v1', + headers: { ...options.getHeaders() }, + data: options + }).then((response) => { + const variants = response.data?.result?.variants + const variant = variants && variants.length ? variants.length > 1 ? variants[1] : variants[0] : null + if (variant) { + download(`${variant}/normal`, (err, imageBody) => { + if (err) logger.error(err) + callback(err, { variant, imageBody }) + setTimeout(() => { + // destroy on Cloudflare just to save storage + + }, 60000) + }) + } + }).catch((err) => callback(err)) + + } +} diff --git a/package-lock.json b/package-lock.json index 280f504..be54ca7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -445,6 +445,21 @@ "integrity": "sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==", "dev": true }, + "@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true + }, + "@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, + "requires": { + "defer-to-connect": "^1.0.1" + } + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -479,6 +494,12 @@ "through": ">=2.2.7 <3" } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -518,6 +539,15 @@ "uri-js": "^4.2.2" } }, + "ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "dev": true, + "requires": { + "string-width": "^4.1.0" + } + }, "ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -539,6 +569,16 @@ "color-convert": "^1.9.0" } }, + "anymatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, "append-field": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", @@ -612,6 +652,11 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, "aws-sdk": { "version": "2.937.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.937.0.tgz", @@ -628,6 +673,14 @@ "xml2js": "0.4.19" } }, + "axios": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz", + "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", + "requires": { + "follow-redirects": "^1.14.7" + } + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -639,6 +692,12 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true + }, "body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", @@ -671,6 +730,85 @@ } } }, + "boxen": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-5.1.2.tgz", + "integrity": "sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ==", + "dev": true, + "requires": { + "ansi-align": "^3.0.0", + "camelcase": "^6.2.0", + "chalk": "^4.1.0", + "cli-boxes": "^2.2.1", + "string-width": "^4.2.2", + "type-fest": "^0.20.2", + "widest-line": "^3.1.0", + "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true + } + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -681,6 +819,15 @@ "concat-map": "0.0.1" } }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, "buffer": { "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", @@ -733,6 +880,38 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dev": true, + "requires": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "dependencies": { + "get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true + } + } + }, "call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -777,12 +956,34 @@ "supports-color": "^5.3.0" } }, + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + } + }, "ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", "dev": true }, + "cli-boxes": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", + "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==", + "dev": true + }, "cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -794,6 +995,15 @@ "wrap-ansi": "^7.0.0" } }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + } + }, "cloudinary": { "version": "1.26.1", "resolved": "https://registry.npmjs.org/cloudinary/-/cloudinary-1.26.1.tgz", @@ -832,6 +1042,14 @@ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "dev": true }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "compare-func": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-2.0.0.tgz", @@ -866,6 +1084,20 @@ "typedarray": "^0.0.6" } }, + "configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dev": true, + "requires": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + } + }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -1295,6 +1527,12 @@ "which": "^2.0.1" } }, + "crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "dev": true + }, "dargs": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/dargs/-/dargs-7.0.0.tgz", @@ -1340,12 +1578,33 @@ } } }, + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==", + "dev": true + }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -1355,6 +1614,11 @@ "object-keys": "^1.0.12" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, "denque": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", @@ -1478,6 +1742,12 @@ } } }, + "duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "dev": true + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1494,6 +1764,15 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, "enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -1555,6 +1834,12 @@ "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", "dev": true }, + "escape-goat": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==", + "dev": true + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -2220,6 +2505,15 @@ "flat-cache": "^2.0.1" } }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, "finalhandler": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", @@ -2285,6 +2579,21 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, + "follow-redirects": { + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==" + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -2321,6 +2630,13 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -2540,6 +2856,36 @@ } } }, + "got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dev": true, + "requires": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + }, + "dependencies": { + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + } + } + }, "graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", @@ -2601,6 +2947,12 @@ "has-symbols": "^1.0.2" } }, + "has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "dev": true + }, "hosted-git-info": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.0.2.tgz", @@ -2610,6 +2962,12 @@ "lru-cache": "^6.0.0" } }, + "http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -2723,6 +3081,12 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=", + "dev": true + }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -2741,6 +3105,12 @@ } } }, + "import-lazy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", + "integrity": "sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=", + "dev": true + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -2805,6 +3175,15 @@ "has-bigints": "^1.0.1" } }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, "is-boolean-object": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", @@ -2821,6 +3200,15 @@ "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", "dev": true }, + "is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "requires": { + "ci-info": "^2.0.0" + } + }, "is-core-module": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz", @@ -2860,12 +3248,51 @@ "is-extglob": "^2.1.1" } }, + "is-installed-globally": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz", + "integrity": "sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==", + "dev": true, + "requires": { + "global-dirs": "^3.0.0", + "is-path-inside": "^3.0.2" + }, + "dependencies": { + "global-dirs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.0.tgz", + "integrity": "sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA==", + "dev": true, + "requires": { + "ini": "2.0.0" + } + }, + "ini": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", + "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", + "dev": true + } + } + }, "is-negative-zero": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", "dev": true }, + "is-npm": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-5.0.0.tgz", + "integrity": "sha512-WW/rQLOazUq+ST/bCAVBp/2oMERWLsR7OrKyt052dNDk4DHcDE0/7QSXITlmi+VBcV13DfIbysG3tZJm5RfdBA==", + "dev": true + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, "is-number-object": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", @@ -2881,6 +3308,12 @@ "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", "dev": true }, + "is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true + }, "is-plain-obj": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", @@ -2930,6 +3363,18 @@ "text-extensions": "^1.0.0" } }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==", + "dev": true + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -2962,6 +3407,12 @@ "esprima": "^4.0.0" } }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=", + "dev": true + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -3027,12 +3478,30 @@ "object.assign": "^4.1.2" } }, + "keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, + "requires": { + "json-buffer": "3.0.0" + } + }, "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, + "latest-version": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "dev": true, + "requires": { + "package-json": "^6.3.0" + } + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -3101,6 +3570,12 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true + }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -3110,6 +3585,23 @@ "yallist": "^4.0.0" } }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "map-obj": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.2.1.tgz", @@ -3180,6 +3672,12 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true + }, "min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", @@ -3311,6 +3809,50 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, + "nodemon": { + "version": "2.0.15", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.15.tgz", + "integrity": "sha512-gdHMNx47Gw7b3kWxJV64NI+Q5nfl0y5DgDbiVtShiwa7Z0IZ07Ll4RLFo6AjrhzMtoEZn5PDE3/c2AbVsiCkpA==", + "dev": true, + "requires": { + "chokidar": "^3.5.2", + "debug": "^3.2.7", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.0.4", + "pstree.remy": "^1.1.8", + "semver": "^5.7.1", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5", + "update-notifier": "^5.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, + "nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=", + "dev": true, + "requires": { + "abbrev": "1" + } + }, "normalize-package-data": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", @@ -3323,6 +3865,18 @@ "validate-npm-package-license": "^3.0.1" } }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, + "normalize-url": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==", + "dev": true + }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -3447,6 +4001,12 @@ "word-wrap": "^1.2.3" } }, + "p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true + }, "p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -3471,6 +4031,26 @@ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, + "package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "dev": true, + "requires": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -3535,6 +4115,12 @@ "pify": "^2.0.0" } }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true + }, "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -3691,6 +4277,12 @@ "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true }, + "prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", + "dev": true + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -3722,12 +4314,37 @@ "ipaddr.js": "1.9.0" } }, + "pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "pupa": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.1.1.tgz", + "integrity": "sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A==", + "dev": true, + "requires": { + "escape-goat": "^2.0.0" + } + }, "q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", @@ -3765,6 +4382,26 @@ "unpipe": "1.0.0" } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + } + } + }, "react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", @@ -3845,6 +4482,15 @@ "util-deprecate": "^1.0.1" } }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, "redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -3900,6 +4546,24 @@ "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", "dev": true }, + "registry-auth-token": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.1.tgz", + "integrity": "sha512-6gkSb4U6aWJB4SF2ZvLb76yCBjcvufXBqvvEx1HbmKPkutswjW1xNVRY0+daljIYRbogN7O0etYSlbiaEQyMyw==", + "dev": true, + "requires": { + "rc": "^1.2.8" + } + }, + "registry-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "dev": true, + "requires": { + "rc": "^1.2.8" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3931,6 +4595,15 @@ "global-dirs": "^0.1.1" } }, + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "dev": true, + "requires": { + "lowercase-keys": "^1.0.0" + } + }, "rimraf": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", @@ -3971,6 +4644,23 @@ "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", "dev": true }, + "semver-diff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "dev": true, + "requires": { + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "semver-regex": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-3.1.2.tgz", @@ -4466,11 +5156,35 @@ "readable-stream": "3" } }, + "to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "touch": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", + "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, + "requires": { + "nopt": "~1.0.10" + } + }, "trim-newlines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz", @@ -4518,6 +5232,15 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "requires": { + "is-typedarray": "^1.0.0" + } + }, "uglify-js": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.14.2.tgz", @@ -4537,6 +5260,21 @@ "which-boxed-primitive": "^1.0.2" } }, + "undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true + }, + "unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dev": true, + "requires": { + "crypto-random-string": "^2.0.0" + } + }, "universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", @@ -4548,6 +5286,79 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, + "update-notifier": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-5.1.0.tgz", + "integrity": "sha512-ItnICHbeMh9GqUy31hFPrD1kcuZ3rpxDZbf4KUDavXwS0bW5m7SLbDQpGX3UYr072cbrF5hFUs3r5tUsPwjfHw==", + "dev": true, + "requires": { + "boxen": "^5.0.0", + "chalk": "^4.1.0", + "configstore": "^5.0.1", + "has-yarn": "^2.1.0", + "import-lazy": "^2.1.0", + "is-ci": "^2.0.0", + "is-installed-globally": "^0.4.0", + "is-npm": "^5.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.1.0", + "pupa": "^2.1.1", + "semver": "^7.3.4", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -4573,6 +5384,15 @@ } } }, + "url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "dev": true, + "requires": { + "prepend-http": "^2.0.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -4637,6 +5457,15 @@ "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", "dev": true }, + "widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "dev": true, + "requires": { + "string-width": "^4.0.0" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -4701,6 +5530,18 @@ "mkdirp": "^0.5.1" } }, + "write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "requires": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, "xdg-basedir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", diff --git a/package.json b/package.json index 65ca6c3..4ad9dba 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "description": "E-Com Plus Storage API Node.js Express App", "main": "main.js", "scripts": { + "start": "node ./main.js", + "start:dev": "nodemon ./main.js", "test": "node ./main.js", "release": "standard-version" }, @@ -12,9 +14,11 @@ "private": true, "dependencies": { "aws-sdk": "^2.937.0", + "axios": "^0.25.0", "body-parser": "^1.19.0", "cloudinary": "^1.26.1", "express": "^4.17.1", + "form-data": "^4.0.0", "multer": "^1.4.2", "redis": "^3.1.2" }, @@ -22,6 +26,7 @@ "@commitlint/cli": "^13.1.0", "@commitlint/config-conventional": "^13.1.0", "husky": "^4.3.8", + "nodemon": "^2.0.15", "standard": "^16.0.3", "standard-version": "^9.3.1" }, From 99d8ca4c3af4042e8adc9193c7d8c235abacdb7e Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Fri, 28 Jan 2022 10:45:42 -0300 Subject: [PATCH 02/19] feat: cloudflare integration --- .gitignore | 1 + bin/web.js | 14 +++++++++++--- lib/Cloudflare.js | 12 +++++++----- lib/Logger.js | 6 ++++-- main.js | 11 +++++++---- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 621038f..bf3ff7f 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ typings/ # dotenv environment variables file .env +config.json diff --git a/bin/web.js b/bin/web.js index 2a710f0..eafd1e1 100644 --- a/bin/web.js +++ b/bin/web.js @@ -279,9 +279,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // zoom uploaded const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` const uri = mountUri(key) - const picture = { - zoom: { url: uri } - } + const picture = { zoom: { url: uri } } const pictureBytes = {} // resize/optimize image let i = -1 @@ -326,6 +324,16 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = } const transformImg = (isRetry = false) => { + cloudflare(imageBase64 || originUrl, fixSize && size, webp, (err, data) => { + if (!err && data) { + const { id, variants, filename, imageBody } = data + } + }) + + + + + cloudinary(imageBase64 || originUrl, fixSize && size, webp, (err, data) => { if (!err && data) { const { id, format, url, bytes, imageBody } = data diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index e065d15..5989085 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -2,7 +2,7 @@ // Dependencies const axios = require('axios').default -const FormData = require('Form-data') +const FormData = require('form-data') const download = require('./Download') // Cloudflare module @@ -19,7 +19,7 @@ module.exports = (auth) => { // API Payload let options = new FormData() - options.append('file', 'my_file') + options.append('file', url) // Log cloudflare image upload logger.log(`[Cloudflare] Image upload:`) @@ -49,15 +49,17 @@ module.exports = (auth) => { headers: { ...options.getHeaders() }, data: options }).then((response) => { - const variants = response.data?.result?.variants + const id = response.data.id + const variants = response.data.result.variants const variant = variants && variants.length ? variants.length > 1 ? variants[1] : variants[0] : null + console.log('[1]', response.data) if (variant) { download(`${variant}/normal`, (err, imageBody) => { if (err) logger.error(err) - callback(err, { variant, imageBody }) + callback(err, { variant, imageBody, ...response.data.result }) setTimeout(() => { // destroy on Cloudflare just to save storage - + cloudinaryClient.delete(`/images/v1/${id}`) }, 60000) }) } diff --git a/lib/Logger.js b/lib/Logger.js index 89298fc..0e9ed93 100644 --- a/lib/Logger.js +++ b/lib/Logger.js @@ -40,8 +40,10 @@ function middleware (out, desc, conn, _obj, callback) { const fs = require('fs') // log files -const output = fs.createWriteStream('/var/log/nodejs/storage.out') -const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') +// const output = fs.createWriteStream('/var/log/nodejs/storage.out') +// const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') +const output = fs.createWriteStream('/home/gmazurco/storage.out') +const outputErrors = fs.createWriteStream('/home/gmazurco/storage.error') // declares logger with Console class of global console const logger = new console.Console(output, outputErrors) diff --git a/main.js b/main.js index 7a1fbef..09d0a7d 100644 --- a/main.js +++ b/main.js @@ -23,10 +23,13 @@ process.on('uncaughtException', (err) => { msg += '\n' } - let fs = require('fs') - fs.appendFile('/var/log/nodejs/_stderr', msg, () => { - process.exit(1) - }) + console.log('here', err) + + //! Restore after commit + // let fs = require('fs') + // fs.appendFile('/var/log/nodejs/_stderr', msg, () => { + // process.exit(1) + // }) }) // web application From 7363dce58f7786e8cd8579d3d91162dd27251a3b Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Fri, 28 Jan 2022 15:40:08 -0300 Subject: [PATCH 03/19] feat: cloudflare integration --- bin/web.js | 226 ++++++++++++++++++++-------------------------- lib/Cloudflare.js | 26 +++--- lib/Logger.js | 6 +- main.js | 11 +-- 4 files changed, 114 insertions(+), 155 deletions(-) diff --git a/bin/web.js b/bin/web.js index eafd1e1..785557a 100644 --- a/bin/web.js +++ b/bin/web.js @@ -6,8 +6,6 @@ const logger = require('./../lib/Logger') const auth = require('./../lib/Auth') // AWS SDK API abstraction const Aws = require('./../lib/Aws') -// Cloudinary API abstraction -const Cloudinary = require('./../lib/Cloudinary') // Cloudflare API abstraxtion const CloudFlare = require('./../lib/Cloudflare') // download image from Kraken temporary CDN @@ -40,7 +38,6 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // hostname, baseUri, doSpace, - cloudinaryAuth, cloudflareAuth, cdnHost, pictureSizes @@ -102,9 +99,6 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = return run(spaces[0]) } - // setup Cloudinary client - const cloudinary = Cloudinary(cloudinaryAuth) - // setup Cloudflare client const cloudflare = CloudFlare(cloudflareAuth) @@ -247,6 +241,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = let key = '@v3/' localUpload.single('file')(req, res, (err) => { + console.log('Upload single', err) if (err) { const usrMsg = { en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', @@ -274,89 +269,60 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = Key: `${storeId}/${key}`, Body: req.file.buffer }) - .then(() => { - logger.log(`${storeId} ${key} Uploaded to ${bucket}`) - // zoom uploaded - const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` - const uri = mountUri(key) - const picture = { zoom: { url: uri } } - const pictureBytes = {} - // resize/optimize image - let i = -1 - let transformedImageBody = null - - const respond = () => { - logger.log(`${storeId} ${key} ${bucket} All optimizations done`) - res.json({ - bucket, - key, - // return complete object URL - uri, - picture - }) - } - - const callback = err => { - if (!err) { - // next image size - i++ - if (i < pictureOptims.length) { - let newKey - const { label, size, webp } = pictureOptims[i] - newKey = `imgs/${label}/${key}` - - const imageBuffer = i === 0 ? req.file.buffer : transformedImageBody - const imageBase64 = imageBuffer - ? `data:${mimetype};base64,${imageBuffer.toString('base64')}` - : null - // free memory - transformedImageBody = req.file = null - - setTimeout(() => { - // image resize/optimization with Cloudinary - let fixSize, originUrl - if (picture[label] && webp) { - fixSize = false - originUrl = picture[label].url - } else { - fixSize = true - originUrl = uri - } - - const transformImg = (isRetry = false) => { - cloudflare(imageBase64 || originUrl, fixSize && size, webp, (err, data) => { - if (!err && data) { - const { id, variants, filename, imageBody } = data - } - }) - - - - + // S3 Response + .then(() => { + logger.log(`${storeId} ${key} Uploaded to ${bucket}`) + // zoom uploaded + const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` + const uri = mountUri(key) + const picture = { zoom: { url: uri } } + const pictureBytes = {} + // resize/optimize image + let i = -1 + let transformedImageBody = null + + const respond = () => { + logger.log(`${storeId} ${key} ${bucket} All optimizations done`) + res.json({ bucket, key, uri, picture }) + } - cloudinary(imageBase64 || originUrl, fixSize && size, webp, (err, data) => { + const callback = err => { + if (!err) { + // next image size + i++ + if (i < pictureOptims.length) { + let newKey + const { label, size, webp } = pictureOptims[i] + newKey = `imgs/${label}/${key}` + + const imageBuffer = i === 0 ? req.file.buffer : transformedImageBody + const imageBase64 = imageBuffer + ? `data:${mimetype};base64,${imageBuffer.toString('base64')}` + : null + // free memory + transformedImageBody = req.file = null + + setTimeout(() => { + + // Retrieve url + let originUrl + if (picture[label] && webp) { + originUrl = picture[label].url + } else { + originUrl = uri + } + + // Transform image updated to cloudflare + const transformImg = (isRetry = false) => { + + cloudflare(imageBase64 || originUrl, (err, data) => { if (!err && data) { - const { id, format, url, bytes, imageBody } = data + const { id, url, filename, imageBody } = data return new Promise(resolve => { + let contentType - if (webp) { - // fix filepath extension and content type header - if (format) { - if (!newKey.endsWith(format)) { - // converted to best optim format - newKey += `.${format}` - } - contentType = format === 'jpg' ? 'image/jpeg' : `image/${format}` - } else { - // converted to WebP - newKey += '.webp' - contentType = 'image/webp' - } - } else { - contentType = mimetype - } - + contentType = `image/${filename.includes('.png') ? 'png' : filename.includes('.webp') ? 'webp' : 'jpeg'}` if (imageBody || id) { const s3Options = { ...baseS3Options, @@ -378,16 +344,14 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = return resolve(mountUri(newKey)) } resolve(url) + + }).then(url => { + if (url && (!picture[label] || pictureBytes[label] > bytes)) { + picture[label] = { url, size } + pictureBytes[label] = bytes + } + callback() }) - - .then(url => { - if (url && (!picture[label] || pictureBytes[label] > bytes)) { - // add to response pictures - picture[label] = { url, size } - pictureBytes[label] = bytes - } - callback() - }) } if ( @@ -398,52 +362,56 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = if (!isRetry) { return setTimeout(() => transformImg(true), 1000) } else { - // return image without all transformations return respond() } } callback(err) }) - } - transformImg() - }, imageBase64 ? 50 : 300) - } else { - setTimeout(() => { - // all done - respond() - }, 50) - } - } else if (uri && typeof err.message === 'string' && err.message.indexOf('cloud_name') > -1) { - // image uploaded but not transformed - respond() - logger.error(err) + + } + + // Transofrm image + transformImg() + + }, imageBase64 ? 50 : 300) } else { - // respond with error - const usrMsg = { - en_us: 'Error while handling image, the file may be protected or corrupted', - pt_br: 'Erro ao manipular a imagem, o arquivo pode estar protegido ou corrompido' - } - sendError(res, 415, uri, err.message, usrMsg) + setTimeout(() => { + // all done + respond() + }, 50) } - } - switch (mimetype) { - case 'image/jpeg': - case 'image/png': - callback() - break - default: - respond() + } else if (uri && typeof err.message === 'string' && err.message.indexOf('cloud_name') > -1) { + // image uploaded but not transformed + respond() + logger.error(err) + } else { + // respond with error + const usrMsg = { + en_us: 'Error while handling image, the file may be protected or corrupted', + pt_br: 'Erro ao manipular a imagem, o arquivo pode estar protegido ou corrompido' + } + sendError(res, 415, uri, err.message, usrMsg) } - }) + } - .catch(err => { - const usrMsg = { - en_us: 'This file cannot be uploaded to CDN', - pt_br: 'O arquivo não pôde ser carregado para o CDN' - } - sendError(res, 400, 3002, err.message, usrMsg) - }) + switch (mimetype) { + case 'image/jpeg': + case 'image/png': + callback() + break + default: + respond() + } + }) + // CDN Upload error + .catch((err) => { + const usrMsg = { + en_us: 'This file cannot be uploaded to CDN', + pt_br: 'O arquivo não pôde ser carregado para o CDN' + } + sendError(res, 400, 3002, err.message, usrMsg) + }) } }) }) diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index 5989085..d6fd391 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -15,15 +15,12 @@ module.exports = (auth) => { }) // Function to Compress image - return function (url, size, webpCompression, __callback) { + return function (url, __callback) { // API Payload let options = new FormData() options.append('file', url) - // Log cloudflare image upload - logger.log(`[Cloudflare] Image upload:`) - // Force timeout with 20s let callbackSent = false const callback = (err, data) => { @@ -49,21 +46,20 @@ module.exports = (auth) => { headers: { ...options.getHeaders() }, data: options }).then((response) => { - const id = response.data.id + const id = response.data.result.id const variants = response.data.result.variants - const variant = variants && variants.length ? variants.length > 1 ? variants[1] : variants[0] : null - console.log('[1]', response.data) - if (variant) { - download(`${variant}/normal`, (err, imageBody) => { + const normalImage = variants.find(v => v.includes('normal')) || variants[0] + if (normalImage) { + download(`${normalImage}`, (err, imageBody) => { if (err) logger.error(err) - callback(err, { variant, imageBody, ...response.data.result }) - setTimeout(() => { - // destroy on Cloudflare just to save storage - cloudinaryClient.delete(`/images/v1/${id}`) - }, 60000) + callback(err, { + url: normalImage, + imageBody, + ...response.data.result + }) + setTimeout(() => cloudinaryClient.delete(`/images/v1/${id}`), 60000) }) } }).catch((err) => callback(err)) - } } diff --git a/lib/Logger.js b/lib/Logger.js index 0e9ed93..89298fc 100644 --- a/lib/Logger.js +++ b/lib/Logger.js @@ -40,10 +40,8 @@ function middleware (out, desc, conn, _obj, callback) { const fs = require('fs') // log files -// const output = fs.createWriteStream('/var/log/nodejs/storage.out') -// const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') -const output = fs.createWriteStream('/home/gmazurco/storage.out') -const outputErrors = fs.createWriteStream('/home/gmazurco/storage.error') +const output = fs.createWriteStream('/var/log/nodejs/storage.out') +const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') // declares logger with Console class of global console const logger = new console.Console(output, outputErrors) diff --git a/main.js b/main.js index 09d0a7d..7a1fbef 100644 --- a/main.js +++ b/main.js @@ -23,13 +23,10 @@ process.on('uncaughtException', (err) => { msg += '\n' } - console.log('here', err) - - //! Restore after commit - // let fs = require('fs') - // fs.appendFile('/var/log/nodejs/_stderr', msg, () => { - // process.exit(1) - // }) + let fs = require('fs') + fs.appendFile('/var/log/nodejs/_stderr', msg, () => { + process.exit(1) + }) }) // web application From 2ef8a2e53ae2ac292235f46b31ab1637d740912b Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Fri, 28 Jan 2022 15:42:44 -0300 Subject: [PATCH 04/19] feat: cloudflare integration --- bin/web.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/web.js b/bin/web.js index 785557a..9ed045b 100644 --- a/bin/web.js +++ b/bin/web.js @@ -241,7 +241,6 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = let key = '@v3/' localUpload.single('file')(req, res, (err) => { - console.log('Upload single', err) if (err) { const usrMsg = { en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', From a431341e63e9c04104a913fefa656c04a724a465 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Fri, 28 Jan 2022 15:45:47 -0300 Subject: [PATCH 05/19] feat: cloudflare integration --- config/config-sample.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/config-sample.json b/config/config-sample.json index aa3946a..ef3d70c 100644 --- a/config/config-sample.json +++ b/config/config-sample.json @@ -16,5 +16,9 @@ "apiKey": "abcdefgh", "apiSecret": "1234567890" }, + "cloudflareAuth": { + "apiKey": "your_key", + "accountId": "your_account" + }, "cdnHost": "ecoms1.com" } From 814bbb3db002f176db1d3792ee2de233fd092f68 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Mon, 31 Jan 2022 14:36:22 -0300 Subject: [PATCH 06/19] fix: ecom review #1 --- bin/web.js | 2 +- lib/Cloudflare.js | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/web.js b/bin/web.js index 9ed045b..5aab334 100644 --- a/bin/web.js +++ b/bin/web.js @@ -314,7 +314,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Transform image updated to cloudflare const transformImg = (isRetry = false) => { - cloudflare(imageBase64 || originUrl, (err, data) => { + cloudflare(imageBase64 || originUrl, 'normal', (err, data) => { if (!err && data) { const { id, url, filename, imageBody } = data diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index d6fd391..064f2a2 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -9,13 +9,13 @@ const download = require('./Download') module.exports = (auth) => { // Setup axios client with api key - const cloudinaryClient = axios.create({ + const cloudflareClient = axios.create({ baseURL: `https://api.cloudflare.com/client/v4/accounts/${auth.accountId}`, headers: { Authorization: `Bearer ${auth.apiKey}` } }) // Function to Compress image - return function (url, __callback) { + return function (url, size = 'normal', __callback = () => {}) { // API Payload let options = new FormData() @@ -40,7 +40,7 @@ module.exports = (auth) => { }, 20000) // Upload image to cloudflare - cloudinaryClient({ + cloudflareClient({ method: 'POST', url: '/images/v1', headers: { ...options.getHeaders() }, @@ -48,16 +48,16 @@ module.exports = (auth) => { }).then((response) => { const id = response.data.result.id const variants = response.data.result.variants - const normalImage = variants.find(v => v.includes('normal')) || variants[0] - if (normalImage) { - download(`${normalImage}`, (err, imageBody) => { + const url = variants.find(v => v.endsWith(`/${size}`)) || variants[0] + if (url) { + download(`${url}`, (err, imageBody) => { if (err) logger.error(err) callback(err, { + ...response.data.result, url: normalImage, - imageBody, - ...response.data.result + imageBody }) - setTimeout(() => cloudinaryClient.delete(`/images/v1/${id}`), 60000) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) }) } }).catch((err) => callback(err)) From 1d844532f66cce7dcf8068320b00cfe5c5e7809a Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Mon, 31 Jan 2022 15:34:43 -0300 Subject: [PATCH 07/19] fix: ecom review #1 --- bin/web.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/web.js b/bin/web.js index 5aab334..5f5909f 100644 --- a/bin/web.js +++ b/bin/web.js @@ -314,7 +314,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Transform image updated to cloudflare const transformImg = (isRetry = false) => { - cloudflare(imageBase64 || originUrl, 'normal', (err, data) => { + cloudflare(imageBase64 || originUrl, label, (err, data) => { if (!err && data) { const { id, url, filename, imageBody } = data From 7f0f8b91237b30515de6994109edcb6a74d40270 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Mon, 31 Jan 2022 16:05:11 -0300 Subject: [PATCH 08/19] fix: ecom review #3 --- bin/web.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/web.js b/bin/web.js index 5f5909f..2e29729 100644 --- a/bin/web.js +++ b/bin/web.js @@ -285,6 +285,13 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = res.json({ bucket, key, uri, picture }) } + const sizeMapping = { + 'normal': 'normal', + 'big': 'big', + 'zoom': 'zoom', + 'small': 'w90' + } + const callback = err => { if (!err) { // next image size @@ -314,7 +321,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Transform image updated to cloudflare const transformImg = (isRetry = false) => { - cloudflare(imageBase64 || originUrl, label, (err, data) => { + cloudflare(imageBase64 || originUrl, sizeMapping[label], (err, data) => { if (!err && data) { const { id, url, filename, imageBody } = data From 4f9d33970e9197e5ae78ed1d3f443569a8853ec6 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Mon, 31 Jan 2022 16:40:34 -0300 Subject: [PATCH 09/19] fix: ecom review #4 --- bin/web.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/web.js b/bin/web.js index 2e29729..9d6d66c 100644 --- a/bin/web.js +++ b/bin/web.js @@ -329,11 +329,12 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = let contentType contentType = `image/${filename.includes('.png') ? 'png' : filename.includes('.webp') ? 'webp' : 'jpeg'}` + const fileFormat = `${filename.split('.')[1]}` if (imageBody || id) { const s3Options = { ...baseS3Options, ContentType: contentType, - Key: `${storeId}/${newKey}` + Key: `${storeId}/${newKey}.${fileFormat}` } if (imageBody) { transformedImageBody = imageBody From efbc64ec5ffa50cb07b51a555741e970925144c0 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Mon, 31 Jan 2022 17:52:59 -0300 Subject: [PATCH 10/19] fix: ecom review #4 --- bin/web.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/web.js b/bin/web.js index 9d6d66c..7231b24 100644 --- a/bin/web.js +++ b/bin/web.js @@ -323,13 +323,11 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = cloudflare(imageBase64 || originUrl, sizeMapping[label], (err, data) => { if (!err && data) { - const { id, url, filename, imageBody } = data - + const { id, url, imageBody } = data return new Promise(resolve => { - - let contentType - contentType = `image/${filename.includes('.png') ? 'png' : filename.includes('.webp') ? 'webp' : 'jpeg'}` - const fileFormat = `${filename.split('.')[1]}` + // Cloudinary keeps image as webp, so we using webp as default + const contentType = 'image/webp' + const fileFormat = 'webp' if (imageBody || id) { const s3Options = { ...baseS3Options, From a4a44ef9dee7c144f9f1479ca900fab169e7b1c0 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Tue, 1 Feb 2022 10:49:28 -0300 Subject: [PATCH 11/19] fix: ecom review #5 --- lib/Cloudflare.js | 8 ++++++-- lib/RequestDownload.js | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 lib/RequestDownload.js diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index 064f2a2..d1d56a1 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -3,7 +3,7 @@ // Dependencies const axios = require('axios').default const FormData = require('form-data') -const download = require('./Download') +const download = require('./RequestDownload') // Cloudflare module module.exports = (auth) => { @@ -50,7 +50,11 @@ module.exports = (auth) => { const variants = response.data.result.variants const url = variants.find(v => v.endsWith(`/${size}`)) || variants[0] if (url) { - download(`${url}`, (err, imageBody) => { + download({ + host: url, + method: 'GET', + headers: { 'Accept': 'image/webp,image/webp,image/*,*/*;q=0.8' } + }, (err, imageBody) => { if (err) logger.error(err) callback(err, { ...response.data.result, diff --git a/lib/RequestDownload.js b/lib/RequestDownload.js new file mode 100644 index 0000000..830d831 --- /dev/null +++ b/lib/RequestDownload.js @@ -0,0 +1,36 @@ +'use strict' + +// Node raw HTTP modules +const http = require('http') +const https = require('https') + +module.exports = (options = {}, callback, timeout = 10000) => { + // Create corret http client instance + let httpClient + if (url.startsWith('https')) { + httpClient = https + } else { + httpClient = http + } + + // GET image body + httpClient.request(options, (res) => { + if (res.statusCode !== 200) { + // consume response data to free up memory + res.resume() + callback(new Error(`Unexpected get image response status ${res.statusCode}`)) + } else { + const imageBody = [] + res.on('data', (chunk) => { imageBody.push(chunk) }) + res.on('end', () => { + callback(null, Buffer.concat(imageBody)) + }) + } + }) + // set request timeout pass errors to callback + .setTimeout(timeout) + .on('timeout', () => { + callback(new Error('Timed out trying to get image body')) + }) + .on('error', callback) +} From ff92c4a97ecd5117f4a7d2f8c31a99b3dfa0ccfe Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Tue, 1 Feb 2022 11:17:26 -0300 Subject: [PATCH 12/19] fix: ecom review #6 --- lib/Cloudflare.js | 6 +----- lib/RequestDownload.js | 36 +++++++++++------------------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index d1d56a1..3a1abde 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -50,11 +50,7 @@ module.exports = (auth) => { const variants = response.data.result.variants const url = variants.find(v => v.endsWith(`/${size}`)) || variants[0] if (url) { - download({ - host: url, - method: 'GET', - headers: { 'Accept': 'image/webp,image/webp,image/*,*/*;q=0.8' } - }, (err, imageBody) => { + download(url, { 'Accept': 'image/webp,image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { if (err) logger.error(err) callback(err, { ...response.data.result, diff --git a/lib/RequestDownload.js b/lib/RequestDownload.js index 830d831..cb28c28 100644 --- a/lib/RequestDownload.js +++ b/lib/RequestDownload.js @@ -1,36 +1,22 @@ 'use strict' // Node raw HTTP modules -const http = require('http') -const https = require('https') - -module.exports = (options = {}, callback, timeout = 10000) => { - // Create corret http client instance - let httpClient - if (url.startsWith('https')) { - httpClient = https - } else { - httpClient = http - } +const axios = require('axios').default +// Downloader +module.exports = (url, headers, callback, timeout = 10000) => { // GET image body - httpClient.request(options, (res) => { - if (res.statusCode !== 200) { + axios.get(url, { headers, timeout, responseType: 'arraybuffer' }).then(res => { + if (res.status !== 200) { // consume response data to free up memory res.resume() - callback(new Error(`Unexpected get image response status ${res.statusCode}`)) + callback(new Error(`Unexpected get image response status ${res.status}`)) } else { - const imageBody = [] - res.on('data', (chunk) => { imageBody.push(chunk) }) - res.on('end', () => { - callback(null, Buffer.concat(imageBody)) - }) + // Res.data sample return + // + callback(null, res.data) } + }).catch((err) => { + callback(err) }) - // set request timeout pass errors to callback - .setTimeout(timeout) - .on('timeout', () => { - callback(new Error('Timed out trying to get image body')) - }) - .on('error', callback) } From da504b62a85e14070a6e735e8f57b014ee4ce535 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Tue, 1 Feb 2022 11:21:14 -0300 Subject: [PATCH 13/19] fix: ecom review #7 --- bin/web.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/web.js b/bin/web.js index 7231b24..cef2189 100644 --- a/bin/web.js +++ b/bin/web.js @@ -285,13 +285,6 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = res.json({ bucket, key, uri, picture }) } - const sizeMapping = { - 'normal': 'normal', - 'big': 'big', - 'zoom': 'zoom', - 'small': 'w90' - } - const callback = err => { if (!err) { // next image size @@ -321,7 +314,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Transform image updated to cloudflare const transformImg = (isRetry = false) => { - cloudflare(imageBase64 || originUrl, sizeMapping[label], (err, data) => { + cloudflare(imageBase64 || originUrl, label === 'small' ? 'w90' : label, (err, data) => { if (!err && data) { const { id, url, imageBody } = data return new Promise(resolve => { From 1d1e711d1b836c5300590065fe83232627127b89 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Tue, 1 Feb 2022 15:40:55 -0300 Subject: [PATCH 14/19] fix: ecom review #8 --- bin/web.js | 5 +---- lib/Cloudflare.js | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/web.js b/bin/web.js index cef2189..ec43af9 100644 --- a/bin/web.js +++ b/bin/web.js @@ -45,10 +45,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = const pictureOptims = (pictureSizes || [700, 350]).reduce((optims, size, i) => { const label = i === 0 ? 'big' : i === 1 ? 'normal' : 'small' - optims.push( - { size, label, webp: false }, - { size, label, webp: true } - ) + optims.push({ size, label, webp: true }) return optims }, []) diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index 3a1abde..bfdd667 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -50,7 +50,7 @@ module.exports = (auth) => { const variants = response.data.result.variants const url = variants.find(v => v.endsWith(`/${size}`)) || variants[0] if (url) { - download(url, { 'Accept': 'image/webp,image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { + download(url, { 'Accept': 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { if (err) logger.error(err) callback(err, { ...response.data.result, From d7f89aa3eed75c3ce7caa3da31316bb2bfd496c0 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Tue, 1 Feb 2022 17:46:02 -0300 Subject: [PATCH 15/19] fix: ecom review #8 --- bin/web.js | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/bin/web.js b/bin/web.js index ec43af9..671bb4b 100644 --- a/bin/web.js +++ b/bin/web.js @@ -290,28 +290,17 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = let newKey const { label, size, webp } = pictureOptims[i] newKey = `imgs/${label}/${key}` + const imageFile = req.file - const imageBuffer = i === 0 ? req.file.buffer : transformedImageBody - const imageBase64 = imageBuffer - ? `data:${mimetype};base64,${imageBuffer.toString('base64')}` - : null // free memory transformedImageBody = req.file = null setTimeout(() => { - // Retrieve url - let originUrl - if (picture[label] && webp) { - originUrl = picture[label].url - } else { - originUrl = uri - } - // Transform image updated to cloudflare const transformImg = (isRetry = false) => { - cloudflare(imageBase64 || originUrl, label === 'small' ? 'w90' : label, (err, data) => { + cloudflare(imageFile, label === 'small' ? 'w90' : label, (err, data) => { if (!err && data) { const { id, url, imageBody } = data return new Promise(resolve => { @@ -368,7 +357,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Transofrm image transformImg() - }, imageBase64 ? 50 : 300) + }, imageFile ? 50 : 300) } else { setTimeout(() => { // all done From 03fe3dd4ec8421dcac14e49686555842deafb509 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Tue, 1 Feb 2022 18:28:16 -0300 Subject: [PATCH 16/19] refactor(uploader): cloudflare uploader --- bin/web.js | 360 ++++++++++++++++++++++++++-------------------- lib/Cloudflare.js | 33 +++-- 2 files changed, 221 insertions(+), 172 deletions(-) diff --git a/bin/web.js b/bin/web.js index 671bb4b..7d2aac2 100644 --- a/bin/web.js +++ b/bin/web.js @@ -231,173 +231,219 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = } app.post(urls.upload, (req, res) => { + logger.log(`${storeId} [storage-api] Upload start...`) + + // Retrieve request parameters + const key = '@v3/' const { storeId } = req const { bucket, host } = spaces[0] - logger.log(`${storeId} Uploading...`) - // unique object key - let key = '@v3/' - - localUpload.single('file')(req, res, (err) => { - if (err) { - const usrMsg = { - en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', - pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' - } - sendError(res, 400, 3001, err.message, usrMsg) - } else { - let dir = req.query.directory - if (typeof dir === 'string' && dir.charAt(0) === '/') { - // remove first char to not duplicate first bar - // normalize, then remove empty paths - dir = dir.substr(1).replace(/[^\w-/]/g, '').replace('//', '') - if (dir.length) { - key += dir.toLowerCase() + '/' - } - } - // keep filename - const filename = req.file.originalname.replace(/[^\w-.]/g, '').toLowerCase() - key += `${Date.now()}-${filename}` - const { mimetype } = req.file - - runMethod('putObject', { - ...baseS3Options, - ContentType: mimetype, - Key: `${storeId}/${key}`, - Body: req.file.buffer - }) - // S3 Response - .then(() => { - logger.log(`${storeId} ${key} Uploaded to ${bucket}`) - // zoom uploaded - const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` - const uri = mountUri(key) - const picture = { zoom: { url: uri } } - const pictureBytes = {} - // resize/optimize image - let i = -1 - let transformedImageBody = null - - const respond = () => { - logger.log(`${storeId} ${key} ${bucket} All optimizations done`) - res.json({ bucket, key, uri, picture }) - } - const callback = err => { - if (!err) { - // next image size - i++ - if (i < pictureOptims.length) { - let newKey - const { label, size, webp } = pictureOptims[i] - newKey = `imgs/${label}/${key}` - const imageFile = req.file + // Validate if received a file + const file = req.file + if (!file) { + sendError(res, 422, 3001, 'Unprocessable entity', { + en_us: 'You must send a file attached to this request', + pt_br: 'Você deve enviar um arquivo anexado a essa requisição' + }) + } - // free memory - transformedImageBody = req.file = null + // Validate if file matchs the recommended size + const fileBytes = file.size + if (fileBytes > 2048) { + sendError(res, 422, 3001, 'Unprocessable entity', { + en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', + pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' + }) + } - setTimeout(() => { - - // Transform image updated to cloudflare - const transformImg = (isRetry = false) => { - - cloudflare(imageFile, label === 'small' ? 'w90' : label, (err, data) => { - if (!err && data) { - const { id, url, imageBody } = data - return new Promise(resolve => { - // Cloudinary keeps image as webp, so we using webp as default - const contentType = 'image/webp' - const fileFormat = 'webp' - if (imageBody || id) { - const s3Options = { - ...baseS3Options, - ContentType: contentType, - Key: `${storeId}/${newKey}.${fileFormat}` - } - if (imageBody) { - transformedImageBody = imageBody - // PUT new image on S3 bucket - return runMethod('putObject', { ...s3Options, Body: imageBody }) - .then(() => resolve(mountUri(newKey))) - .catch((err) => { - logger.error(err) - resolve(url) - }) - } - // async handle with callback URL - redisClient.setex(genRedisKey(id, true), 600, JSON.stringify(s3Options)) - return resolve(mountUri(newKey)) - } - resolve(url) - - }).then(url => { - if (url && (!picture[label] || pictureBytes[label] > bytes)) { - picture[label] = { url, size } - pictureBytes[label] = bytes - } - callback() - }) - } - - if ( - err && - typeof err.message === 'string' && - (err.message.indexOf('504 Gateway Timeout') > -1 || err.message.indexOf('503 Service Unavailable') > -1) - ) { - if (!isRetry) { - return setTimeout(() => transformImg(true), 1000) - } else { - return respond() - } - } - callback(err) - }) - - } - - // Transofrm image - transformImg() - - }, imageFile ? 50 : 300) - } else { - setTimeout(() => { - // all done - respond() - }, 50) - } + // Upload to cloudiflare + cloudflare(file, (err, data) => { + + // No errors by the way + if (!err && data) { - } else if (uri && typeof err.message === 'string' && err.message.indexOf('cloud_name') > -1) { - // image uploaded but not transformed - respond() - logger.error(err) - } else { - // respond with error - const usrMsg = { - en_us: 'Error while handling image, the file may be protected or corrupted', - pt_br: 'Erro ao manipular a imagem, o arquivo pode estar protegido ou corrompido' - } - sendError(res, 415, uri, err.message, usrMsg) - } - } + // Retrieve converted images + const { convertedImages } = data - switch (mimetype) { - case 'image/jpeg': - case 'image/png': - callback() - break - default: - respond() - } - }) - // CDN Upload error - .catch((err) => { - const usrMsg = { - en_us: 'This file cannot be uploaded to CDN', - pt_br: 'O arquivo não pôde ser carregado para o CDN' - } - sendError(res, 400, 3002, err.message, usrMsg) + // Map converted images and upload to S3 + convertedImages.forEach(({ label, imageBody }) => { + }) + + } + + // Sadness and sorrow + if ( + err && + typeof err.message === 'string' && + (err.message.indexOf('504 Gateway Timeout') > -1 || err.message.indexOf('503 Service Unavailable') > -1) + ) { + return respond() } }) + + + // localUpload.single('file')(req, res, (err) => { + // if (err) { + // const usrMsg = { + // en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', + // pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' + // } + // sendError(res, 400, 3001, err.message, usrMsg) + // } else { + // let dir = req.query.directory + // if (typeof dir === 'string' && dir.charAt(0) === '/') { + // // remove first char to not duplicate first bar + // // normalize, then remove empty paths + // dir = dir.substr(1).replace(/[^\w-/]/g, '').replace('//', '') + // if (dir.length) { + // key += dir.toLowerCase() + '/' + // } + // } + // // keep filename + // const filename = req.file.originalname.replace(/[^\w-.]/g, '').toLowerCase() + // key += `${Date.now()}-${filename}` + // const { mimetype } = req.file + + // runMethod('putObject', { + // ...baseS3Options, + // ContentType: mimetype, + // Key: `${storeId}/${key}`, + // Body: req.file.buffer + // }) + // // S3 Response + // .then(() => { + // logger.log(`${storeId} ${key} Uploaded to ${bucket}`) + // // zoom uploaded + // const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` + // const uri = mountUri(key) + // const picture = { zoom: { url: uri } } + // const pictureBytes = {} + // // resize/optimize image + // let i = -1 + // let transformedImageBody = null + + // const respond = () => { + // logger.log(`${storeId} ${key} ${bucket} All optimizations done`) + // res.json({ bucket, key, uri, picture }) + // } + + // const callback = err => { + // if (!err) { + // // next image size + // i++ + // if (i < pictureOptims.length) { + // let newKey + // const { label, size, webp } = pictureOptims[i] + // newKey = `imgs/${label}/${key}` + // const imageFile = req.file + + // // free memory + // transformedImageBody = req.file = null + + // setTimeout(() => { + + // // Transform image updated to cloudflare + // const transformImg = (isRetry = false) => { + + // cloudflare(imageFile, label === 'small' ? 'w90' : label, (err, data) => { + // if (!err && data) { + // const { id, url, imageBody } = data + // return new Promise(resolve => { + // // Cloudinary keeps image as webp, so we using webp as default + // const contentType = 'image/webp' + // const fileFormat = 'webp' + // if (imageBody || id) { + // const s3Options = { + // ...baseS3Options, + // ContentType: contentType, + // Key: `${storeId}/${newKey}.${fileFormat}` + // } + // if (imageBody) { + // transformedImageBody = imageBody + // // PUT new image on S3 bucket + // return runMethod('putObject', { ...s3Options, Body: imageBody }) + // .then(() => resolve(mountUri(newKey))) + // .catch((err) => { + // logger.error(err) + // resolve(url) + // }) + // } + // // async handle with callback URL + // redisClient.setex(genRedisKey(id, true), 600, JSON.stringify(s3Options)) + // return resolve(mountUri(newKey)) + // } + // resolve(url) + + // }).then(url => { + // if (url && (!picture[label] || pictureBytes[label] > bytes)) { + // picture[label] = { url, size } + // pictureBytes[label] = bytes + // } + // callback() + // }) + // } + + // if ( + // err && + // typeof err.message === 'string' && + // (err.message.indexOf('504 Gateway Timeout') > -1 || err.message.indexOf('503 Service Unavailable') > -1) + // ) { + // if (!isRetry) { + // return setTimeout(() => transformImg(true), 1000) + // } else { + // return respond() + // } + // } + // callback(err) + // }) + + // } + + // // Transofrm image + // transformImg() + + // }, imageFile ? 50 : 300) + // } else { + // setTimeout(() => { + // // all done + // respond() + // }, 50) + // } + + // } else if (uri && typeof err.message === 'string' && err.message.indexOf('cloud_name') > -1) { + // // image uploaded but not transformed + // respond() + // logger.error(err) + // } else { + // // respond with error + // const usrMsg = { + // en_us: 'Error while handling image, the file may be protected or corrupted', + // pt_br: 'Erro ao manipular a imagem, o arquivo pode estar protegido ou corrompido' + // } + // sendError(res, 415, uri, err.message, usrMsg) + // } + // } + + // switch (mimetype) { + // case 'image/jpeg': + // case 'image/png': + // callback() + // break + // default: + // respond() + // } + // }) + // // CDN Upload error + // .catch((err) => { + // const usrMsg = { + // en_us: 'This file cannot be uploaded to CDN', + // pt_br: 'O arquivo não pôde ser carregado para o CDN' + // } + // sendError(res, 400, 3002, err.message, usrMsg) + // }) + // } + // }) }) app.use(urls.manipulationCallback, (req, res) => { diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index bfdd667..29296ac 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -15,11 +15,11 @@ module.exports = (auth) => { }) // Function to Compress image - return function (url, size = 'normal', __callback = () => {}) { + return function (imageFile, __callback = () => {}) { // API Payload let options = new FormData() - options.append('file', url) + options.append('file', imageFile) // Force timeout with 20s let callbackSent = false @@ -33,11 +33,11 @@ module.exports = (auth) => { } } - // Verify if responded in 20s + // Verify if responded in 30s const timer = setTimeout(() => { callback(new Error('Cloudflare optimization timed out')) logger.log(`Cloudflare timed out`) - }, 20000) + }, 30000) // Upload image to cloudflare cloudflareClient({ @@ -48,18 +48,21 @@ module.exports = (auth) => { }).then((response) => { const id = response.data.result.id const variants = response.data.result.variants - const url = variants.find(v => v.endsWith(`/${size}`)) || variants[0] - if (url) { - download(url, { 'Accept': 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { - if (err) logger.error(err) - callback(err, { - ...response.data.result, - url: normalImage, - imageBody + const fetchedImages = [] + variants.forEach(vari => { + if (vari.includes('normal') || vari.includes('zoom') || vari.includes('big') || vari.includes('w90')) { + const labels = vari.split('/') + const label = labels[labels.length - 1] + download(vari, { 'Accept': 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { + if (err) logger.error(err) + fetchedImages.push({ imageBody, label }) + if (fetchedImages.length === 4) { + callback(err, { convertedImages: fetchedImages }) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + } }) - setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) - }) - } + } + }) }).catch((err) => callback(err)) } } From ff432387993badeed4f853be9943ee411ae52ef2 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Wed, 2 Feb 2022 10:04:30 -0300 Subject: [PATCH 17/19] test(logic): testing logic --- bin/web.js | 4 ++++ lib/Cloudflare.js | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bin/web.js b/bin/web.js index 7d2aac2..082d599 100644 --- a/bin/web.js +++ b/bin/web.js @@ -268,8 +268,12 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Map converted images and upload to S3 convertedImages.forEach(({ label, imageBody }) => { + console.log ('[upload] here', label) + }) + return respond() + } // Sadness and sorrow diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index 29296ac..06ae245 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -46,19 +46,32 @@ module.exports = (auth) => { headers: { ...options.getHeaders() }, data: options }).then((response) => { + // Retrieve Upload data const id = response.data.result.id const variants = response.data.result.variants const fetchedImages = [] + let processed = 0 + + // Download the correct variations variants.forEach(vari => { if (vari.includes('normal') || vari.includes('zoom') || vari.includes('big') || vari.includes('w90')) { + // Retrieve image type const labels = vari.split('/') const label = labels[labels.length - 1] - download(vari, { 'Accept': 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { + + // Download image, zoom must be jpeg + download(vari, { 'Accept': label === 'zoom' ? 'image/jpeg,image/*,*/*;q=0.8' : 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { + processed++ if (err) logger.error(err) fetchedImages.push({ imageBody, label }) - if (fetchedImages.length === 4) { + + // IF all images was fetched, then proceed + if (fetchedImages.length === 4 && processed === 4) { callback(err, { convertedImages: fetchedImages }) setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + } else { + callback(err, { convertedImages: fetchedImages, error: true }) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) } }) } From 12340cd73eafcf93f93ce263c0bda1bf3e849a3a Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Wed, 2 Feb 2022 15:12:36 -0300 Subject: [PATCH 18/19] refactor: logic --- bin/web.js | 320 ++++++++++++++++------------------------------ lib/Cloudflare.js | 123 ++++++++++-------- lib/Logger.js | 6 +- main.js | 8 +- package-lock.json | 13 +- package.json | 3 +- 6 files changed, 200 insertions(+), 273 deletions(-) diff --git a/bin/web.js b/bin/web.js index 082d599..b8bfea6 100644 --- a/bin/web.js +++ b/bin/web.js @@ -23,6 +23,8 @@ const Express = require('express') const multer = require('multer') // body parsing middleware const bodyParser = require('body-parser') +// UUID Generator +const { v4 } = require('uuid') // Redis to store buckets and Kraken async requests IDs const redis = require('redis') @@ -219,10 +221,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // setup multer for file upload const localUpload = multer({ storage: multer.memoryStorage(), - limits: { - // maximum 2mb - fileSize: 2000000 - } + limits: { fileSize: 2000000 } }) const baseS3Options = { @@ -231,223 +230,121 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = } app.post(urls.upload, (req, res) => { - logger.log(`${storeId} [storage-api] Upload start...`) - - // Retrieve request parameters - const key = '@v3/' - const { storeId } = req - const { bucket, host } = spaces[0] - - // Validate if received a file - const file = req.file - if (!file) { - sendError(res, 422, 3001, 'Unprocessable entity', { - en_us: 'You must send a file attached to this request', - pt_br: 'Você deve enviar um arquivo anexado a essa requisição' - }) - } + logger.log(`[storage-api] Upload start...`,) + + // Validate file + localUpload.single('file')(req, res, (err) => { + if (err) { + sendError(res, 400, 3001, err.message, { + en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', + pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' + }) + } else { - // Validate if file matchs the recommended size - const fileBytes = file.size - if (fileBytes > 2048) { - sendError(res, 422, 3001, 'Unprocessable entity', { - en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', - pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' - }) - } + // Retrieve request parameters + let key = '@v3/' + const { bucket, host } = spaces[0] - // Upload to cloudiflare - cloudflare(file, (err, data) => { - - // No errors by the way - if (!err && data) { + // Retrieve local image data + let dir = req.query.directory + if (typeof dir === 'string' && dir.charAt(0) === '/') { + dir = dir.substr(1).replace(/[^\w-/]/g, '').replace('//', '') + if (dir.length) { + key += dir.toLowerCase() + '/' + } + } - // Retrieve converted images - const { convertedImages } = data + // Keep filename + const filename = req.file.originalname.replace(/[^\w-.]/g, '').toLowerCase() + key += `${v4()}-${filename}` + + // Aux methods + const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${1282}/${key}` + let picture = {} + const respond = (suc = true) => { + if (suc) { + logger.log(`${1282} ${key} ${bucket} All optimizations done`) + res.json({ bucket, key, uri: picture['zoom'] ? picture['zoom'].url : '' , picture }) + } else { + res.status(500).json({ message: { + en_us: 'An error ocourred whlie uploading your file.', + pt_br: 'Ocorreu um erro ao persistir seu arquivo' + }}) + } + } - // Map converted images and upload to S3 - convertedImages.forEach(({ label, imageBody }) => { + // Upload to cloudiflare + console.log('[rdy to cloudflare]') + cloudflare(req.file, (err, data) => { - console.log ('[upload] here', label) - - }) + // No errors by the way + if (!err && data) { + + // Retrieve converted images + const { convertedImages } = data + + // Map converted images and upload to S3 + convertedImages.forEach(({ id, label, imageBody }) => { + + console.log ('[upload] here', label) + const newKey = `imgs/${label}/${key}` + let s3attempts = 0 + + // Upload image to s3 + return new Promise((resolve) => { + + // Define s3 options + const contentType = label === 'zoom' ? 'image/jpeg' : 'image/webp' + const fileFormat = label === 'zoom' ? 'jpg' : 'webp' + const s3Options = { + ...baseS3Options, + ContentType: contentType, + Key: `${1282}/${newKey}.${fileFormat}` + } + + // Put s3 object + if (imageBody) { + return runMethod('putObject', { ...s3Options, Body: imageBody }) + .then(() => resolve(mountUri(newKey))) + .catch((err) => { + logger.error(err) + resolve(key) + }) + } + + // async handle with callback URL + redisClient.setex(genRedisKey(id, true), 600, JSON.stringify(s3Options)) + return resolve(mountUri(newKey)) + }) + .then((url) => { + s3attempts++ + if (url && (!picture[label])) { + picture[label] = { url, size: label } + // pictureBytes[label] = bytes + } + if (s3attempts === 4 && Object.keys(picture).length === 4) { + return respond() + } else if (s3attempts === 4) { + return sendError(res, 500, 3002, 'Internal server error', { + en_us: 'An error ocourred whlie uploading your file.', + pt_br: 'Ocorreu um erro ao persistir seu arquivo' + }) + } + }) + }) - return respond() + return respond() - } + } + + // Sadness and sorrow + if (err) { + return respond(false) + } + }) - // Sadness and sorrow - if ( - err && - typeof err.message === 'string' && - (err.message.indexOf('504 Gateway Timeout') > -1 || err.message.indexOf('503 Service Unavailable') > -1) - ) { - return respond() } }) - - - // localUpload.single('file')(req, res, (err) => { - // if (err) { - // const usrMsg = { - // en_us: 'This file cannot be uploaded, make sure it is a valid image with up to 2mb', - // pt_br: 'O arquivo não pôde ser carregado, verifique se é uma imagem válida com até 2mb' - // } - // sendError(res, 400, 3001, err.message, usrMsg) - // } else { - // let dir = req.query.directory - // if (typeof dir === 'string' && dir.charAt(0) === '/') { - // // remove first char to not duplicate first bar - // // normalize, then remove empty paths - // dir = dir.substr(1).replace(/[^\w-/]/g, '').replace('//', '') - // if (dir.length) { - // key += dir.toLowerCase() + '/' - // } - // } - // // keep filename - // const filename = req.file.originalname.replace(/[^\w-.]/g, '').toLowerCase() - // key += `${Date.now()}-${filename}` - // const { mimetype } = req.file - - // runMethod('putObject', { - // ...baseS3Options, - // ContentType: mimetype, - // Key: `${storeId}/${key}`, - // Body: req.file.buffer - // }) - // // S3 Response - // .then(() => { - // logger.log(`${storeId} ${key} Uploaded to ${bucket}`) - // // zoom uploaded - // const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${storeId}/${key}` - // const uri = mountUri(key) - // const picture = { zoom: { url: uri } } - // const pictureBytes = {} - // // resize/optimize image - // let i = -1 - // let transformedImageBody = null - - // const respond = () => { - // logger.log(`${storeId} ${key} ${bucket} All optimizations done`) - // res.json({ bucket, key, uri, picture }) - // } - - // const callback = err => { - // if (!err) { - // // next image size - // i++ - // if (i < pictureOptims.length) { - // let newKey - // const { label, size, webp } = pictureOptims[i] - // newKey = `imgs/${label}/${key}` - // const imageFile = req.file - - // // free memory - // transformedImageBody = req.file = null - - // setTimeout(() => { - - // // Transform image updated to cloudflare - // const transformImg = (isRetry = false) => { - - // cloudflare(imageFile, label === 'small' ? 'w90' : label, (err, data) => { - // if (!err && data) { - // const { id, url, imageBody } = data - // return new Promise(resolve => { - // // Cloudinary keeps image as webp, so we using webp as default - // const contentType = 'image/webp' - // const fileFormat = 'webp' - // if (imageBody || id) { - // const s3Options = { - // ...baseS3Options, - // ContentType: contentType, - // Key: `${storeId}/${newKey}.${fileFormat}` - // } - // if (imageBody) { - // transformedImageBody = imageBody - // // PUT new image on S3 bucket - // return runMethod('putObject', { ...s3Options, Body: imageBody }) - // .then(() => resolve(mountUri(newKey))) - // .catch((err) => { - // logger.error(err) - // resolve(url) - // }) - // } - // // async handle with callback URL - // redisClient.setex(genRedisKey(id, true), 600, JSON.stringify(s3Options)) - // return resolve(mountUri(newKey)) - // } - // resolve(url) - - // }).then(url => { - // if (url && (!picture[label] || pictureBytes[label] > bytes)) { - // picture[label] = { url, size } - // pictureBytes[label] = bytes - // } - // callback() - // }) - // } - - // if ( - // err && - // typeof err.message === 'string' && - // (err.message.indexOf('504 Gateway Timeout') > -1 || err.message.indexOf('503 Service Unavailable') > -1) - // ) { - // if (!isRetry) { - // return setTimeout(() => transformImg(true), 1000) - // } else { - // return respond() - // } - // } - // callback(err) - // }) - - // } - - // // Transofrm image - // transformImg() - - // }, imageFile ? 50 : 300) - // } else { - // setTimeout(() => { - // // all done - // respond() - // }, 50) - // } - - // } else if (uri && typeof err.message === 'string' && err.message.indexOf('cloud_name') > -1) { - // // image uploaded but not transformed - // respond() - // logger.error(err) - // } else { - // // respond with error - // const usrMsg = { - // en_us: 'Error while handling image, the file may be protected or corrupted', - // pt_br: 'Erro ao manipular a imagem, o arquivo pode estar protegido ou corrompido' - // } - // sendError(res, 415, uri, err.message, usrMsg) - // } - // } - - // switch (mimetype) { - // case 'image/jpeg': - // case 'image/png': - // callback() - // break - // default: - // respond() - // } - // }) - // // CDN Upload error - // .catch((err) => { - // const usrMsg = { - // en_us: 'This file cannot be uploaded to CDN', - // pt_br: 'O arquivo não pôde ser carregado para o CDN' - // } - // sendError(res, 400, 3002, err.message, usrMsg) - // }) - // } - // }) }) app.use(urls.manipulationCallback, (req, res) => { @@ -520,6 +417,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = }) app.listen(port, () => { + console.log('App is running') logger.log('Storage API running with Express on port ' + port) }) } diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index 06ae245..4411859 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -8,6 +8,8 @@ const download = require('./RequestDownload') // Cloudflare module module.exports = (auth) => { + console.log('run here') + // Setup axios client with api key const cloudflareClient = axios.create({ baseURL: `https://api.cloudflare.com/client/v4/accounts/${auth.accountId}`, @@ -16,66 +18,83 @@ module.exports = (auth) => { // Function to Compress image return function (imageFile, __callback = () => {}) { + console.log('run here too', imageFile) // API Payload - let options = new FormData() - options.append('file', imageFile) - - // Force timeout with 20s - let callbackSent = false - const callback = (err, data) => { - if (!callbackSent) { - callbackSent = true - __callback(err, data) + try { + + const fData = new FormData() + let options = { + file: imageFile } - if (timer) { - clearTimeout(timer) + + // Force timeout with 20s + let callbackSent = false + const callback = (err, data) => { + if (!callbackSent) { + callbackSent = true + __callback(err, data) + } + if (timer) { + clearTimeout(timer) + } } - } - // Verify if responded in 30s - const timer = setTimeout(() => { - callback(new Error('Cloudflare optimization timed out')) - logger.log(`Cloudflare timed out`) - }, 30000) + // Verify if responded in 30s + const timer = setTimeout(() => { + callback(new Error('Cloudflare optimization timed out')) + logger.log(`Cloudflare timed out`) + }, 30000) - // Upload image to cloudflare - cloudflareClient({ - method: 'POST', - url: '/images/v1', - headers: { ...options.getHeaders() }, - data: options - }).then((response) => { - // Retrieve Upload data - const id = response.data.result.id - const variants = response.data.result.variants - const fetchedImages = [] - let processed = 0 + // Upload image to cloudflare + console.log('[before upload]') + cloudflareClient({ + method: 'POST', + url: '/images/v1', + headers: { ...fData.getHeaders() }, + data: options + }).then((response) => { + console.log('[after upload]',response) - // Download the correct variations - variants.forEach(vari => { - if (vari.includes('normal') || vari.includes('zoom') || vari.includes('big') || vari.includes('w90')) { - // Retrieve image type - const labels = vari.split('/') - const label = labels[labels.length - 1] - - // Download image, zoom must be jpeg - download(vari, { 'Accept': label === 'zoom' ? 'image/jpeg,image/*,*/*;q=0.8' : 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { - processed++ - if (err) logger.error(err) - fetchedImages.push({ imageBody, label }) + // Retrieve Upload data + const id = response.data.result.id + const variants = response.data.result.variants + const fetchedImages = [] + let processed = 0 - // IF all images was fetched, then proceed - if (fetchedImages.length === 4 && processed === 4) { - callback(err, { convertedImages: fetchedImages }) - setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) - } else { - callback(err, { convertedImages: fetchedImages, error: true }) - setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) - } - }) - } + // Download the correct variations + variants.forEach(vari => { + if (vari.includes('normal') || vari.includes('zoom') || vari.includes('big') || vari.includes('w90')) { + // Retrieve image type + const labels = vari.split('/') + const label = labels[labels.length - 1] + + // Download image, zoom must be jpeg + download(vari, { 'Accept': label === 'zoom' ? 'image/jpeg,image/*,*/*;q=0.8' : 'image/webp,image/*,*/*;q=0.8' }, (err, imageBody) => { + processed++ + if (err) logger.error(err) + fetchedImages.push({ id, imageBody, label }) + + // IF all images was fetched, then proceed + if (fetchedImages.length === 4 && processed === 4) { + callback(err, { convertedImages: fetchedImages }) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + } else if (processed === 4) { + callback(err, { convertedImages: fetchedImages, error: true }) + setTimeout(() => cloudflareClient.delete(`/images/v1/${id}`), 60000) + } + }) + } + }) + }).catch((err) => { + console.log('[here with error]', err.response.data) + callback(err) }) - }).catch((err) => callback(err)) + + } catch (err) { + console.log('[error]', err) + __callback(err, null) + } + } } diff --git a/lib/Logger.js b/lib/Logger.js index 89298fc..0e9ed93 100644 --- a/lib/Logger.js +++ b/lib/Logger.js @@ -40,8 +40,10 @@ function middleware (out, desc, conn, _obj, callback) { const fs = require('fs') // log files -const output = fs.createWriteStream('/var/log/nodejs/storage.out') -const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') +// const output = fs.createWriteStream('/var/log/nodejs/storage.out') +// const outputErrors = fs.createWriteStream('/var/log/nodejs/storage.error') +const output = fs.createWriteStream('/home/gmazurco/storage.out') +const outputErrors = fs.createWriteStream('/home/gmazurco/storage.error') // declares logger with Console class of global console const logger = new console.Console(output, outputErrors) diff --git a/main.js b/main.js index 7a1fbef..2c5e823 100644 --- a/main.js +++ b/main.js @@ -23,10 +23,10 @@ process.on('uncaughtException', (err) => { msg += '\n' } - let fs = require('fs') - fs.appendFile('/var/log/nodejs/_stderr', msg, () => { - process.exit(1) - }) + // let fs = require('fs') + // fs.appendFile('/var/log/nodejs/_stderr', msg, () => { + // process.exit(1) + // }) }) // web application diff --git a/package-lock.json b/package-lock.json index be54ca7..890ca5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -671,6 +671,13 @@ "url": "0.10.3", "uuid": "3.3.2", "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } } }, "axios": { @@ -5404,9 +5411,9 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, "v8-compile-cache": { "version": "2.3.0", diff --git a/package.json b/package.json index 4ad9dba..702a0a4 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "express": "^4.17.1", "form-data": "^4.0.0", "multer": "^1.4.2", - "redis": "^3.1.2" + "redis": "^3.1.2", + "uuid": "^8.3.2" }, "devDependencies": { "@commitlint/cli": "^13.1.0", From 47c175dc19f74368712a45676e212256b38e2de2 Mon Sep 17 00:00:00 2001 From: Gabriel Mazurco Date: Wed, 2 Feb 2022 15:30:04 -0300 Subject: [PATCH 19/19] refactor: logic --- bin/web.js | 16 ++++++---------- lib/Cloudflare.js | 15 ++------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/bin/web.js b/bin/web.js index b8bfea6..98f1faa 100644 --- a/bin/web.js +++ b/bin/web.js @@ -259,11 +259,11 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = key += `${v4()}-${filename}` // Aux methods - const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${1282}/${key}` + const mountUri = (key, baseUrl = cdnHost || host) => `https://${baseUrl}/${req.storeId}/${key}` let picture = {} const respond = (suc = true) => { if (suc) { - logger.log(`${1282} ${key} ${bucket} All optimizations done`) + logger.log(`${req.storeId} ${key} ${bucket} All optimizations done`) res.json({ bucket, key, uri: picture['zoom'] ? picture['zoom'].url : '' , picture }) } else { res.status(500).json({ message: { @@ -274,7 +274,6 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = } // Upload to cloudiflare - console.log('[rdy to cloudflare]') cloudflare(req.file, (err, data) => { // No errors by the way @@ -282,13 +281,12 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = // Retrieve converted images const { convertedImages } = data + let s3attempts = 0 // Map converted images and upload to S3 convertedImages.forEach(({ id, label, imageBody }) => { - console.log ('[upload] here', label) const newKey = `imgs/${label}/${key}` - let s3attempts = 0 // Upload image to s3 return new Promise((resolve) => { @@ -299,7 +297,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = const s3Options = { ...baseS3Options, ContentType: contentType, - Key: `${1282}/${newKey}.${fileFormat}` + Key: `${req.storeId}/${newKey}.${fileFormat}` } // Put s3 object @@ -319,7 +317,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = .then((url) => { s3attempts++ if (url && (!picture[label])) { - picture[label] = { url, size: label } + picture[label] = { url: mountUri(url), size: label } // pictureBytes[label] = bytes } if (s3attempts === 4 && Object.keys(picture).length === 4) { @@ -333,8 +331,7 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = }) }) - return respond() - + //return respond() } // Sadness and sorrow @@ -342,7 +339,6 @@ fs.readFile(path.join(__dirname, '../config/config.json'), 'utf8', (err, data) = return respond(false) } }) - } }) }) diff --git a/lib/Cloudflare.js b/lib/Cloudflare.js index 4411859..a7e69a4 100644 --- a/lib/Cloudflare.js +++ b/lib/Cloudflare.js @@ -8,8 +8,6 @@ const download = require('./RequestDownload') // Cloudflare module module.exports = (auth) => { - console.log('run here') - // Setup axios client with api key const cloudflareClient = axios.create({ baseURL: `https://api.cloudflare.com/client/v4/accounts/${auth.accountId}`, @@ -18,15 +16,11 @@ module.exports = (auth) => { // Function to Compress image return function (imageFile, __callback = () => {}) { - console.log('run here too', imageFile) - // API Payload try { const fData = new FormData() - let options = { - file: imageFile - } + fData.append('file', imageFile.buffer, { filename: imageFile.filename }) // Force timeout with 20s let callbackSent = false @@ -47,15 +41,12 @@ module.exports = (auth) => { }, 30000) // Upload image to cloudflare - console.log('[before upload]') cloudflareClient({ method: 'POST', url: '/images/v1', headers: { ...fData.getHeaders() }, - data: options + data: fData }).then((response) => { - console.log('[after upload]',response) - // Retrieve Upload data const id = response.data.result.id const variants = response.data.result.variants @@ -87,12 +78,10 @@ module.exports = (auth) => { } }) }).catch((err) => { - console.log('[here with error]', err.response.data) callback(err) }) } catch (err) { - console.log('[error]', err) __callback(err, null) }