Replace list buckets with HEAD buckets

carboneio · Apr 30, 2024 · f68d735 · f68d735
1 parent 05c3c4c
commit f68d735
Show file tree

Hide file tree

Showing 5 changed files with 79 additions and 101 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.4.0
+- Released the 2024/04/30
+- S3 Bucket Connection is not listing Buckets anymore, but only verifying if `templatesBucket` and `rendersBucket` are accessible with a `HEAD /bucket` request. If options `BUCKET_TEMPLATES/templatesBucket` or `BUCKET_RENDER/rendersBucket` are missing, it won't try to connect.
+
 ## 1.3.0
 - Released the 2024/04/29
 - Fixed S3 Credential as Environment Variable

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "carbone-ee-plugin-s3",
-  "version": "1.2.1",
+  "version": "1.4.0",
   "description": "S3 Plugin for Carbone On-premise & On-AWS (carbone-ee) ",
   "main": "null",
   "scripts": {

diff --git a/storage.js b/storage.js
@@ -11,11 +11,15 @@ let s3 = {};
 if (_config?.storageCredentials) {
   s3 = require('tiny-storage-client')(_config.storageCredentials);
   s3.setTimeout(30000)
-  connection((err) => {
+  connection("Templates", (err) => {
     if (err) {
-      console.log("🔴 S3 connection error:", err.toString());
-      process.exit(1);
+      console.log("🔴 S3 Connection |", err.toString());
     }
+    connection("Renders", (err) => {
+      if (err) {
+        console.log("🔴 S3 Connection |", err.toString());
+      }
+    })
   })
 }
 
@@ -175,27 +179,21 @@ module.exports = {
  * 
  * @param {function} callback (err) => {} 
  */
-function connection(callback) {
-  s3.listBuckets((err, res) => {
-    if (err) {
-      return callback(err);
-    }
-    /** If the connection gets an error, the storage-client may have switched to another storage, the activeStorage must be 0 */
-    if (s3.getConfig().activeStorage !== 0) {
-      return callback(new Error('Something went wrong when connecting to the S3.'));
-
-    }
-    logListBuckets(err, res, s3.getConfig()?.storages[0]);
-    return callback();
-  })
-}
+function connection(type, callback) {
+  const _keyName = type === 'Templates' ? 'templatesBucket' : 'rendersBucket';
 
-function logListBuckets(err, res, store) {
-  if(err) {
-    console.log( "🔴 Storage error " + store?.url + " | " + store?.region + " | Error: " + err?.toString());
-  } else if (res?.statusCode !== 200) {
-    console.log( "🔴 Storage error " + store?.url + " | " + store?.region + " | Status" + res?.statusCode + '| Response: ' + res?.body);
+  if (config.getConfig()?.[_keyName]) {
+    return s3.headBucket(config.getConfig()?.[_keyName], function(err, resp) {
+      if (err) {
+        return callback(new Error(`${type} S3 Bucket Connection | ${config.getConfig()?.[_keyName]} | ${err.toString()}`));
+      }
+      if (resp?.statusCode !== 200) {
+        return callback(new Error(`${type} S3 Bucket Connection | ${config.getConfig()?.[_keyName]} | Status ` + resp?.statusCode + ' | Response: ' + resp?.body?.error?.message ?? resp?.body ));
+      }
+      console.log(`${type} S3 Bucket Connected | ${config.getConfig()?.[_keyName]} | Status ${resp?.statusCode}`);
+      return callback();
+    })
   } else {
-    console.log(`Storage ok | ` + store?.url + " | " + store?.region + " | Status " + res?.statusCode + ' | Buckets: ' + res.body?.bucket?.reduce((total, val) => total += '[ ' + val?.name + ' ]', ''));
+    return callback();
   }
-}
+}
diff --git a/test/config.js b/test/config.js
@@ -4,6 +4,13 @@ const assert = require('assert');
 describe('Config', function () {
 
     it('should return an empty object if config.json does not exist, and environment variables are not available.', function (done) {
+        delete process.env.AWS_SECRET_ACCESS_KEY;
+        delete process.env.AWS_ACCESS_KEY_ID;
+        delete process.env.AWS_ENDPOINT_URL;
+        delete process.env.AWS_REGION
+        delete process.env.BUCKET_RENDERS;
+        delete process.env.BUCKET_TEMPLATES;
+        config.setConfig({});
         const _config = config.getConfig();
         assert.strictEqual(JSON.stringify(_config), '{}');
         done();

diff --git a/test/storage.js b/test/storage.js
@@ -6,18 +6,6 @@ const config = require('../config');
 
 const _rendersBucket = 'renders-bucket';
 const _templatesBucket = 'templates-bucket';
-const _config = {
-  storageCredentials : {
-      accessKeyId     : 'accessKeyId',
-      secretAccessKey : 'secretAccessKey',
-      url             : 's3.gra.first.cloud.test',
-      region          : 'gra'
-  },
-  rendersBucket  : _rendersBucket,
-  templatesBucket: _templatesBucket,
-  templatePath: path.join(__dirname, 'datasets'),
-  renderPath: path.join(__dirname, 'datasets')
-}
 
 const pathFileTxt = path.join(__dirname, 'datasets', 'file.txt');
 
@@ -26,16 +14,31 @@ const url1S3 = 'https://s3.gra.first.cloud.test';
 describe('Storage', function () {
   let storage = null;
 
-  this.beforeAll(function(done) {
-    config.setConfig(_config)
-    done();
-  })
-
   before(function (done) {
+
+    config.setConfig({
+      storageCredentials : {
+          accessKeyId     : 'accessKeyId',
+          secretAccessKey : 'secretAccessKey',
+          url             : 's3.gra.first.cloud.test',
+          region          : 'gra'
+      },
+      rendersBucket  : _rendersBucket,
+      templatesBucket: _templatesBucket,
+      templatePath: path.join(__dirname, 'datasets'),
+      renderPath: path.join(__dirname, 'datasets')
+    });
+
+    const _listFilesResponse = `<?xml version='1.0' encoding='UTF-8'?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>Bucket</Name><Prefix/><KeyCount>4</KeyCount><MaxKeys>1</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>file-1.docx</Key><LastModified>2023-03-07T17:03:54.000Z</LastModified><ETag>"7ad22b1297611d62ef4a4704c97afa6b"</ETag><Size>61396</Size><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>`;
+
     /** S3 Call for testing storage connection */
     nock(url1S3)
-      .get('/')
-      .reply(200, '<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>89123456:user-feiowjfOEIJW</ID><DisplayName>12345678:user-feiowjfOEIJW</DisplayName></Owner><Buckets><Bucket><Name>templates-bucket</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket><Bucket><Name>renders-bucket</Name><CreationDate>2023-02-27T11:46:24.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>');
+      .intercept(`/${_templatesBucket}`, "HEAD")
+      .reply(200, _listFilesResponse);
+
+    nock(url1S3)
+      .intercept(`/${_rendersBucket}`, "HEAD")
+      .reply(200, _listFilesResponse);
 
     storage = require('../storage');
     setTimeout(done, 500);
@@ -62,6 +65,7 @@ describe('Storage', function () {
         .reply(403, '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message>Access Denied.</Message><RequestId>tx439620795cdd41b08c58c-0064186222</RequestId></Error>');
 
       storage.writeTemplate({}, {}, 'templateId', pathFileTxt, (err) => {
+        console.log(err);
         assert.strictEqual(err.toString().includes(403), true);
         assert.strictEqual(err.toString().includes('AccessDenied'), true);
         done();
@@ -79,18 +83,6 @@ describe('Storage', function () {
         done();
       });
     });
-
-    it('should write template but should not send to s3 if the configuration `templatesBucket` does not exist', (done) => {
-
-      config.setConfig(null);
-
-      storage.writeTemplate({}, {}, 'templateId', pathFileTxt, (err, templateName) => {
-        assert.strictEqual(err, null);
-        assert.strictEqual(templateName, 'templateId');
-        config.setConfig(_config);
-        done();
-      });
-    });
   });
 
   describe('Read template', () => {
@@ -124,18 +116,6 @@ describe('Storage', function () {
       });
     });
 
-    it('should read a template even if the "templatesBucket" does not exist', (done) => {
-
-      config.setConfig(null);
-
-      storage.readTemplate({ }, {}, 'template.odt', (err, templatePath) => {
-        assert.strictEqual(err, null);
-        assert.strictEqual(path.basename(templatePath), 'template.odt');
-        config.setConfig(_config);
-        done();
-      });
-    });
-
     it('should return an error if s3 return an error 403', (done) => {
       nock(url1S3)
         .get(uri => uri.includes(`/${_templatesBucket}/`))
@@ -185,24 +165,19 @@ describe('Storage', function () {
         .delete(uri => uri.includes(`/${_templatesBucket}`))
         .reply(200);
 
-      storage.deleteTemplate({}, {}, 'template.docx', (err, templatePath) => {
-        assert.strictEqual(err, null);
-        assert.strictEqual(templatePath.endsWith('/test/datasets/template.docx'), true);
-        done();
-      });
-    });
-
-
-    it('should delete the template even if the "templatesBucket" option does not exist', (done) => {
-
-      const _configCopy = {..._config}
-      delete _configCopy.templatesBucket;
-      config.setConfig(_configCopy);
+      const res = {
+        send (result) {
+          assert.deepStrictEqual(result, {
+            success : true,
+            message : 'Template deleted'
+          });
+          done();
+        }
+      };
 
-      storage.deleteTemplate({}, {}, 'template.docx', (err, templatePath) => {
+      storage.deleteTemplate({}, res, 'template.docx', (err, templatePath) => {
         assert.strictEqual(err, null);
         assert.strictEqual(templatePath.endsWith('/test/datasets/template.docx'), true);
-        config.setConfig(_config);
         done();
       });
     });
@@ -251,19 +226,6 @@ describe('Storage', function () {
         });
     });
 
-    it('should save a generated doccument locally even if the option "rendersBucket" does not exist', function(done) {
-
-      const _configCopy = {..._config}
-      delete _configCopy.rendersBucket;
-      config.setConfig(_configCopy);
-
-      storage.afterRender({}, {}, null, pathFileTxt, _renderName, {}, (err) => {
-          assert.strictEqual(err, undefined);
-          config.setConfig(_config);
-          done();
-      });
-  });
-
     it('should return an error if the rendering failled', function(done) {
         storage.afterRender({}, {}, new Error('Something went wrong'), pathFileTxt, _renderName, {}, (err) => {
             assert.strictEqual(err.toString(), 'Error: Something went wrong');
@@ -320,23 +282,30 @@ describe('Storage', function () {
         storage.readRender({}, {}, _renderID2, function(err, renderPath) {
             assert.strictEqual(null, err);
             assert.strictEqual(renderPath.includes('datasets/' + _renderID2), true)
+            console.log(renderPath);
             toDelete.push(renderPath);
             done();
         });
     });
 
-    it('should return the renderPath even if the "rendersBucket" option does not exist', function(done) {
+    it('should download and delete the generated document from s3', function(done) {
 
         const _renderID = '89rf2jd9302jf329sok.pdf';
 
-        const _configCopy = {..._config}
-        delete _configCopy.rendersBucket;
-        config.setConfig(_configCopy);
+        nock(url1S3)
+            .get(uri => uri.includes(`/${_rendersBucket}/${_renderID}`))
+            .reply(200, () => {
+                return fs.createReadStream(pathFileTxt);
+            });
+
+        nock(url1S3)
+            .delete(uri => uri.includes(`/${_rendersBucket}/${_renderID}`))
+            .reply(200);
 
         storage.readRender({}, {}, _renderID, function(err, renderPath) {
             assert.strictEqual(null, err);
             assert.strictEqual(renderPath.includes('datasets/' + _renderID), true)
-            config.setConfig(_config);
+            toDelete.push(renderPath);
             done();
         });
     });