Google cloud storage util (#395)

Close #404

Author: munkhorgil

.env.sample

@@ -58,7 +58,16 @@ GOOGLE_SUBSCRIPTION_NAME=''
 GOOGLE_PROJECT_ID=''
 GMAIL_REDIRECT_URL = http://localhost:3300/
 
+# Google Cloud Storage
+GOOGLE_CLOUD_STORAGE_BUCKET=''
+
 ERXES_PATH=''
 API_PATH=''
 WIDGET_PATH=''
 WIDGET_API_PATH=''
+
+# AWS | GCS
+UPLOAD_SERVICE_TYPE='AWS'
+
+# File system: true | ''
+FILE_SYSTEM_PUBLIC='true'

.gitignore

@@ -12,5 +12,5 @@ dump.rdb
 *.swo
 .migrate
 
-# Firebase config key
-serviceAccount.json
+# Google config file
+google_cred.json

serviceAccount.json.sample google_cred.json.sampleserviceAccount.json.sample renamed to google_cred.json.sample

@@ -46,6 +46,7 @@
   },
   "dependencies": {
     "@google-cloud/pubsub": "0.18.0",
+    "@google-cloud/storage": "^2.5.0",
     "apollo-server-express": "^2.3.1",
     "aws-sdk": "^2.151.0",
     "bcrypt": "^3.0.2",

package.json

@@ -1,3 +1,4 @@
+import { Storage } from '@google-cloud/storage';
 import * as AWS from 'aws-sdk';
 import * as EmailValidator from 'email-deep-validator';
 import * as fileType from 'file-type';
@@ -50,20 +51,54 @@ export const checkFile = async file => {
   return 'ok';
 };
 
-/*
- * Save binary data to amazon s3
+/**
+ * Create AWS instance
  */
-export const uploadFile = async (file: { name: string; path: string }): Promise<string> => {
+const createAWS = () => {
   const AWS_ACCESS_KEY_ID = getEnv({ name: 'AWS_ACCESS_KEY_ID' });
   const AWS_SECRET_ACCESS_KEY = getEnv({ name: 'AWS_SECRET_ACCESS_KEY' });
   const AWS_BUCKET = getEnv({ name: 'AWS_BUCKET' });
-  const AWS_PREFIX = getEnv({ name: 'AWS_PREFIX', defaultValue: '' });
+
+  if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY || !AWS_BUCKET) {
+    throw new Error('AWS credentials are not configured');
+  }
 
   // initialize s3
-  const s3 = new AWS.S3({
+  return new AWS.S3({
     accessKeyId: AWS_ACCESS_KEY_ID,
     secretAccessKey: AWS_SECRET_ACCESS_KEY,
   });
+};
+
+/**
+ * Create Google Cloud Storage instance
+ */
+const createGCS = () => {
+  const GOOGLE_APPLICATION_CREDENTIALS = getEnv({ name: 'GOOGLE_APPLICATION_CREDENTIALS' });
+  const GOOGLE_PROJECT_ID = getEnv({ name: 'GOOGLE_PROJECT_ID' });
+  const BUCKET = getEnv({ name: 'GOOGLE_CLOUD_STORAGE_BUCKET' });
+
+  if (!GOOGLE_PROJECT_ID || !GOOGLE_APPLICATION_CREDENTIALS || !BUCKET) {
+    throw new Error('Google Cloud Storage credentials are not configured');
+  }
+
+  // initializing Google Cloud Storage
+  return new Storage({
+    projectId: GOOGLE_PROJECT_ID,
+    keyFilename: GOOGLE_APPLICATION_CREDENTIALS,
+  });
+};
+
+/*
+ * Save binary data to amazon s3
+ */
+export const uploadFileAWS = async (file: { name: string; path: string }): Promise<string> => {
+  const AWS_BUCKET = getEnv({ name: 'AWS_BUCKET' });
+  const AWS_PREFIX = getEnv({ name: 'AWS_PREFIX', defaultValue: '' });
+  const IS_PUBLIC = getEnv({ name: 'FILE_SYSTEM_PUBLIC', defaultValue: 'true' });
+
+  // initialize s3
+  const s3 = createAWS();
 
   // generate unique name
   const fileName = `${AWS_PREFIX}${Math.random()}${file.name}`;
@@ -78,7 +113,7 @@ export const uploadFile = async (file: { name: string; path: string }): Promise<
         Bucket: AWS_BUCKET,
         Key: fileName,
         Body: buffer,
-        ACL: 'public-read',
+        ACL: IS_PUBLIC === 'true' ? 'public-read' : undefined,
       },
       (err, res) => {
         if (err) {
@@ -90,7 +125,102 @@ export const uploadFile = async (file: { name: string; path: string }): Promise<
     );
   });
 
-  return response.Location;
+  return IS_PUBLIC === 'true' ? response.Location : fileName;
+};
+
+/*
+ * Save file to google cloud storage
+ */
+export const uploadFileGCS = async (file: { name: string; path: string; type: string }): Promise<string> => {
+  const BUCKET = getEnv({ name: 'GOOGLE_CLOUD_STORAGE_BUCKET' });
+  const IS_PUBLIC = getEnv({ name: 'FILE_SYSTEM_PUBLIC', defaultValue: 'true' });
+
+  // initialize GCS
+  const storage = createGCS();
+
+  // select bucket
+  const bucket = storage.bucket(BUCKET);
+
+  // generate unique name
+  const fileName = `${Math.random()}${file.name}`;
+
+  bucket.file(fileName);
+
+  const response: any = await new Promise((resolve, reject) => {
+    bucket.upload(
+      file.path,
+      {
+        metadata: { contentType: file.type },
+        public: IS_PUBLIC === 'true',
+      },
+      (err, res) => {
+        if (err) {
+          return reject(err);
+        }
+
+        if (res) {
+          return resolve(res);
+        }
+      },
+    );
+  });
+
+  const { metadata, name } = response;
+
+  return IS_PUBLIC === 'true' ? metadata.mediaLink : name;
+};
+
+/**
+ * Read file from GCS, AWS
+ */
+export const readFileRequest = async (key: string): Promise<any> => {
+  const UPLOAD_SERVICE_TYPE = getEnv({ name: 'UPLOAD_SERVICE_TYPE', defaultValue: 'AWS' });
+
+  if (UPLOAD_SERVICE_TYPE === 'GCS') {
+    const GCS_BUCKET = getEnv({ name: 'GOOGLE_CLOUD_STORAGE_BUCKET' });
+    const storage = createGCS();
+
+    const bucket = storage.bucket(GCS_BUCKET);
+
+    const file = bucket.file(key);
+
+    // get a file buffer
+    const [contents] = await file.download({});
+
+    return contents;
+  }
+
+  const AWS_BUCKET = getEnv({ name: 'AWS_BUCKET' });
+  const s3 = createAWS();
+
+  return new Promise((resolve, reject) => {
+    s3.getObject(
+      {
+        Bucket: AWS_BUCKET,
+        Key: key,
+      },
+      (error, response) => {
+        if (error) {
+          return reject(error);
+        }
+
+        return resolve(response.Body);
+      },
+    );
+  });
+};
+
+/*
+ * Save binary data to amazon s3
+ */
+export const uploadFile = async (file): Promise<string> => {
+  const UPLOAD_SERVICE_TYPE = getEnv({ name: 'UPLOAD_SERVICE_TYPE', defaultValue: 'AWS' });
+
+  if (UPLOAD_SERVICE_TYPE === 'AWS') {
+    return uploadFileAWS(file);
+  }
+
+  return uploadFileGCS(file);
 };
 
 /**

src/data/utils.ts

@@ -12,7 +12,7 @@ import { userMiddleware } from './auth';
 import resolvers from './data/resolvers';
 import { handleEngageUnSubscribe } from './data/resolvers/mutations/engageUtils';
 import typeDefs from './data/schema';
-import { checkFile, getEnv, importXlsFile, uploadFile } from './data/utils';
+import { checkFile, getEnv, importXlsFile, readFileRequest, uploadFile } from './data/utils';
 import { connect } from './db/connection';
 import { Conversations, Customers } from './db/models';
 import { graphqlPubsub } from './pubsub';
@@ -27,13 +27,13 @@ const MAIN_APP_DOMAIN = getEnv({ name: 'MAIN_APP_DOMAIN', defaultValue: '' });
 const WIDGETS_DOMAIN = getEnv({ name: 'WIDGETS_DOMAIN', defaultValue: '' });
 
 // firebase app initialization
-fs.exists(path.join(__dirname, '..', '/serviceAccount.json'), exists => {
+fs.exists(path.join(__dirname, '..', '/google_cred.json'), exists => {
   if (!exists) {
     return;
   }
 
   const admin = require('firebase-admin').default;
-  const serviceAccount = require('../serviceAccount.json');
+  const serviceAccount = require('../google_cred.json');
   const firebaseServiceAccount = serviceAccount;
 
   if (firebaseServiceAccount.private_key) {
@@ -196,6 +196,25 @@ app.get('/status', async (_req, res) => {
   res.end('ok');
 });
 
+// read file
+app.get('/read-file', async (req: any, res) => {
+  const key = req.query.key;
+
+  if (!key) {
+    return res.send('Invalid key');
+  }
+
+  try {
+    const response = await readFileRequest(key);
+
+    res.attachment(key);
+
+    return res.send(response);
+  } catch (e) {
+    return res.end(e.message);
+  }
+});
+
 // file upload
 app.post('/upload-file', async (req, res) => {
   const form = new formidable.IncomingForm();

src/index.ts

@@ -151,7 +151,7 @@
     through2 "^2.0.3"
     uuid "^3.1.0"
 
-"@google-cloud/storage@^2.3.0":
+"@google-cloud/storage@^2.3.0", "@google-cloud/storage@^2.5.0":
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/@google-cloud/storage/-/storage-2.5.0.tgz#9dd3566d8155cf5ba0c212208f69f9ecd47fbd7e"
   dependencies: