Merge 6a32ae9 into 384f5b8

cloud/filestore/package.json

@@ -10,7 +10,7 @@
     "clean": "del coverage_report src/**/*.js src/**/*.map test/**/*.js test/**/*.map",
     "build": "tsc",
     "start": "ts-node src/index.ts",
-    "test": "npm run clean && nyc mocha"
+    "test": "nyc mocha"
   },
   "nyc": {
     "include": [
@@ -27,22 +27,26 @@
       "text"
     ],
     "report-dir": "coverage_report",
-    "check-coverage": true,
     "lines": 75,
-    "functions": 100,
-    "branches": 80
+    "functions": 75,
+    "branches": 50
   },
   "devDependencies": {
     "@types/bluebird": "^3.5.16",
+    "@types/del": "^3.0.0",
     "@types/gridfs-stream": "^0.5.30",
+    "@types/mkdirp": "^0.5.1",
     "@types/mocha": "^2.2.41",
+    "@types/multer": "^1.3.5",
     "@types/node": "^8.0.46",
     "@types/proxyquire": "^1.3.27",
+    "del": "^3.0.0",
     "del-cli": "^1.0.0",
-    "mocha": "^3.4.2",
+    "mocha": "^3.5.3",
     "nyc": "^11.0.1",
     "proxyquire": "^1.8.0",
     "source-map-support": "^0.4.15",
+    "string-to-stream": "^1.1.0",
     "ts-node": "^3.0.4",
     "typescript": "^2.3.4"
   },
@@ -53,6 +57,7 @@
     "express": "4.15.2",
     "gridfs-stream": "^1.1.1",
     "lodash": "4.17.4",
+    "mkdirp": "^0.5.1",
     "mongodb": "^2.2.25",
     "multer": "1.1.0",
     "q": "1.4.1",

cloud/filestore/src/impl/GridFsStorage.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@raincatcher/logger';
-import * as BlueBird from 'bluebird';
+import * as Promise from 'bluebird';
 import * as fs from 'fs';
 import * as gridfs from 'gridfs-stream';
 import * as mongo from 'mongodb';
@@ -8,38 +8,45 @@ import { Stream } from 'stream';
 import { FileMetadata } from '../file-api/FileMetadata';
 import { FileStorage } from '../file-api/FileStorage';
 
+const connectAsync = Promise.promisify<mongo.Db, string>(MongoClient.connect);
+
 /**
  * Reference implementation for GridFsStorage.
  * See MongoDB documentation for more details.
  */
 export class GridFsStorage implements FileStorage {
 
-  private gridFileSystem: gridfs.Grid;
+  public gridFileSystem: gridfs.Grid;
+  private fileSystemPromise: Promise<gridfs.Grid>;
 
   /**
    * Creates instance of the GridFsStorage that will connect to specified mongo url
    * @param mongoConnectionURl - MongoDB connection url
    */
-  constructor(mongoConnectionUrl: string) {
-    const self = this;
-    MongoClient.connect(mongoConnectionUrl, function(err, connection) {
-      if (err) {
-        getLogger().error('Cannot connect to mongodb server. Gridfs storage will be disabled');
-        return;
-      }
-      self.gridFileSystem = gridfs(connection, mongo);
-    });
+  constructor(connection: string | mongo.Db) {
+    if (typeof connection === 'string') {
+      this.fileSystemPromise = connectAsync(connection)
+        .then(conn => this.gridFileSystem = gridfs(connection, mongo))
+        .tapCatch(() => getLogger().error('Cannot connect to mongodb server. Gridfs storage will be disabled'));
+    } else {
+      this.gridFileSystem = gridfs(connection, mongo);
+      this.fileSystemPromise = Promise.resolve(this.gridFileSystem);
+    }
+  }
+
+  public getFileSystem() {
+    return this.fileSystemPromise;
   }
 
   public writeFile(metadata: FileMetadata, fileLocation: string): Promise<string> {
     const self = this;
     if (!self.gridFileSystem) {
-      return BlueBird.reject('Not initialized');
+      return Promise.reject('Not initialized');
     }
     const options = {
       filename: metadata.id
     };
-    return new BlueBird(function(resolve, reject) {
+    return new Promise(function(resolve, reject) {
       const writeStream = self.gridFileSystem.createWriteStream(options);
       writeStream.on('error', function(err) {
         getLogger().error('An error occurred!', err);
@@ -55,12 +62,12 @@ export class GridFsStorage implements FileStorage {
   public readFile(id: string): Promise<Stream> {
     const self = this;
     if (!self.gridFileSystem) {
-      return BlueBird.reject('Not initialized');
+      return Promise.reject('Not initialized');
     }
     const options = {
       filename: id
     };
-    return new BlueBird(function(resolve, reject) {
+    return new Promise(function(resolve, reject) {
       const readstream = self.gridFileSystem.createReadStream(options);
       readstream.on('error', function(err) {
         getLogger().error('An error occurred when reading file from gridfs!', err);

cloud/filestore/src/impl/S3Storage.ts

@@ -30,7 +30,7 @@ export interface S3StorageConfiguration {
  * Implementation that using server filesystem to store files.
  * This storage is not executing any actions as files are already stored in the disc drive.
  */
-export class LocalStorage implements FileStorage {
+export class S3Storage implements FileStorage {
   private awsClient: any;
   private storageConfig: S3StorageConfiguration;
   /**

cloud/filestore/src/services/FileService.ts

@@ -1,16 +1,18 @@
 import { getLogger } from '@raincatcher/logger';
 import * as base64 from 'base64-stream';
+import * as Promise from 'bluebird';
 import { Request } from 'express';
 import * as fs from 'fs';
-import multer = require('multer');
-import * as  os from 'os';
+import * as mkdirp from 'mkdirp';
+import * as multer from 'multer';
+import * as os from 'os';
 import { join } from 'path';
 import * as path from 'path';
-import q from 'q';
 import { Stream } from 'stream';
-import through from 'through2';
 import { FileMetadata } from '../file-api/FileMetadata';
 
+const mkdirpAsync = Promise.promisify<string, string>(mkdirp);
+
 /**
  * Location of the stored files in the server's local disk
  * Uses the OS' temporary directory
@@ -21,50 +23,44 @@ export const FILE_STORAGE_DIRECTORY = path.join(os.tmpdir(), '/raincatcher-file-
  * Create temporary storage folder used by multer to store files before
  * uploading to permanent storage
  */
-export function createTemporaryStorageFolder() {
-  fs.mkdir(FILE_STORAGE_DIRECTORY, '0775', function(err: any) {
-    if (err && err.code !== 'EEXIST') {
-      getLogger().error(err);
-      throw new Error(err);
-    }
-  });
+export function createTemporaryStorageFolder(directory: string = FILE_STORAGE_DIRECTORY): Promise<string> {
+  return mkdirpAsync(directory).then(() => directory);
 }
 
 /**
  * Utility function for saving file in temp folder
  * @param metadata
  * @param stream
  */
-export function writeStreamToFile(metadata: FileMetadata, stream: Stream) {
-  const deferred = q.defer();
-  stream.on('end', function() {
-    deferred.resolve(metadata);
-  });
-  stream.on('error', function(error) {
-    deferred.reject(error);
+export function writeStreamToFile(metadata: FileMetadata, stream: Stream): Promise<FileMetadata> {
+  return new Promise((resolve, reject) => {
+    stream.on('end', function() {
+      resolve(metadata);
+    });
+    stream.on('error', function(error) {
+      reject(error);
+    });
+    const filename = buildFilePath(metadata.id);
+    stream.pipe(fs.createWriteStream(filename));
   });
-  const filename = path.join(FILE_STORAGE_DIRECTORY, metadata.id);
-  stream.pipe(fs.createWriteStream(filename));
-  return deferred.promise;
 }
 
 /**
  * Returns the full path to a file stored with the service
  * @param fileName Name of the file to build a path for, usually the file's id
  */
-export function buildFilePath(fileName) {
-  return path.join(FILE_STORAGE_DIRECTORY, fileName);
+export function buildFilePath(fileName: string, root: string = FILE_STORAGE_DIRECTORY) {
+  return path.join(root, fileName);
 }
 
-/**
- * Returns a new multer-based middleware that's capable of processing the `multipart/form-data` uploads
- */
-export function multerMiddleware() {
+const diskStorageDefaultOptions = {
+  destination(req, file, cb) {
+    cb(null, FILE_STORAGE_DIRECTORY);
+  }
+};
+
+export function multerMiddleware(storage: multer.StorageEngine = multer.diskStorage(diskStorageDefaultOptions)) {
   return multer({
-    storage: multer.diskStorage({
-      destination(req, file, cb) {
-        cb(null, FILE_STORAGE_DIRECTORY);
-      }
-    })
+    storage
   });
 }

cloud/filestore/test/fixtures/test.txt

@@ -0,0 +1 @@
+hello world

cloud/filestore/test/impl/GridFsStorage-spec.ts

@@ -0,0 +1,62 @@
+import * as Promise from 'bluebird';
+import { expect } from 'chai';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+import { readFileSync } from 'fs';
+import * as mongo from 'mongodb';
+import { MongoClient } from 'mongodb';
+import * as path from 'path';
+import { FileMetadata } from '../../src/file-api/FileMetadata';
+import { GridFsStorage } from '../../src/impl/GridFsStorage';
+
+const connectAsync = Promise.promisify<mongo.Db, string>(MongoClient.connect);
+
+chai.use(chaiAsPromised);
+
+function readStream(stream: NodeJS.ReadableStream): Promise<string> {
+  let data = '';
+  stream.setEncoding('utf8');
+  stream.on('data', chunk => data += chunk);
+  return new Promise(resolve => stream.on('end', () => resolve(data)));
+}
+
+describe('GridFsStorage', function() {
+  const mongoUrl = 'mongodb://127.0.0.1:27017/testdb';
+  const dbFactory = () => connectAsync(mongoUrl);
+  let db: mongo.Db;
+  const metadata: FileMetadata = {
+    id: 'test-file'
+  };
+  const fileLocation = path.resolve(__dirname, '../fixtures/test.txt');
+  const fileContents = readFileSync(fileLocation, {
+    encoding: 'utf8'
+  });
+
+  before(function() {
+    return dbFactory().then(database => db = database);
+  });
+  after(function() {
+    db.close();
+  });
+
+  describe('constructor', function() {
+    it('should accept a mongodb connection', function() {
+      const storage = new GridFsStorage(db);
+      return expect(storage.gridFileSystem).to.exist &&
+        expect(storage.getFileSystem()).to.eventually.be.fulfilled;
+    });
+    it('should accept a mongodb url', function() {
+      const storage = new GridFsStorage(mongoUrl);
+    });
+  });
+
+  it('should write and read back a file from storage', function() {
+    const storage = new GridFsStorage(db);
+    const write = storage.writeFile(metadata, fileLocation);
+    const read = write
+      .then(() => storage.readFile(metadata.id))
+      .then(readStream);
+    return expect(write).to.eventually.be.fulfilled &&
+      expect(read).to.eventually.equal(fileContents);
+  });
+});

cloud/filestore/test/mocha.opts

@@ -1,3 +1,3 @@
 --compilers ts:ts-node/register
 --require source-map-support/register
-test/**.ts
+test/**/*.ts

cloud/filestore/test/services/FileService-spec.ts

@@ -0,0 +1,78 @@
+import * as Promise from 'bluebird';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+import * as del from 'del';
+import * as fs from 'fs';
+import { memoryStorage } from 'multer';
+import { Stream } from 'stream';
+import { FileMetadata } from '../../src/file-api/FileMetadata';
+import * as fileService from '../../src/services/FileService';
+
+import strToStream = require('string-to-stream');
+
+chai.use(chaiAsPromised);
+
+const existsAsync = (path) =>
+  // can't use promisify because this doesn't return an error
+  new Promise(resolve => fs.exists(path, exists => resolve(exists)));
+const readFileAsync = Promise.promisify<string, string, string>(fs.readFile);
+
+const { expect } = chai;
+
+describe('FileService', function() {
+  describe('createTemporaryStorageFolder', function() {
+    const deleteDir = () => del(fileService.FILE_STORAGE_DIRECTORY, { force: true });
+    beforeEach(() => deleteDir());
+
+    it('should create a default directory', function() {
+      const recreate = deleteDir()
+        .then(() => fileService.createTemporaryStorageFolder())
+        .then(() => existsAsync(fileService.FILE_STORAGE_DIRECTORY));
+      return expect(recreate).to.eventually.be.true;
+    });
+    it('should be idempotent', function() {
+      const createTwice = fileService.createTemporaryStorageFolder()
+        .then(() => fileService.createTemporaryStorageFolder())
+        .then(() => existsAsync(fileService.FILE_STORAGE_DIRECTORY));
+      return expect(createTwice).to.eventually.be.true;
+    });
+
+    after(() => deleteDir());
+  });
+  describe('writeStreamToFile', function() {
+    const metadata: FileMetadata = {
+      id: 'some-uuid'
+    };
+    const testContent = 'test';
+    const contentStream = strToStream(testContent);
+    const filePath = fileService.buildFilePath(metadata.id);
+
+    it('should write data to the file identified by id', function() {
+      const writeAndRead = fileService.createTemporaryStorageFolder()
+        .then(() => fileService.writeStreamToFile(metadata, contentStream))
+        .then(() => readFileAsync(filePath, 'utf-8'));
+      return expect(writeAndRead).to.eventually.equal(testContent);
+    });
+
+    after(function() {
+      // reset written file
+      return del(filePath, { force: true });
+    });
+  });
+  describe('buildFilePath', function() {
+    it('should return a relative path to the root directory', function() {
+      const value = fileService.buildFilePath('fake-file-id');
+      expect(value).to.contain(fileService.FILE_STORAGE_DIRECTORY);
+    });
+  });
+  describe('multerMiddleware', function() {
+    it('should return a middleware', function() {
+      const middleware = fileService.multerMiddleware().any();
+      expect(middleware).to.be.a('function');
+    });
+    it('should accept other storage implementations', function() {
+      const middleware = fileService.multerMiddleware(memoryStorage()).any();
+      expect(middleware).to.be.a('function');
+    });
+  });
+});