RAINCATCH-1351 - File Server

cloud/datasync/example/README.md

@@ -4,7 +4,7 @@ Example for fh-sync cloud module integration.
 
 ## Running
 
-    ts-node ./integration/index.ts
+    ts-node ./example/index.ts
 
 ## Requirements
 

cloud/filestore/.gitignore

@@ -0,0 +1,6 @@
+# generated code
+src/**/*.js
+src/**/*.map
+test/**/*.js
+test/**/*.map
+coverage_report/

cloud/filestore/.npmignore

@@ -0,0 +1 @@
+test/

cloud/filestore/README.md

@@ -0,0 +1,3 @@
+# RainCatcher FileStore
+
+ WIP TODO

cloud/filestore/example/README.md

@@ -0,0 +1,16 @@
+## File store server
+
+File
+
+## Running
+
+    ts-node ./example/index.ts
+
+## Requirements
+
+`MONGO_CONNECTION_URL` environment variable that points to a mongodb instance.
+By default using: mongodb://127.0.0.1:27017/sync
+
+`REDIS_HOST` and `REDIS_PORT` environment variables that points to a running redis instance.
+By default using: 127.0.0.1 and 6379
+

cloud/filestore/example/index.ts

@@ -0,0 +1,26 @@
+import { getLogger } from '@raincatcher/logger';
+import { createRouter, FileMetadata, FileStorage, LocalStorage } from '../src/index';
+
+const logger = getLogger();
+
+// Create express middleware
+import * as bodyParser from 'body-parser';
+import * as cors from 'cors';
+import * as express from 'express';
+
+const app = express();
+
+// middleware
+app.use(bodyParser.json());
+app.use(cors());
+
+const engine: FileStorage = new LocalStorage();
+const router = createRouter(engine);
+
+app.use('/', router);
+
+app.listen(3000, function() {
+  logger.info('Example app listening on port 3000!');
+});
+// If you wish to see logs;
+process.env.DEBUG = 'fh-mbaas-api:sync';

cloud/filestore/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@raincatcher/filestore",
+  "version": "1.0.0",
+  "description": "RainCatcher file storage server",
+  "types": "src/index.ts",
+  "author": "feedhenry-raincatcher@redhat.com",
+  "license": "Apache-2.0",
+  "main": "src/",
+  "scripts": {
+    "clean": "del coverage_report src/**/*.js src/**/*.map test/**/*.js test/**/*.map",
+    "build": "tsc",
+    "start": "ts-node src/index.ts",
+    "test": "npm run clean && nyc mocha"
+  },
+  "nyc": {
+    "include": [
+      "src/**/*.ts"
+    ],
+    "extension": [
+      ".ts"
+    ],
+    "require": [
+      "ts-node/register"
+    ],
+    "reporter": [
+      "lcov",
+      "text"
+    ],
+    "report-dir": "coverage_report",
+    "check-coverage": true,
+    "lines": 75,
+    "functions": 100,
+    "branches": 80
+  },
+  "devDependencies": {
+    "@types/bluebird": "^3.5.16",
+    "@types/gridfs-stream": "^0.5.30",
+    "@types/mocha": "^2.2.41",
+    "@types/node": "^8.0.46",
+    "@types/proxyquire": "^1.3.27",
+    "del-cli": "^1.0.0",
+    "mocha": "^3.4.2",
+    "nyc": "^11.0.1",
+    "proxyquire": "^1.8.0",
+    "source-map-support": "^0.4.15",
+    "ts-node": "^3.0.4",
+    "typescript": "^2.3.4"
+  },
+  "dependencies": {
+    "@raincatcher/logger": "0.0.1",
+    "base64-stream": "0.1.3",
+    "bluebird": "^3.5.1",
+    "express": "4.15.2",
+    "gridfs-stream": "^1.1.1",
+    "lodash": "4.17.4",
+    "mongodb": "^2.2.25",
+    "multer": "1.1.0",
+    "q": "1.4.1",
+    "s3": "^4.4.0",
+    "shortid": "^2.2.8",
+    "through2": "2.0.1",
+    "uuid-js": "0.7.5"
+  }
+}

cloud/filestore/src/FileRoutes.ts

@@ -0,0 +1,69 @@
+import { getLogger } from '@raincatcher/logger';
+import { Router } from 'express';
+import uuid = require('uuid-js');
+import { FileStorage } from './file-api/FileStorage';
+import * as fileService from './services/FileService';
+
+/**
+ * Create express based router router for fileService
+ *
+ * @param {FileStorage} storageEngine - engine used to store files
+ * @returns router instance of express router
+ */
+export function createRouter(storageEngine: FileStorage) {
+  const router = Router();
+  router.route('/:filename').post(function(req, res, next) {
+    const id = uuid.create().toString();
+    const fileMeta = {
+      owner: req.params.owner,
+      name: req.params.filename,
+      namespace: req.params.namespace,
+      id
+    };
+    const stream = fileService.parseBase64Stream(req);
+    fileService.writeStreamToFile(fileMeta, stream).then(function() {
+      const location = fileService.buildFilePath(fileMeta.id);
+      return storageEngine.writeFile(fileMeta, location);
+    }).then(function() {
+      res.json(fileMeta);
+    }).catch(function(err) {
+      getLogger().error(err);
+      next(err);
+    });
+  });
+
+  const binaryUploadInitMiddleware = function(req, res, next) {
+    req.fileMeta = {};
+    req.fileMeta.id = uuid.create().toString();
+    req.fileMeta.name = req.body.fileName;
+    req.fileMeta.namespace = req.body.namespace;
+    req.fileMeta.owner = req.body.ownerId;
+    req.fileMeta.mimetype = req.file.mimetype;
+    next();
+  };
+
+  router.route('/:filename/binary').post(binaryUploadInitMiddleware, fileService.mutlerMiddleware,
+    function(req: any, res, next) {
+      const fileMeta = req.fileMeta;
+      const location = fileService.buildFilePath(fileMeta.id);
+      storageEngine.writeFile(fileMeta, location).then(function() {
+        res.json(fileMeta);
+      }).catch(function(err) {
+        getLogger().error(err);
+        next(err);
+      });
+    });
+
+  router.route('/:filename').get(function(req, res) {
+    const fileName = req.params.filename;
+    const namespace = req.params.namespace;
+    storageEngine.streamFile(namespace, fileName).then(function(buffer) {
+      if (buffer) {
+        buffer.pipe(res);
+      } else {
+        res.sendFile(fileService.buildFilePath(fileName));
+      }
+    });
+  });
+  return router;
+}

cloud/filestore/src/file-api/FileMetadata.ts

@@ -0,0 +1,33 @@
+
+/**
+ * Interface contains documented fields that can be used by implementation to save and query files.
+ * Actual implementation can use any other fields that are required to be saved along with the file.
+ */
+export interface FileMetadata {
+  /**
+   * Identifier for the saved file. It needs to be unique to represent file in the storage.
+   * When using {namespace} field id needs to be unique in namespace.
+   * It's recommended to use UUID generated strings for the file id.
+   */
+  id: string;
+
+  /**
+   * Original file name that can be restored and send back to user instead of the auto generated id.
+   */
+  originalName?: string;
+
+  /**
+   * Allows to categorize file. Depending on the implementation namespace can be represented as folder, bucket etc.
+   */
+  namespace?: string;
+
+  /**
+   * String representing file owner (it can be id/email etc.).
+   */
+  owner?: string;
+
+  /**
+   * Tags that can be used to query specific file.
+   */
+  tags?: string[];
+}

cloud/filestore/src/file-api/FileStorage.ts

@@ -0,0 +1,25 @@
+
+import { FileMetadata } from './FileMetadata';
+
+/**
+ * Interface used to retrieve file from the storage.
+ * Implementors can relay on the interface to store or retrieve files stream.
+ */
+export interface FileStorage {
+
+  /**
+   * Write file that is saved in temporary local storage to the permanent storage.
+   *
+   * @param {FileMetadata} metadata for the file to be saved
+   * @param {string} fileLocation - local filesystem location to the file
+   */
+  writeFile(metadata: FileMetadata, fileLocation: string): Promise<any>;
+
+  /**
+   * Retrieve file stream from storage
+   *
+   * @param {string} namespace - location (folder) used to place saved file.
+   * @param {string} fileName - filename that should be unique within namespace
+   */
+  streamFile(namespace: string, fileName: string): Promise<any>;
+}

cloud/filestore/src/impl/GridFsStorage.ts

@@ -0,0 +1,73 @@
+import { getLogger } from '@raincatcher/logger';
+import * as BlueBird from 'bluebird';
+import * as fs from 'fs';
+import * as gridfs from 'gridfs-stream';
+import * as mongo from 'mongodb';
+import { MongoClient } from 'mongodb';
+import { FileMetadata } from '../file-api/FileMetadata';
+import { FileStorage } from '../file-api/FileStorage';
+
+/**
+ * Reference implementation for GridFsStorage.
+ * See MongoDB documentation for more details.
+ */
+export class GridFsStorage implements FileStorage {
+
+  private gridFileSystem: gridfs.Grid;
+
+  /**
+   * Creates instance of the GridFsStorage that will connect to specified mongo url
+   * @param mongoConnectionURl - MongoDB connection url
+   */
+  constructor(mongoConnectionUrl: string) {
+    const self = this;
+    MongoClient.connect(mongoConnectionUrl, function(err, connection) {
+      if (err) {
+        getLogger().error('Cannot connect to mongodb server. Gridfs storage will be disabled');
+        return;
+      }
+      self.gridFileSystem = gridfs(connection, mongo);
+    });
+  }
+
+  public writeFile(metadata: FileMetadata, fileLocation: string): Promise<any> {
+    const self = this;
+    if (!self.gridFileSystem) {
+      return BlueBird.reject('Not initialized');
+    }
+    const options = {
+      root: metadata.namespace,
+      filename: metadata.id
+    };
+    return new BlueBird(function(resolve, reject) {
+      const writeStream = self.gridFileSystem.createWriteStream(options);
+      writeStream.on('error', function(err) {
+        getLogger().error('An error occurred!', err);
+        reject(err);
+      });
+      writeStream.on('close', function(file) {
+        resolve(file);
+      });
+      fs.createReadStream(fileLocation).pipe(writeStream);
+    });
+  }
+
+  public streamFile(namespace: string, fileName: string): Promise<any> {
+    const self = this;
+    if (!self.gridFileSystem) {
+      return BlueBird.reject('Not initialized');
+    }
+    const options = {
+      filename: fileName,
+      root: namespace
+    };
+    return new BlueBird(function(resolve, reject) {
+      const readstream = self.gridFileSystem.createReadStream(options);
+      readstream.on('error', function(err) {
+        getLogger().error('An error occurred when reading file from gridfs!', err);
+        reject(err);
+      });
+      resolve(readstream);
+    });
+  }
+}

cloud/filestore/src/impl/LocalStorage.ts

@@ -0,0 +1,19 @@
+import { getLogger } from '@raincatcher/logger';
+import * as BlueBird from 'bluebird';
+import { FileMetadata } from '../file-api/FileMetadata';
+import { FileStorage } from '../file-api/FileStorage';
+
+/**
+ * Implementation that using server filesystem to store files.
+ * This storage is not executing any actions as files are already stored in the disc drive.
+ */
+export class LocalStorage implements FileStorage {
+
+  public writeFile(metadata: FileMetadata, fileLocation: string): Promise<any> {
+    return BlueBird.resolve(fileLocation);
+  }
+
+  public streamFile(namespace: string, fileName: string): Promise<any> {
+    return BlueBird.resolve();
+  }
+}