RAINCATCH-1351 - File Server

cloud/file-store/.gitignore

@@ -0,0 +1,6 @@
+# generated code
+src/**/*.js
+src/**/*.map
+test/**/*.js
+test/**/*.map
+coverage_report/

cloud/file-store/.npmignore

@@ -0,0 +1 @@
+test/

cloud/file-store/README.md

@@ -0,0 +1,3 @@
+# RainCatcher FileStore
+
+ WIP TODO

cloud/file-store/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@raincatcher/example-base",
+  "version": "1.0.0",
+  "description": "NPM package template for the RainCatcher project",
+  "types": "src/index.ts",
+  "author": "feedhenry-raincatcher@redhat.com",
+  "license": "Apache-2.0",
+  "main": "src/",
+  "scripts": {
+    "clean": "del coverage_report src/**/*.js src/**/*.map test/**/*.js test/**/*.map",
+    "build": "tsc",
+    "start": "ts-node src/index.ts",
+    "test": "npm run clean && nyc mocha"
+  },
+  "nyc": {
+    "include": [
+      "src/**/*.ts"
+    ],
+    "extension": [
+      ".ts"
+    ],
+    "require": [
+      "ts-node/register"
+    ],
+    "reporter": [
+      "lcov",
+      "text"
+    ],
+    "report-dir": "coverage_report",
+    "check-coverage": true,
+    "lines": 75,
+    "functions": 100,
+    "branches": 80
+  },
+  "devDependencies": {
+    "@types/proxyquire": "^1.3.27",
+    "@types/mocha": "^2.2.41",
+    "proxyquire": "^1.8.0",
+    "del-cli": "^1.0.0",
+    "mocha": "^3.4.2",
+    "nyc": "^11.0.1",
+    "source-map-support": "^0.4.15",
+    "ts-node": "^3.0.4",
+    "typescript": "^2.3.4"
+  },
+  "dependencies": {
+    "base64-stream": "0.1.3",
+    "bluebird": "^3.5.1",
+    "express": "4.15.2",
+    "fh-wfm-mediator": "0.3.3",
+    "gridfs-stream": "^1.1.1",
+    "lodash": "4.17.4",
+    "mongodb": "^2.2.25",
+    "multer": "1.1.0",
+    "q": "1.4.1",
+    "s3": "^4.4.0",
+    "shortid": "^2.2.8",
+    "through2": "2.0.1",
+    "uuid-js": "0.7.5"
+  }
+}

cloud/file-store/src/FileRoutes.ts

@@ -0,0 +1,71 @@
+import { Router } from 'express';
+import uuid from 'uuid-js';
+import { FileStorage } from '../file-api/FileStorage';
+import fileService from './services/File';
+
+/**
+ * Create express based router router for fileService
+ *
+ * @param {FileStorage} storageEngine - engine used to store files
+ * @returns router instance of express router
+ */
+function initRouter(storageEngine: FileStorage) {
+  const router = Router();
+
+  router.route('/:filename').post(function(req, res, next) {
+    const id = uuid.create().toString();
+    const fileMeta = {
+      owner: req.params.owner,
+      name: req.params.filename,
+      namespace: req.params.namespace,
+      id
+    };
+    const stream = fileService.parseBase64Stream(req);
+    fileService.writeStreamToFile(fileMeta, stream).then(function() {
+      const location = fileService.filePath(fileMeta.id);
+      return storageEngine.writeFile(fileMeta.namespace, fileMeta.id, location);
+    }).then(function() {
+      res.json(fileMeta);
+    }).catch(function(err) {
+      next(err);
+    });
+  });
+
+  const binaryUploadInitMiddleware = function(req, res, next) {
+    req.fileMeta = {};
+    req.fileMeta.id = uuid.create().toString();
+    req.fileMeta.uid = req.fileMeta.id;
+    req.fileMeta.name = req.body.fileName;
+    req.fileMeta.namespace = req.body.namespace;
+    req.fileMeta.owner = req.body.ownerId;
+    req.fileMeta.mimetype = req.file.mimetype;
+    next();
+  };
+
+  router.route('/:filename/binary').post(binaryUploadInitMiddleware, fileService.mutlerMiddleware,
+    function(req, res, next) {
+      const fileMeta = req.fileMeta;
+      const location = fileService.filePath(fileMeta.uid);
+      storageEngine.writeFile(fileMeta.namespace, fileMeta.uid, location).then(function() {
+        return mediatorService.createFileMetadata(fileMeta);
+      }).then(function() {
+        res.json(fileMeta);
+      }).catch(function(err) {
+        console.log(err);
+        next(err);
+      });
+    });
+
+  router.route('/:filename').get(function(req, res) {
+    const fileName = req.params.filename;
+    const namespace = req.params.namespace;
+    storageEngine.streamFile(namespace, fileName).then(function(buffer) {
+      if (buffer) {
+        buffer.pipe(res);
+      } else {
+        res.sendFile(fileService.filePath(fileName));
+      }
+    });
+  });
+  return router;
+}

cloud/file-store/src/file-api/FileMetadata.ts

@@ -0,0 +1,32 @@
+
+/**
+ * Interface contains documented fields that can be used by implementation to save and query files.
+ * Actual implementation can use any other fields that are required to be saved along with the file.
+ */
+export interface FileMetadata {
+  /**
+   * Identifier for the saved file. It needs to be unique to represent file in the storage.
+   * When using {namespace} field id needs to be unique in namespace.
+   * It's recomended to use UUID generated strings for the file id.
+   */
+  id: string;
+  /**
+   * Original file name that can be restored and send back to user instead of the autogenerated id.
+   */
+  originalName?: string;
+
+  /**
+   * Allows to categorize file. Depending on the implementation namespace can be represented as folder, bucket etc.
+   */
+  namespace?: string;
+
+  /**
+   * String representing file owner (it can be id/email etc.).
+   */
+  owner?: string;
+
+  /**
+   * Tags that can be used to query specific file.
+   */
+  tags?: string[];
+}

cloud/file-store/src/file-api/FileStorage.ts

@@ -0,0 +1,25 @@
+
+import { FileMetadata } from './FileMetadata';
+
+/**
+ * Interface used to retrieve file from the storage.
+ * Implementors can relay on the interface to store or retrieve files stream.
+ */
+export interface FileStorage {
+
+  /**
+   * Write file that is saved in temporary local storage to the permanent storage.
+   *
+   * @param {FileMetadata} metadata for the file to be saved
+   * @param {string} fileLocation - local filesystem location to the file
+   */
+  writeFile(metadata: FileMetadata, fileLocation: string): Promise<any>;
+
+  /**
+   * Retrieve file stream from storage
+   *
+   * @param {string} namespace - location (folder) used to place saved file.
+   * @param {string} fileName - filename that should be unique within namespace
+   */
+  streamFile(namespace: string, fileName: string): Promise<any>;
+}

cloud/file-store/src/impl/GridFsStorage.ts

@@ -0,0 +1,13 @@
+import { FileStorage } from '../file-api/FileStorage';
+
+
+export class GridFsStorage implements FileStorage {
+  writeFile(metadata: FileMetadata, fileLocation: string): Promise<any> {
+    throw new Error("Method not implemented.");
+  }
+  streamFile(namespace: string, fileName: string): Promise<any> {
+    throw new Error("Method not implemented.");
+  }
+
+}
+

cloud/file-store/src/services/FileService.ts

@@ -0,0 +1,105 @@
+var base64 = require('base64-stream');
+var path = require("path");
+var q = require("q");
+var through = require('through2');
+var os = require('os');
+var fs = require('fs');
+var multer = require('multer');
+
+var imageDir = os.tmpdir() + '/raincatcher-file-store';
+
+/**
+ * Create temporary storage folder used by mutler to store files before
+ * uploading to permanent storage
+ */
+function createTemporaryStorageFolder() {
+  fs.mkdir(imageDir, '0775', function(err) {
+    if (err && err.code !== 'EEXIST') {
+      console.log(err);
+      throw new Error(err);
+    }
+  });
+}
+
+/**
+ * Utility function for saving file in temp folder
+ * @param fileMeta
+ * @param stream
+ */
+function writeStreamToFile(fileMeta, stream) {
+  var deferred = q.defer();
+  stream.on('end', function() {
+    deferred.resolve(fileMeta);
+  });
+  stream.on('error', function(error) {
+    deferred.reject(error);
+  });
+  var filename = imageDir + '/' + fileMeta.uid;
+  stream.pipe(fs.createWriteStream(filename));
+  return deferred.promise;
+}
+
+/**
+ * Parse Base64 URL into stream that can be saved as file.
+ *
+ * @param req
+ */
+function parseBase64Stream(req) {
+  var passthrough = false;
+  var accumulation = '';
+  var stream = req.pipe(through(function(chunk, enc, callback) {
+    if (!passthrough) {
+      accumulation += chunk;
+      var test = ';base64,';
+      var index = accumulation.indexOf(test);
+      if (index > -1) {
+        passthrough = true;
+        chunk = accumulation.substr(index + test.length);
+      }
+    }
+    if (passthrough) {
+      this.push(chunk);
+    }
+    callback();
+  }))
+    .pipe(base64.decode());
+  return stream;
+}
+
+function buildFilePath(fileName) {
+  return path.join(imageDir, fileName);
+}
+
+function mutlerMiddleware() {
+  return multer({
+    storage: multer.diskStorage({
+      destination: function(req, file, cb) {
+        cb(null, imageDir);
+      },
+      filename: function(req, file, cb) {
+        cb(null, req.fileMeta.uid);
+      }
+    })
+  }).single('binaryfile');
+}
+
+/**
+ * Util class for file operations
+ *
+ * @type {{
+ *         imageDir: string,
+ *         writeStreamToFile: function,
+ *         parseBase64Stream: function,
+ *         createTemporaryStorageFolder: function,
+ *         filePath: function,
+ *         mutlerMiddleware: mutlerMiddleware
+ * }}
+ */
+module.exports = {
+  imageDir: imageDir,
+  writeStreamToFile: writeStreamToFile,
+  parseBase64Stream: parseBase64Stream,
+  createTemporaryStorageFolder: createTemporaryStorageFolder,
+  filePath: buildFilePath,
+  mutlerMiddleware: mutlerMiddleware
+};

cloud/file-store/test/index.ts

@@ -0,0 +1 @@
+/// <reference types="mocha" />

cloud/file-store/test/mocha.opts

@@ -0,0 +1,3 @@
+--compilers ts:ts-node/register
+--require source-map-support/register
+test/**.ts

cloud/file-store/tsconfig.json

@@ -0,0 +1,14 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "es5",
+        "noImplicitAny": false,
+        "experimentalDecorators": true,
+        "strictNullChecks": true,
+        "sourceMap": true
+    },
+    "include": [
+        "src/",
+        "test/"
+    ]
+}