Skip to content
This repository has been archived by the owner on Apr 17, 2023. It is now read-only.

Commit

Permalink
[RAINCATCH-1388] Filestore tests (#180)
Browse files Browse the repository at this point in the history
* Start refactoring for tests

* Add some tests for filestore

* Add tests for writeStreamToFile

* Finish tests for fileService

* Lower coverage reqs

* Rename storage class to match filename

* Add string type for createTemporaryStorageFolder

* Add first tests for gridfs storage

* Add tests for gridfs

* Update jsdocs

* Update package version to pre-release one

* update dependencies to match hoisted versions

* Fix hanging test
  • Loading branch information
paolobueno authored and wtrocki committed Nov 15, 2017
1 parent e3618a2 commit 6cbed01
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 75 deletions.
16 changes: 8 additions & 8 deletions client/filestore-client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,23 +35,23 @@
"branches": 80
},
"devDependencies": {
"@types/bluebird": "^3.5.16",
"@types/bluebird": "^3.5.8",
"@types/cordova-plugin-file": "0.0.3",
"@types/cordova-plugin-file-transfer": "0.0.3",
"@types/lodash": "^4.14.79",
"@types/lodash": "^4.14.73",
"@types/mocha": "^2.2.41",
"@types/proxyquire": "^1.3.27",
"del-cli": "^1.0.0",
"mocha": "^3.4.2",
"nyc": "^11.0.1",
"mocha": "^4.0.1",
"nyc": "^11.1.0",
"proxyquire": "^1.8.0",
"source-map-support": "^0.4.15",
"ts-node": "^3.0.4",
"typescript": "^2.3.4"
"source-map-support": "^0.5.0",
"ts-node": "^3.3.0",
"typescript": "^2.5.0"
},
"dependencies": {
"b64-to-blob": "^1.2.19",
"bluebird": "^3.5.1",
"bluebird": "^3.5.0",
"lodash": "^4.17.4"
}
}
37 changes: 21 additions & 16 deletions cloud/filestore/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"clean": "del coverage_report src/**/*.js src/**/*.map test/**/*.js test/**/*.map",
"build": "tsc",
"start": "ts-node src/index.ts",
"test": "npm run clean && nyc mocha"
"test": "nyc mocha"
},
"publishConfig": {
"access": "public"
Expand All @@ -30,33 +30,38 @@
"text"
],
"report-dir": "coverage_report",
"check-coverage": true,
"lines": 75,
"functions": 100,
"branches": 80
"functions": 75,
"branches": 50
},
"devDependencies": {
"@types/bluebird": "^3.5.16",
"@types/bluebird": "^3.5.8",
"@types/del": "^3.0.0",
"@types/gridfs-stream": "^0.5.30",
"@types/mkdirp": "^0.5.1",
"@types/mocha": "^2.2.41",
"@types/node": "^8.0.46",
"@types/multer": "^1.3.5",
"@types/node": "^8.0.7",
"@types/proxyquire": "^1.3.27",
"del-cli": "^1.0.0",
"mocha": "^3.4.2",
"nyc": "^11.0.1",
"del": "^3.0.0",
"del-cli": "^1.1.0",
"mocha": "^4.0.1",
"nyc": "^11.1.0",
"proxyquire": "^1.8.0",
"source-map-support": "^0.4.15",
"ts-node": "^3.0.4",
"typescript": "^2.3.4"
"source-map-support": "^0.5.0",
"string-to-stream": "^1.1.0",
"ts-node": "^3.3.0",
"typescript": "^2.5.0"
},
"dependencies": {
"@raincatcher/logger": "0.0.1",
"base64-stream": "0.1.3",
"bluebird": "^3.5.1",
"express": "4.15.2",
"bluebird": "^3.5.0",
"express": "4.15.4",
"gridfs-stream": "^1.1.1",
"lodash": "4.17.4",
"mongodb": "^2.2.25",
"lodash": "^4.17.4",
"mkdirp": "^0.5.1",
"mongodb": "^2.2.31",
"multer": "1.1.0",
"q": "1.4.1",
"s3": "^4.4.0",
Expand Down
42 changes: 26 additions & 16 deletions cloud/filestore/src/impl/GridFsStorage.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { getLogger } from '@raincatcher/logger';
import * as BlueBird from 'bluebird';
import * as Promise from 'bluebird';
import * as fs from 'fs';
import * as gridfs from 'gridfs-stream';
import * as mongo from 'mongodb';
Expand All @@ -8,38 +8,48 @@ import { Stream } from 'stream';
import { FileMetadata } from '../file-api/FileMetadata';
import { FileStorage } from '../file-api/FileStorage';

const connectAsync = Promise.promisify<mongo.Db, string>(MongoClient.connect);

/**
* Reference implementation for GridFsStorage.
* See MongoDB documentation for more details.
*/
export class GridFsStorage implements FileStorage {

private gridFileSystem: gridfs.Grid;
public gridFileSystem: gridfs.Grid;
private fileSystemPromise: Promise<gridfs.Grid>;

/**
* Creates instance of the GridFsStorage that will connect to specified mongo url
* @param mongoConnectionURl - MongoDB connection url
* @param connection - MongoDB connection url or connection instance
*/
constructor(mongoConnectionUrl: string) {
const self = this;
MongoClient.connect(mongoConnectionUrl, function(err, connection) {
if (err) {
getLogger().error('Cannot connect to mongodb server. Gridfs storage will be disabled');
return;
}
self.gridFileSystem = gridfs(connection, mongo);
});
constructor(connection: string | mongo.Db) {
if (typeof connection === 'string') {
this.fileSystemPromise = connectAsync(connection)
.then(conn => this.gridFileSystem = gridfs(conn, mongo));
} else {
this.gridFileSystem = gridfs(connection, mongo);
this.fileSystemPromise = Promise.resolve(this.gridFileSystem);
}
}

/**
* Gets the underlying gridfs filesystem instance
* @return A promise of the underlying fileSystem
*/
public getFileSystem() {
return this.fileSystemPromise;
}

public writeFile(metadata: FileMetadata, fileLocation: string): Promise<string> {
const self = this;
if (!self.gridFileSystem) {
return BlueBird.reject('Not initialized');
return Promise.reject('Not initialized');
}
const options = {
filename: metadata.id
};
return new BlueBird(function(resolve, reject) {
return new Promise(function(resolve, reject) {
const writeStream = self.gridFileSystem.createWriteStream(options);
writeStream.on('error', function(err) {
getLogger().error('An error occurred!', err);
Expand All @@ -55,12 +65,12 @@ export class GridFsStorage implements FileStorage {
public readFile(id: string): Promise<Stream> {
const self = this;
if (!self.gridFileSystem) {
return BlueBird.reject('Not initialized');
return Promise.reject('Not initialized');
}
const options = {
filename: id
};
return new BlueBird(function(resolve, reject) {
return new Promise(function(resolve, reject) {
const readstream = self.gridFileSystem.createReadStream(options);
readstream.on('error', function(err) {
getLogger().error('An error occurred when reading file from gridfs!', err);
Expand Down
7 changes: 6 additions & 1 deletion cloud/filestore/src/impl/S3Storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export interface S3StorageConfiguration {
* Implementation that using server filesystem to store files.
* This storage is not executing any actions as files are already stored in the disc drive.
*/
export class LocalStorage implements FileStorage {
export class S3Storage implements FileStorage {
private awsClient: any;
private storageConfig: S3StorageConfiguration;
/**
Expand Down Expand Up @@ -93,6 +93,10 @@ export class LocalStorage implements FileStorage {

}

/**
* Validates whether the supplied S3 configuration has the required keys
* @param config S3 storage configuration object
*/
private validateConfig(config: S3StorageConfiguration) {
if (!config.bucket) {
throw Error('Invalid configuration for s3 storage: Please specify bucket name');
Expand All @@ -109,5 +113,6 @@ export class LocalStorage implements FileStorage {
if (!config.s3Config.region) {
throw Error('Invalid configuration for s3 storage: region missing');
}
return true;
}
}
65 changes: 33 additions & 32 deletions cloud/filestore/src/services/FileService.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
import { getLogger } from '@raincatcher/logger';
import * as base64 from 'base64-stream';
import * as Promise from 'bluebird';
import { Request } from 'express';
import * as fs from 'fs';
import multer = require('multer');
import * as os from 'os';
import * as mkdirp from 'mkdirp';
import * as multer from 'multer';
import * as os from 'os';
import { join } from 'path';
import * as path from 'path';
import q from 'q';
import { Stream } from 'stream';
import through from 'through2';
import { FileMetadata } from '../file-api/FileMetadata';

const mkdirpAsync = Promise.promisify<string, string>(mkdirp);

/**
* Location of the stored files in the server's local disk
* Uses the OS' temporary directory
Expand All @@ -20,51 +22,50 @@ export const FILE_STORAGE_DIRECTORY = path.join(os.tmpdir(), '/raincatcher-file-
/**
* Create temporary storage folder used by multer to store files before
* uploading to permanent storage
* @param directory Optional alternative root directory for testing
*/
export function createTemporaryStorageFolder() {
fs.mkdir(FILE_STORAGE_DIRECTORY, '0775', function(err: any) {
if (err && err.code !== 'EEXIST') {
getLogger().error(err);
throw new Error(err);
}
});
export function createTemporaryStorageFolder(directory: string = FILE_STORAGE_DIRECTORY): Promise<string> {
return mkdirpAsync(directory).then(() => directory);
}

/**
* Utility function for saving file in temp folder
* @param metadata
* @param stream
* @param metadata data related to the file being saved
* @param stream a Node readable stream with the file's binary data
*/
export function writeStreamToFile(metadata: FileMetadata, stream: Stream) {
const deferred = q.defer();
stream.on('end', function() {
deferred.resolve(metadata);
export function writeStreamToFile(metadata: FileMetadata, stream: Stream): Promise<FileMetadata> {
return new Promise((resolve, reject) => {
stream.on('end', function() {
resolve(metadata);
});
stream.on('error', function(error) {
reject(error);
});
const filename = buildFilePath(metadata.id);
stream.pipe(fs.createWriteStream(filename));
});
stream.on('error', function(error) {
deferred.reject(error);
});
const filename = path.join(FILE_STORAGE_DIRECTORY, metadata.id);
stream.pipe(fs.createWriteStream(filename));
return deferred.promise;
}

/**
* Returns the full path to a file stored with the service
* @param fileName Name of the file to build a path for, usually the file's id
*/
export function buildFilePath(fileName) {
return path.join(FILE_STORAGE_DIRECTORY, fileName);
export function buildFilePath(fileName: string, root: string = FILE_STORAGE_DIRECTORY) {
return path.join(root, fileName);
}

const diskStorageDefaultOptions = {
destination(req, file, cb) {
cb(null, FILE_STORAGE_DIRECTORY);
}
};

/**
* Returns a new multer-based middleware that's capable of processing the `multipart/form-data` uploads
* Returns a multer middleware that handles multipart/form-data requests
* @param storage A multer storage implementation
*/
export function multerMiddleware() {
export function multerMiddleware(storage: multer.StorageEngine = multer.diskStorage(diskStorageDefaultOptions)) {
return multer({
storage: multer.diskStorage({
destination(req, file, cb) {
cb(null, FILE_STORAGE_DIRECTORY);
}
})
storage
});
}
1 change: 1 addition & 0 deletions cloud/filestore/test/fixtures/test.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hello world
59 changes: 59 additions & 0 deletions cloud/filestore/test/impl/GridFsStorage-spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import * as Promise from 'bluebird';
import { expect } from 'chai';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import { readFileSync } from 'fs';
import * as mongo from 'mongodb';
import { MongoClient } from 'mongodb';
import * as path from 'path';
import { FileMetadata } from '../../src/file-api/FileMetadata';
import { GridFsStorage } from '../../src/impl/GridFsStorage';

const connectAsync = Promise.promisify<mongo.Db, string>(MongoClient.connect);

chai.use(chaiAsPromised);

function readStream(stream: NodeJS.ReadableStream): Promise<string> {
let data = '';
stream.setEncoding('utf8');
stream.on('data', chunk => data += chunk);
return new Promise(resolve => stream.on('end', () => resolve(data)));
}

describe('GridFsStorage', function() {
const mongoUrl = 'mongodb://127.0.0.1:27017/testdb';
const dbFactory = () => connectAsync(mongoUrl);
let db: mongo.Db;
const metadata: FileMetadata = {
id: 'test-file'
};
const fileLocation = path.resolve(__dirname, '../fixtures/test.txt');
const fileContents = readFileSync(fileLocation, {
encoding: 'utf8'
});

before(function() {
return dbFactory().then(database => db = database);
});
after(function() {
db.close();
});

describe('constructor', function() {
it('should accept a mongodb connection', function() {
const storage = new GridFsStorage(db);
return expect(storage.gridFileSystem).to.exist &&
expect(storage.getFileSystem()).to.eventually.be.fulfilled;
});
});

it('should write and read back a file from storage', function() {
const storage = new GridFsStorage(db);
const write = storage.writeFile(metadata, fileLocation);
const read = write
.then(() => storage.readFile(metadata.id))
.then(readStream);
return expect(write).to.eventually.be.fulfilled &&
expect(read).to.eventually.equal(fileContents);
});
});
2 changes: 1 addition & 1 deletion cloud/filestore/test/mocha.opts
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
--compilers ts:ts-node/register
--require source-map-support/register
test/**.ts
test/**/*.ts
Loading

0 comments on commit 6cbed01

Please sign in to comment.