This repository has been archived by the owner on Apr 17, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
335 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
# generated code | ||
src/**/*.js | ||
src/**/*.map | ||
test/**/*.js | ||
test/**/*.map | ||
coverage_report/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
test/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# RainCatcher FileStore | ||
|
||
WIP TODO |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
{ | ||
"name": "@raincatcher/example-base", | ||
"version": "1.0.0", | ||
"description": "NPM package template for the RainCatcher project", | ||
"types": "src/index.ts", | ||
"author": "feedhenry-raincatcher@redhat.com", | ||
"license": "Apache-2.0", | ||
"main": "src/", | ||
"scripts": { | ||
"clean": "del coverage_report src/**/*.js src/**/*.map test/**/*.js test/**/*.map", | ||
"build": "tsc", | ||
"start": "ts-node src/index.ts", | ||
"test": "npm run clean && nyc mocha" | ||
}, | ||
"nyc": { | ||
"include": [ | ||
"src/**/*.ts" | ||
], | ||
"extension": [ | ||
".ts" | ||
], | ||
"require": [ | ||
"ts-node/register" | ||
], | ||
"reporter": [ | ||
"lcov", | ||
"text" | ||
], | ||
"report-dir": "coverage_report", | ||
"check-coverage": true, | ||
"lines": 75, | ||
"functions": 100, | ||
"branches": 80 | ||
}, | ||
"devDependencies": { | ||
"@types/proxyquire": "^1.3.27", | ||
"@types/mocha": "^2.2.41", | ||
"proxyquire": "^1.8.0", | ||
"del-cli": "^1.0.0", | ||
"mocha": "^3.4.2", | ||
"nyc": "^11.0.1", | ||
"source-map-support": "^0.4.15", | ||
"ts-node": "^3.0.4", | ||
"typescript": "^2.3.4" | ||
}, | ||
"dependencies": { | ||
"base64-stream": "0.1.3", | ||
"bluebird": "^3.5.1", | ||
"express": "4.15.2", | ||
"fh-wfm-mediator": "0.3.3", | ||
"gridfs-stream": "^1.1.1", | ||
"lodash": "4.17.4", | ||
"mongodb": "^2.2.25", | ||
"multer": "1.1.0", | ||
"q": "1.4.1", | ||
"s3": "^4.4.0", | ||
"shortid": "^2.2.8", | ||
"through2": "2.0.1", | ||
"uuid-js": "0.7.5" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
import { Router } from 'express'; | ||
import uuid from 'uuid-js'; | ||
import { FileStorage } from '../file-api/FileStorage'; | ||
import fileService from './services/File'; | ||
|
||
/** | ||
* Create express based router router for fileService | ||
* | ||
* @param {FileStorage} storageEngine - engine used to store files | ||
* @returns router instance of express router | ||
*/ | ||
function initRouter(storageEngine: FileStorage) { | ||
const router = Router(); | ||
|
||
router.route('/:filename').post(function(req, res, next) { | ||
const id = uuid.create().toString(); | ||
const fileMeta = { | ||
owner: req.params.owner, | ||
name: req.params.filename, | ||
namespace: req.params.namespace, | ||
id | ||
}; | ||
const stream = fileService.parseBase64Stream(req); | ||
fileService.writeStreamToFile(fileMeta, stream).then(function() { | ||
const location = fileService.filePath(fileMeta.id); | ||
return storageEngine.writeFile(fileMeta.namespace, fileMeta.id, location); | ||
}).then(function() { | ||
res.json(fileMeta); | ||
}).catch(function(err) { | ||
next(err); | ||
}); | ||
}); | ||
|
||
const binaryUploadInitMiddleware = function(req, res, next) { | ||
req.fileMeta = {}; | ||
req.fileMeta.id = uuid.create().toString(); | ||
req.fileMeta.uid = req.fileMeta.id; | ||
req.fileMeta.name = req.body.fileName; | ||
req.fileMeta.namespace = req.body.namespace; | ||
req.fileMeta.owner = req.body.ownerId; | ||
req.fileMeta.mimetype = req.file.mimetype; | ||
next(); | ||
}; | ||
|
||
router.route('/:filename/binary').post(binaryUploadInitMiddleware, fileService.mutlerMiddleware, | ||
function(req, res, next) { | ||
const fileMeta = req.fileMeta; | ||
const location = fileService.filePath(fileMeta.uid); | ||
storageEngine.writeFile(fileMeta.namespace, fileMeta.uid, location).then(function() { | ||
return mediatorService.createFileMetadata(fileMeta); | ||
}).then(function() { | ||
res.json(fileMeta); | ||
}).catch(function(err) { | ||
console.log(err); | ||
next(err); | ||
}); | ||
}); | ||
|
||
router.route('/:filename').get(function(req, res) { | ||
const fileName = req.params.filename; | ||
const namespace = req.params.namespace; | ||
storageEngine.streamFile(namespace, fileName).then(function(buffer) { | ||
if (buffer) { | ||
buffer.pipe(res); | ||
} else { | ||
res.sendFile(fileService.filePath(fileName)); | ||
} | ||
}); | ||
}); | ||
return router; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
|
||
/** | ||
* Interface contains documented fields that can be used by implementation to save and query files. | ||
* Actual implementation can use any other fields that are required to be saved along with the file. | ||
*/ | ||
export interface FileMetadata { | ||
/** | ||
* Identifier for the saved file. It needs to be unique to represent file in the storage. | ||
* When using {namespace} field id needs to be unique in namespace. | ||
* It's recomended to use UUID generated strings for the file id. | ||
*/ | ||
id: string; | ||
/** | ||
* Original file name that can be restored and send back to user instead of the autogenerated id. | ||
*/ | ||
originalName?: string; | ||
|
||
/** | ||
* Allows to categorize file. Depending on the implementation namespace can be represented as folder, bucket etc. | ||
*/ | ||
namespace?: string; | ||
|
||
/** | ||
* String representing file owner (it can be id/email etc.). | ||
*/ | ||
owner?: string; | ||
|
||
/** | ||
* Tags that can be used to query specific file. | ||
*/ | ||
tags?: string[]; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
|
||
import { FileMetadata } from './FileMetadata'; | ||
|
||
/** | ||
* Interface used to retrieve file from the storage. | ||
* Implementors can relay on the interface to store or retrieve files stream. | ||
*/ | ||
export interface FileStorage { | ||
|
||
/** | ||
* Write file that is saved in temporary local storage to the permanent storage. | ||
* | ||
* @param {FileMetadata} metadata for the file to be saved | ||
* @param {string} fileLocation - local filesystem location to the file | ||
*/ | ||
writeFile(metadata: FileMetadata, fileLocation: string): Promise<any>; | ||
|
||
/** | ||
* Retrieve file stream from storage | ||
* | ||
* @param {string} namespace - location (folder) used to place saved file. | ||
* @param {string} fileName - filename that should be unique within namespace | ||
*/ | ||
streamFile(namespace: string, fileName: string): Promise<any>; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
import { FileStorage } from '../file-api/FileStorage'; | ||
|
||
|
||
export class GridFsStorage implements FileStorage { | ||
writeFile(metadata: FileMetadata, fileLocation: string): Promise<any> { | ||
throw new Error("Method not implemented."); | ||
} | ||
streamFile(namespace: string, fileName: string): Promise<any> { | ||
throw new Error("Method not implemented."); | ||
} | ||
|
||
} | ||
|
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
var base64 = require('base64-stream'); | ||
var path = require("path"); | ||
var q = require("q"); | ||
var through = require('through2'); | ||
var os = require('os'); | ||
var fs = require('fs'); | ||
var multer = require('multer'); | ||
|
||
var imageDir = os.tmpdir() + '/raincatcher-file-store'; | ||
|
||
/** | ||
* Create temporary storage folder used by mutler to store files before | ||
* uploading to permanent storage | ||
*/ | ||
function createTemporaryStorageFolder() { | ||
fs.mkdir(imageDir, '0775', function(err) { | ||
if (err && err.code !== 'EEXIST') { | ||
console.log(err); | ||
throw new Error(err); | ||
} | ||
}); | ||
} | ||
|
||
/** | ||
* Utility function for saving file in temp folder | ||
* @param fileMeta | ||
* @param stream | ||
*/ | ||
function writeStreamToFile(fileMeta, stream) { | ||
var deferred = q.defer(); | ||
stream.on('end', function() { | ||
deferred.resolve(fileMeta); | ||
}); | ||
stream.on('error', function(error) { | ||
deferred.reject(error); | ||
}); | ||
var filename = imageDir + '/' + fileMeta.uid; | ||
stream.pipe(fs.createWriteStream(filename)); | ||
return deferred.promise; | ||
} | ||
|
||
/** | ||
* Parse Base64 URL into stream that can be saved as file. | ||
* | ||
* @param req | ||
*/ | ||
function parseBase64Stream(req) { | ||
var passthrough = false; | ||
var accumulation = ''; | ||
var stream = req.pipe(through(function(chunk, enc, callback) { | ||
if (!passthrough) { | ||
accumulation += chunk; | ||
var test = ';base64,'; | ||
var index = accumulation.indexOf(test); | ||
if (index > -1) { | ||
passthrough = true; | ||
chunk = accumulation.substr(index + test.length); | ||
} | ||
} | ||
if (passthrough) { | ||
this.push(chunk); | ||
} | ||
callback(); | ||
})) | ||
.pipe(base64.decode()); | ||
return stream; | ||
} | ||
|
||
function buildFilePath(fileName) { | ||
return path.join(imageDir, fileName); | ||
} | ||
|
||
function mutlerMiddleware() { | ||
return multer({ | ||
storage: multer.diskStorage({ | ||
destination: function(req, file, cb) { | ||
cb(null, imageDir); | ||
}, | ||
filename: function(req, file, cb) { | ||
cb(null, req.fileMeta.uid); | ||
} | ||
}) | ||
}).single('binaryfile'); | ||
} | ||
|
||
/** | ||
* Util class for file operations | ||
* | ||
* @type {{ | ||
* imageDir: string, | ||
* writeStreamToFile: function, | ||
* parseBase64Stream: function, | ||
* createTemporaryStorageFolder: function, | ||
* filePath: function, | ||
* mutlerMiddleware: mutlerMiddleware | ||
* }} | ||
*/ | ||
module.exports = { | ||
imageDir: imageDir, | ||
writeStreamToFile: writeStreamToFile, | ||
parseBase64Stream: parseBase64Stream, | ||
createTemporaryStorageFolder: createTemporaryStorageFolder, | ||
filePath: buildFilePath, | ||
mutlerMiddleware: mutlerMiddleware | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
/// <reference types="mocha" /> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
--compilers ts:ts-node/register | ||
--require source-map-support/register | ||
test/**.ts |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
{ | ||
"compilerOptions": { | ||
"module": "commonjs", | ||
"target": "es5", | ||
"noImplicitAny": false, | ||
"experimentalDecorators": true, | ||
"strictNullChecks": true, | ||
"sourceMap": true | ||
}, | ||
"include": [ | ||
"src/", | ||
"test/" | ||
] | ||
} |