This repository has been archived by the owner on Apr 17, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
21 changed files
with
579 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
# generated code | ||
src/**/*.js | ||
src/**/*.map | ||
test/**/*.js | ||
test/**/*.map | ||
coverage_report/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
test/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# RainCatcher FileStore | ||
|
||
WIP TODO |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
## File store server | ||
|
||
File | ||
|
||
## Running | ||
|
||
ts-node ./example/index.ts | ||
|
||
## Requirements | ||
|
||
`MONGO_CONNECTION_URL` environment variable that points to a mongodb instance. | ||
By default using: mongodb://127.0.0.1:27017/sync | ||
|
||
`REDIS_HOST` and `REDIS_PORT` environment variables that points to a running redis instance. | ||
By default using: 127.0.0.1 and 6379 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import { getLogger } from '@raincatcher/logger'; | ||
import { createRouter, FileMetadata, FileStorage, LocalStorage } from '../src/index'; | ||
|
||
const logger = getLogger(); | ||
|
||
// Create express middleware | ||
import * as bodyParser from 'body-parser'; | ||
import * as cors from 'cors'; | ||
import * as express from 'express'; | ||
|
||
const app = express(); | ||
|
||
// middleware | ||
app.use(bodyParser.json()); | ||
app.use(cors()); | ||
|
||
const engine: FileStorage = new LocalStorage(); | ||
const router = createRouter(engine); | ||
|
||
app.use('/', router); | ||
|
||
app.listen(3000, function() { | ||
logger.info('Example app listening on port 3000!'); | ||
}); | ||
// If you wish to see logs; | ||
process.env.DEBUG = 'fh-mbaas-api:sync'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
{ | ||
"name": "@raincatcher/filestore", | ||
"version": "1.0.0", | ||
"description": "RainCatcher file storage server", | ||
"types": "src/index.ts", | ||
"author": "feedhenry-raincatcher@redhat.com", | ||
"license": "Apache-2.0", | ||
"main": "src/", | ||
"scripts": { | ||
"clean": "del coverage_report src/**/*.js src/**/*.map test/**/*.js test/**/*.map", | ||
"build": "tsc", | ||
"start": "ts-node src/index.ts", | ||
"test": "npm run clean && nyc mocha" | ||
}, | ||
"nyc": { | ||
"include": [ | ||
"src/**/*.ts" | ||
], | ||
"extension": [ | ||
".ts" | ||
], | ||
"require": [ | ||
"ts-node/register" | ||
], | ||
"reporter": [ | ||
"lcov", | ||
"text" | ||
], | ||
"report-dir": "coverage_report", | ||
"check-coverage": true, | ||
"lines": 75, | ||
"functions": 100, | ||
"branches": 80 | ||
}, | ||
"devDependencies": { | ||
"@types/bluebird": "^3.5.16", | ||
"@types/gridfs-stream": "^0.5.30", | ||
"@types/mocha": "^2.2.41", | ||
"@types/node": "^8.0.46", | ||
"@types/proxyquire": "^1.3.27", | ||
"del-cli": "^1.0.0", | ||
"mocha": "^3.4.2", | ||
"nyc": "^11.0.1", | ||
"proxyquire": "^1.8.0", | ||
"source-map-support": "^0.4.15", | ||
"ts-node": "^3.0.4", | ||
"typescript": "^2.3.4" | ||
}, | ||
"dependencies": { | ||
"@raincatcher/logger": "0.0.1", | ||
"base64-stream": "0.1.3", | ||
"bluebird": "^3.5.1", | ||
"express": "4.15.2", | ||
"gridfs-stream": "^1.1.1", | ||
"lodash": "4.17.4", | ||
"mongodb": "^2.2.25", | ||
"multer": "1.1.0", | ||
"q": "1.4.1", | ||
"s3": "^4.4.0", | ||
"shortid": "^2.2.8", | ||
"through2": "2.0.1", | ||
"uuid-js": "0.7.5" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
import { getLogger } from '@raincatcher/logger'; | ||
import { Router } from 'express'; | ||
import uuid = require('uuid-js'); | ||
import { FileStorage } from './file-api/FileStorage'; | ||
import * as fileService from './services/FileService'; | ||
|
||
/** | ||
* Create express based router router for fileService | ||
* | ||
* @param {FileStorage} storageEngine - engine used to store files | ||
* @returns router instance of express router | ||
*/ | ||
export function createRouter(storageEngine: FileStorage) { | ||
const router = Router(); | ||
router.route('/:filename').post(function(req, res, next) { | ||
const id = uuid.create().toString(); | ||
const fileMeta = { | ||
owner: req.params.owner, | ||
name: req.params.filename, | ||
namespace: req.params.namespace, | ||
id | ||
}; | ||
const stream = fileService.parseBase64Stream(req); | ||
fileService.writeStreamToFile(fileMeta, stream).then(function() { | ||
const location = fileService.buildFilePath(fileMeta.id); | ||
return storageEngine.writeFile(fileMeta, location); | ||
}).then(function() { | ||
res.json(fileMeta); | ||
}).catch(function(err) { | ||
getLogger().error(err); | ||
next(err); | ||
}); | ||
}); | ||
|
||
const binaryUploadInitMiddleware = function(req, res, next) { | ||
req.fileMeta = {}; | ||
req.fileMeta.id = uuid.create().toString(); | ||
req.fileMeta.name = req.body.fileName; | ||
req.fileMeta.namespace = req.body.namespace; | ||
req.fileMeta.owner = req.body.ownerId; | ||
req.fileMeta.mimetype = req.file.mimetype; | ||
next(); | ||
}; | ||
|
||
router.route('/:filename/binary').post(binaryUploadInitMiddleware, fileService.mutlerMiddleware, | ||
function(req: any, res, next) { | ||
const fileMeta = req.fileMeta; | ||
const location = fileService.buildFilePath(fileMeta.id); | ||
storageEngine.writeFile(fileMeta, location).then(function() { | ||
res.json(fileMeta); | ||
}).catch(function(err) { | ||
getLogger().error(err); | ||
next(err); | ||
}); | ||
}); | ||
|
||
router.route('/:filename').get(function(req, res) { | ||
const fileName = req.params.filename; | ||
const namespace = req.params.namespace; | ||
storageEngine.streamFile(namespace, fileName).then(function(buffer) { | ||
if (buffer) { | ||
buffer.pipe(res); | ||
} else { | ||
res.sendFile(fileService.buildFilePath(fileName)); | ||
} | ||
}); | ||
}); | ||
return router; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
|
||
/** | ||
* Interface contains documented fields that can be used by implementation to save and query files. | ||
* Actual implementation can use any other fields that are required to be saved along with the file. | ||
*/ | ||
export interface FileMetadata { | ||
/** | ||
* Identifier for the saved file. It needs to be unique to represent file in the storage. | ||
* When using {namespace} field id needs to be unique in namespace. | ||
* It's recommended to use UUID generated strings for the file id. | ||
*/ | ||
id: string; | ||
|
||
/** | ||
* Original file name that can be restored and send back to user instead of the auto generated id. | ||
*/ | ||
originalName?: string; | ||
|
||
/** | ||
* Allows to categorize file. Depending on the implementation namespace can be represented as folder, bucket etc. | ||
*/ | ||
namespace?: string; | ||
|
||
/** | ||
* String representing file owner (it can be id/email etc.). | ||
*/ | ||
owner?: string; | ||
|
||
/** | ||
* Tags that can be used to query specific file. | ||
*/ | ||
tags?: string[]; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
|
||
import { FileMetadata } from './FileMetadata'; | ||
|
||
/** | ||
* Interface used to retrieve file from the storage. | ||
* Implementors can relay on the interface to store or retrieve files stream. | ||
*/ | ||
export interface FileStorage { | ||
|
||
/** | ||
* Write file that is saved in temporary local storage to the permanent storage. | ||
* | ||
* @param {FileMetadata} metadata for the file to be saved | ||
* @param {string} fileLocation - local filesystem location to the file | ||
*/ | ||
writeFile(metadata: FileMetadata, fileLocation: string): Promise<any>; | ||
|
||
/** | ||
* Retrieve file stream from storage | ||
* | ||
* @param {string} namespace - location (folder) used to place saved file. | ||
* @param {string} fileName - filename that should be unique within namespace | ||
*/ | ||
streamFile(namespace: string, fileName: string): Promise<any>; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
import { getLogger } from '@raincatcher/logger'; | ||
import * as BlueBird from 'bluebird'; | ||
import * as fs from 'fs'; | ||
import * as gridfs from 'gridfs-stream'; | ||
import * as mongo from 'mongodb'; | ||
import { MongoClient } from 'mongodb'; | ||
import { FileMetadata } from '../file-api/FileMetadata'; | ||
import { FileStorage } from '../file-api/FileStorage'; | ||
|
||
/** | ||
* Reference implementation for GridFsStorage. | ||
* See MongoDB documentation for more details. | ||
*/ | ||
export class GridFsStorage implements FileStorage { | ||
|
||
private gridFileSystem: gridfs.Grid; | ||
|
||
/** | ||
* Creates instance of the GridFsStorage that will connect to specified mongo url | ||
* @param mongoConnectionURl - MongoDB connection url | ||
*/ | ||
constructor(mongoConnectionUrl: string) { | ||
const self = this; | ||
MongoClient.connect(mongoConnectionUrl, function(err, connection) { | ||
if (err) { | ||
getLogger().error('Cannot connect to mongodb server. Gridfs storage will be disabled'); | ||
return; | ||
} | ||
self.gridFileSystem = gridfs(connection, mongo); | ||
}); | ||
} | ||
|
||
public writeFile(metadata: FileMetadata, fileLocation: string): Promise<any> { | ||
const self = this; | ||
if (!self.gridFileSystem) { | ||
return BlueBird.reject('Not initialized'); | ||
} | ||
const options = { | ||
root: metadata.namespace, | ||
filename: metadata.id | ||
}; | ||
return new BlueBird(function(resolve, reject) { | ||
const writeStream = self.gridFileSystem.createWriteStream(options); | ||
writeStream.on('error', function(err) { | ||
getLogger().error('An error occurred!', err); | ||
reject(err); | ||
}); | ||
writeStream.on('close', function(file) { | ||
resolve(file); | ||
}); | ||
fs.createReadStream(fileLocation).pipe(writeStream); | ||
}); | ||
} | ||
|
||
public streamFile(namespace: string, fileName: string): Promise<any> { | ||
const self = this; | ||
if (!self.gridFileSystem) { | ||
return BlueBird.reject('Not initialized'); | ||
} | ||
const options = { | ||
filename: fileName, | ||
root: namespace | ||
}; | ||
return new BlueBird(function(resolve, reject) { | ||
const readstream = self.gridFileSystem.createReadStream(options); | ||
readstream.on('error', function(err) { | ||
getLogger().error('An error occurred when reading file from gridfs!', err); | ||
reject(err); | ||
}); | ||
resolve(readstream); | ||
}); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
import { getLogger } from '@raincatcher/logger'; | ||
import * as BlueBird from 'bluebird'; | ||
import { FileMetadata } from '../file-api/FileMetadata'; | ||
import { FileStorage } from '../file-api/FileStorage'; | ||
|
||
/** | ||
* Implementation that using server filesystem to store files. | ||
* This storage is not executing any actions as files are already stored in the disc drive. | ||
*/ | ||
export class LocalStorage implements FileStorage { | ||
|
||
public writeFile(metadata: FileMetadata, fileLocation: string): Promise<any> { | ||
return BlueBird.resolve(fileLocation); | ||
} | ||
|
||
public streamFile(namespace: string, fileName: string): Promise<any> { | ||
return BlueBird.resolve(); | ||
} | ||
} |
Oops, something went wrong.