Skip to content
This repository has been archived by the owner on Sep 23, 2022. It is now read-only.

Commit

Permalink
Changelog 1.1.0 :
Browse files Browse the repository at this point in the history
- Cluster CPU
- Improve performance
  • Loading branch information
aalfiann committed Feb 3, 2020
1 parent ae68542 commit 19e7126
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 163 deletions.
3 changes: 3 additions & 0 deletions config-master.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ const config_master = {
// Block upload to spesific node destination.
blockNode: [], // Ex. ['/ns2','/ns3'] means the file will not upload to /n2 and /ns3.

// Use Worker / CPU Cluster
useWorker: true,

// Print the log
logger: true

Expand Down
3 changes: 3 additions & 0 deletions config-node.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ const config_node = {
// set directory path location for static file
staticDirPath: 'public', // without leading and trailing slash

// Use Worker / CPU Cluster
useWorker: true,

// Print the log
logger: true,

Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "cluster-static-server",
"version": "1.0.10",
"version": "1.1.0",
"description": "Cluster Static Server based on Fastify NodeJS.",
"main": "app.js",
"scripts": {
Expand Down
7 changes: 4 additions & 3 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ The main problem of a static server is when you have limit with your server stor
This cluster static server was created with Fastify NodeJS Framework which is the fastest and very low overhead of any NodeJS framework out there. You can start with single or multiple shared hosting to implement this cluster static server.

## Features
- Cluster
- Cluster Storage
- Cluster CPU
- Upload File
- Remote Upload File
- Get File
Expand All @@ -39,8 +40,8 @@ The main problem of a static server is when you have limit with your server stor
5. Done

### How to run the server
- To start master server `$ npm run master`.
- To start node server `$ npm run node`.
- To start master server `$ npm run master` or `$ node server-master.js`.
- To start node server `$ npm run node` or `$ node server-node.js`.

**Note :**
- You are able to start master and node server in same machine server with different port. But, actualy Master and Node server should be on different machine server so you will get the best performance.
Expand Down
178 changes: 95 additions & 83 deletions routes/master.js
Original file line number Diff line number Diff line change
Expand Up @@ -49,36 +49,40 @@ async function masterRoute (server, options) {
fs.rename(path.normalize(fileArr[0].tempFilePath), path.normalize(fileArr[0].tempFilePath+'.'+fileArr[0].ext), err => {
if(!err) {
var nosql = new FlyJson();
var result = nosql.set(config.nodeServer);
var x = config.blockNode;
for(var z=0;z<x.length;z++) {
result.where('prefix', '!=', x[z]);
}
var listhost = result.exec();
var host = listhost[helper.randomItem(listhost)];

const ls = spawn("node", ["./transfer.js",
"-u", host.upstream+"/node/upload",
"-f", fileArr[0].tempFilePath+'.'+fileArr[0].ext,
"-x", config.node_x_token,
"-o", config.origin
]);

ls.stdout.on("data", function(data) {
fs.unlink(path.normalize(fileArr[0].tempFilePath+'.'+fileArr[0].ext),function(err){
if(err) console.log(err);
});
var result = JSON.parse(data.toString());
if(result.status === 200) {
result.body[0].name = fileArr[0].name;
reply.code(result.status).send({status:result.status,message:'Upload file successfully!',response:result.body[0]});
} else {
reply.code(409).send({status:409,message:'Failed to upload! Please try again!'});
nosql.promisify((builder) => {return builder}).then((table) => {
var result = table.set(config.nodeServer);
var x = config.blockNode;
for(var z=0;z<x.length;z++) {
result.where('prefix', '!=', x[z]);
}
});
var listhost = result.exec();
var host = listhost[helper.randomItem(listhost)];

const ls = spawn("node", ["./transfer.js",
"-u", host.upstream+"/node/upload",
"-f", fileArr[0].tempFilePath+'.'+fileArr[0].ext,
"-x", config.node_x_token,
"-o", config.origin
]);

ls.stdout.on("data", function(data) {
fs.unlink(path.normalize(fileArr[0].tempFilePath+'.'+fileArr[0].ext),function(err){
if(err) console.log(err);
});
var result = JSON.parse(data.toString());
if(result.status === 200) {
result.body[0].name = fileArr[0].name;
reply.code(result.status).send({status:result.status,message:'Upload file successfully!',response:result.body[0]});
} else {
reply.code(409).send({status:409,message:'Failed to upload! Please try again!'});
}
});

ls.stderr.on("data", function(data) {
reply.code(500).send({status:500,message:'Something went wrong!',error:data.toString()});
ls.stderr.on("data", function(data) {
reply.code(500).send({status:500,message:'Something went wrong!',error:data.toString()});
});
}).catch((err) => {
reply.code(409).send({status:409,message:'Failed to upload! Wrong configuration!'});
});
} else {
reply.code(409).send({status:409,message:'Failed to rename file! Permission denied!'});
Expand All @@ -98,33 +102,37 @@ async function masterRoute (server, options) {
var data = request.body;
if(data.node && data.year && data.month && data.date && data.filename) {
var nosql = new FlyJson();
var listhost = nosql.set(config.nodeServer).where('prefix','==','/'+data.node.toLowerCase()).exec();
if(listhost.length > 0) {
var transfer = new ParallelRequest();
transfer.add({
url: listhost[0].upstream+'/node/delete',
method: 'post',
headers:{
'Content-Type':'application/json',
'x_token':config.node_x_token
},
body: {
year:data.year,
month:data.month,
date:data.date,
filename:data.filename
}
});
transfer.send(function(response) {
if(response[0].status === 200) {
reply.send({status:200,message:"File deleted successfully!"});
} else {
reply.code(response[0].status).send({status:response[0].status,message:response[0].body.message});
}
});
} else {
reply.code(404).send({status:404,message:"File not found!"});
}
nosql.promisify((builder) => {return builder}).then((table) => {
var listhost = table.set(config.nodeServer).where('prefix','==','/'+data.node.toLowerCase()).exec();
if(listhost.length > 0) {
var transfer = new ParallelRequest();
transfer.add({
url: listhost[0].upstream+'/node/delete',
method: 'post',
headers:{
'Content-Type':'application/json',
'x_token':config.node_x_token
},
body: {
year:data.year,
month:data.month,
date:data.date,
filename:data.filename
}
});
transfer.send(function(response) {
if(response[0].status === 200) {
reply.send({status:200,message:"File deleted successfully!"});
} else {
reply.code(response[0].status).send({status:response[0].status,message:response[0].body.message});
}
});
} else {
reply.code(404).send({status:404,message:"File not found!"});
}
}).catch((err) => {
reply.code(409).send({status:409,message:'Failed to delete file! Wrong configuration!'});
});
} else {
reply.code(400).send({status:400,message:"Bad Request!"});
}
Expand Down Expand Up @@ -161,36 +169,40 @@ async function masterRoute (server, options) {
fs.access(path.normalize(directory+body.filename), fs.constants.F_OK, err => {
if(!err) {
var nosql = new FlyJson();
var result = nosql.set(config.nodeServer);
var x = config.blockNode;
for(var z=0;z<x.length;z++) {
result.where('prefix', '!=', x[z]);
}
var listhost = result.exec();
var host = listhost[helper.randomItem(listhost)];

const ls = spawn("node", ["./transfer.js",
"-u", host.upstream+"/node/upload",
"-f", path.normalize(directory+body.filename),
"-x", config.node_x_token,
"-o", config.origin
]);

ls.stdout.on("data", function(data) {
fs.unlink(path.normalize(directory+body.filename),function(err){
if(err) console.log(err);
});
var result = JSON.parse(data.toString());
if(result.status === 200) {
result.body[0].name = body.filename;
reply.code(result.status).send({status:result.status,message:'Upload file successfully!',response:result.body[0]});
} else {
reply.code(result.status).send({status:result.status,message:result.body[0].message});
nosql.promisify((builder) => {return builder}).then((table) => {
var result = table.set(config.nodeServer);
var x = config.blockNode;
for(var z=0;z<x.length;z++) {
result.where('prefix', '!=', x[z]);
}
});
var listhost = result.exec();
var host = listhost[helper.randomItem(listhost)];

const ls = spawn("node", ["./transfer.js",
"-u", host.upstream+"/node/upload",
"-f", path.normalize(directory+body.filename),
"-x", config.node_x_token,
"-o", config.origin
]);

ls.stdout.on("data", function(data) {
fs.unlink(path.normalize(directory+body.filename),function(err){
if(err) console.log(err);
});
var result = JSON.parse(data.toString());
if(result.status === 200) {
result.body[0].name = body.filename;
reply.code(result.status).send({status:result.status,message:'Upload file successfully!',response:result.body[0]});
} else {
reply.code(result.status).send({status:result.status,message:result.body[0].message});
}
});

ls.stderr.on("data", function(data) {
reply.code(500).send({status:500,message:'Something went wrong!',error:data.toString()});
ls.stderr.on("data", function(data) {
reply.code(500).send({status:500,message:'Something went wrong!',error:data.toString()});
});
}).catch((err) => {
reply.code(409).send({status:409,message:'Failed to upload! Wrong configuration network!'});
});
} else {
reply.code(409).send({status:409,message:'Failed to upload! Please try again!'});
Expand Down
86 changes: 54 additions & 32 deletions server-master.js
Original file line number Diff line number Diff line change
@@ -1,44 +1,66 @@
const config = require('./config-master.js');
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
const proxy = require('fastify-http-proxy');
const server = require('fastify')({
logger: config.logger
});

for(var i=0;i<config.nodeServer.length;i++) {
server.register(proxy, config.nodeServer[i]);
}
function App() {

// Hooks
server.addHook('onRequest', (request, reply, done) => {
if(request.raw.method !== 'GET') {
if(request.headers.x_token===config.x_token){
done();
for(var i=0;i<config.nodeServer.length;i++) {
server.register(proxy, config.nodeServer[i]);
}

// Hooks
server.addHook('onRequest', (request, reply, done) => {
if(request.raw.method !== 'GET') {
if(request.headers.x_token===config.x_token){
done();
} else {
reply.code(400).send({status:400,message:'Bad Request!'});
}
} else {
reply.code(400).send({status:400,message:'Bad Request!'});
done();
}
} else {
done();
}
});

server.register(require('fastify-file-upload'), {
limits: { fileSize: config.limitSize },
useTempFiles : true,
tempFileDir : 'temp/'
});

server.register(require('./routes/default.js'));
server.register(require('./routes/master.js'));
});

server.register(require('fastify-file-upload'), {
limits: { fileSize: config.limitSize },
useTempFiles : true,
tempFileDir : 'temp/'
});

server.register(require('./routes/default.js'));
server.register(require('./routes/master.js'));

// Custom Error Handler
server.setErrorHandler(function (error, request, reply) {
server.log.error(error);
reply.send({status:500,message:'Whoops, Something went wrong!'});
});

server.listen(config.masterPort, function (err) {
if (err) {
server.log.error(err);
process.exit(1);
}
});

// Custom Error Handler
server.setErrorHandler(function (error, request, reply) {
server.log.error(error);
reply.send({status:500,message:'Whoops, Something went wrong!'});
});
}

server.listen(config.masterPort, function (err, address) {
if (err) {
server.log.error(err);
process.exit(1);
if(config.useWorker) {
if (cluster.isMaster) {
console.log(`Master ${process.pid} is running`);
for (let i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('exit', worker => {
console.log(`Worker ${worker.process.pid} died`);
});
} else {
App();
}
});
} else {
App();
}

0 comments on commit 19e7126

Please sign in to comment.