Skip to content

Commit

Permalink
Mega SPEED BOOST
Browse files Browse the repository at this point in the history 
Removed needle library, removed semaphores in the database adapter for supercharged performance!
  • Loading branch information
@silvae86
silvae86 committed Apr 10, 2017
1 parent 18f385a commit ffa5028
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 137 deletions.
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
"coverage": "nyc --reporter=lcov --reporter=text --reporter=text-lcov node node_modules/.bin/_mocha test_new_structure -t 10000",
"report-coverage": "nyc npm test && nyc report --reporter=text-lcov | coveralls",
"everything": "rm -rf node_modules && npm cache clean && npm install && npm run test && npm run coverage && npm run report-coverage",
"remote_debug": "node app.js --debug=5858"
"remote_debug": "node app.js --debug=5858",
"profile": "node --prof node_modules/.bin/_mocha test_new_structure -t 10000"
},
"dependencies": {
"archiver": "*",
Expand Down Expand Up @@ -68,7 +69,6 @@
"multiparty": "^4.1.2",
"mysql": "^2.13.0",
"n3": "^0.9.1",
"needle": "^1.5.0",
"node-b2share-v2": "^1.2.0",
"node-fs": "0.1.7",
"nodemailer": "^3.1.4",
Expand All @@ -81,6 +81,7 @@
"qs": "^6.1.0",
"redis": "^2.6.2",
"request": "*",
"request-promise": "^4.2.0",
"restler": "^3.4.0",
"rmdir": "^1.2.0",
"semaphore": "^1.0.5",
Expand Down
4 changes: 2 additions & 2 deletions src/controllers/files.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1839,10 +1839,10 @@ exports.serve_static = function(req, res, pathOfIntendedFileRelativeToProjectRoo
};

exports.data = function(req, res){

let path = require('path');
var requestedExtension = path.extname(req.params.filepath).replace(".", "");

if(files.dataParsers[requestedExtension] != null)
if(exports.dataParsers[requestedExtension] != null)
{
var resourceURI = req.params.requestedResource;

Expand Down
248 changes: 115 additions & 133 deletions src/kb/db.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
var util = require('util');
var Config = function() { return GLOBAL.Config; }();
var needle = require('needle');
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var uuid = require('uuid');

Expand Down Expand Up @@ -53,7 +52,7 @@ DbConnection.prototype.create = function(callback) {
{
if(xmlHttp.status == 200)
{
self.queueSemaphore = require('semaphore')(self.maxSimultaneousConnections);
//self.queueSemaphore = require('semaphore')(self.maxSimultaneousConnections);
callback(self);
}
else
Expand Down Expand Up @@ -247,182 +246,165 @@ DbConnection.prototype.execute = function(queryStringWithArguments, argumentsArr
{
console.error("database operation timeout for query " + queryStringWithArguments + " with transaction ID " + transactionID);
delete self.pendingTransactionIDs[transactionID];
self.queueSemaphore.leave();
//self.queueSemaphore.leave();
}
else
{
//console.log("database operation timeout for transaction ID " + transactionID + " but operation has been completed.");
self.queueSemaphore.leave();
//self.queueSemaphore.leave();
}

}, Config.dbOperationTimeout);

self.queueSemaphore.take(function ()
{
try
//self.queueSemaphore.take(function ()
//{
if (Config.debug.active && Config.debug.database.log_all_queries)
{
if (Config.debug.active && Config.debug.database.log_all_queries)
{
console.log("POSTING QUERY: \n" + query);
}
console.log("POSTING QUERY: \n" + query);
}

needle.post(fullUrl, {
const options = {
method: 'POST',
uri: fullUrl,
form: {
query: query,
maxRows: maxRows,
maxrows: maxRows,
format: resultsFormat,
open_timeout: Config.dbOperationTimeout
}, function (err, response, body)
{
if (!err && response.statusCode == 200)
},
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
json: true
};

let rp = require('request-promise');
rp(options)
.then(function (parsedBody) {
var transformedResults = [];
// iterate through all the rows in the result list

if (parsedBody.boolean != null)
{
var parsedRes = body;
if (!parsedRes instanceof Object)
callback(null, parsedBody.boolean);
}
else
{
var numberOfRows = parsedBody.results.bindings.length;

if (numberOfRows == 0)
{
callback(1, "Invalid response from Virtuoso Server. Invalid JSON received.");
} else
callback(null, []);
}
else
{
var transformedResults = [];
// iterate through all the rows in the result list
// initialize list of headers and matching datatypes
var datatypes = [];
var columnHeaders = [];

if (parsedRes.boolean != null)
{
callback(null, parsedRes.boolean);
}
else
for (var i = 0; i < parsedBody.head.vars.length; i++)
{
var numberOfRows = parsedRes.results.bindings.length;
var columnHeader = parsedBody.head.vars[i];

if (numberOfRows == 0)
{
callback(null, []);
}
else
//handling OPTIONAL clauses, where a header will not have any value
if (parsedBody.results.bindings[0][columnHeader] != null)
{
// initialize list of headers and matching datatypes
var datatypes = [];
var columnHeaders = [];
columnHeaders.push(columnHeader);

for (var i = 0; i < parsedRes.head.vars.length; i++)
if (parsedBody.results.bindings[0][columnHeader] == null)
{
var columnHeader = parsedRes.head.vars[i];
console.log("invalid binding");
}

//handling OPTIONAL clauses, where a header will not have any value
if (parsedRes.results.bindings[0][columnHeader] != null)
{
columnHeaders.push(columnHeader);
var type = parsedBody.results.bindings[0][columnHeader].type;
datatypes.push(type);
}
}

if (parsedRes.results.bindings[0][columnHeader] == null)
{
console.log("invalid binding");
}
var numberOfHeaders = columnHeaders.length;

var type = parsedRes.results.bindings[0][columnHeader].type;
datatypes.push(type);
}
}
// util.debug("Headers:\n" + util.inspect(columnHeaders, true,
// null));
// util.debug("Datatypes:\n" + util.inspect(datatypes, true,
// null));

var numberOfHeaders = columnHeaders.length;
// build results table
for (i = 0; i < numberOfRows; i++)
{
// for each result header, create an empty object to push
// the results
transformedResults[i] = {};

// util.debug("Headers:\n" + util.inspect(columnHeaders, true,
// null));
// util.debug("Datatypes:\n" + util.inspect(datatypes, true,
// null));
// util.debug("Transformed Results A:\n" +
// util.inspect(transformedResults, true, null));

// build results table
for (i = 0; i < numberOfRows; i++)
{
// for each result header, create an empty object to push
// the results
transformedResults[i] = {};
for (var j = 0; j < numberOfHeaders; j++)
{
var header = columnHeaders[j];

// util.debug("Transformed Results A:\n" +
// util.inspect(transformedResults, true, null));
var binding = parsedBody.results.bindings[i];

for (var j = 0; j < numberOfHeaders; j++)
if (binding != null)
{
if (binding[header] != null)
{
var header = columnHeaders[j];
var datatype;
if (binding[header] != null)
{
datatype = binding[header].type;
}
else
{
datatype = datatypes[j];
}

var binding = parsedRes.results.bindings[i];
var value = binding[header].value;

if (binding != null)
switch (datatype)
{
if (binding[header] != null)
case ("http://www.w3.org/2001/XMLSchema#integer"):
{
var newInt = parseInt(value);
transformedResults[" + i + "].header = newInt;
break;
}
case ("uri"):
{
var datatype;
if (binding[header] != null)
{
datatype = binding[header].type;
}
else
{
datatype = datatypes[j];
}

var value = binding[header].value;

switch (datatype)
{
case ("http://www.w3.org/2001/XMLSchema#integer"):
{
var newInt = parseInt(value);
transformedResults[" + i + "].header = newInt;
break;
}
case ("uri"):
{
transformedResults[i][header] = decodeURI(value);
break;
}
// default is a string value
default:
{
var valueWithQuotes = decodeURIComponent(value).replace(/\"/g, "\\\"");
transformedResults[i][header] = valueWithQuotes;
break;
}
}
transformedResults[i][header] = decodeURI(value);
break;
}
// default is a string value
default:
{
var valueWithQuotes = decodeURIComponent(value).replace(/\"/g, "\\\"");
transformedResults[i][header] = valueWithQuotes;
break;
}
}
}
}

// util.debug("Transformed Results :\n" +
// util.inspect(transformedResults, true, null));

delete self.pendingTransactionIDs[transactionID];
//self.queueSemaphore.leave();
callback(null, transformedResults);
}
}
}
}
else
{
var error = "Unknown error occurred when executing query " + query;

if (response != null)
{
error = "Virtuoso server returned: \n \n STATUS CODE " + response.statusCode + "\n \nMESSAGE " + util.inspect(err) + "\n \n ERROR DESCRIPTION : \n \n " + response.body + " RUNNING QUERY \n \n" + query;
//console.trace(err);
console.error(error);
}
// util.debug("Transformed Results :\n" +
// util.inspect(transformedResults, true, null));

delete self.pendingTransactionIDs[transactionID];
//self.queueSemaphore.leave();
callback(1, err);
//delete self.pendingTransactionIDs[transactionID];
//self.queueSemaphore.leave();
callback(null, transformedResults);
}
}
});
}
catch (e)
{
var error = "Exception occurred when querying the database " + e;
console.error(error);
delete self.pendingTransactionIDs[transactionID];
//self.queueSemaphore.leave();
callback(1, error);
})
.catch(function(err){
const error = "Virtuoso server returned error: \n " + util.inspect(err);
//console.trace(err);
console.error(error);

}
});
//delete self.pendingTransactionIDs[transactionID];
//self.queueSemaphore.leave();
callback(1, err);
});
//});
}
else
{
Expand Down

0 comments on commit ffa5028

Please sign in to comment.