Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow REST access to raw node data from REST. #646

Merged
merged 1 commit into from
Oct 22, 2015
Merged

Allow REST access to raw node data from REST. #646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/common/storage/util.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ define(['common/storage/constants'], function (CONSTANTS) {
if (projectId) {
return projectId.substring(projectId.indexOf(CONSTANTS.PROJECT_ID_SEP) + 1);
}
},
getHashTaggedHash: function (hash) {
if (typeof hash === 'string') {
return hash[0] === '#' ? hash : '#' + hash;
}
return hash;
}
};
});
112 changes: 110 additions & 2 deletions src/server/api/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,43 @@ function createAPI(app, mountPath, middlewareOpts) {

// PROJECTS

function loadNodePathByCommitHash(userId, projectId, commitHash, path) {
var getCommitParams = {
username: userId,
projectId: projectId,
number: 1,
before: commitHash
};

return safeStorage.getCommits(getCommitParams)
.then(function (commits) {
var loadPathsParams = {
projectId: projectId,
username: userId,
pathsInfo: [
{
parentHash: commits[0].root,
path: path
}
],
excludeParents: true
};

return safeStorage.loadPaths(loadPathsParams);
})
.then(function (dataObjects) {
var hashes = Object.keys(dataObjects);
if (hashes.length === 1) {
return dataObjects[hashes[0]];
} else if (hashes.length === 0) {
throw new Error('Path does not exist ' + path);
} else {
// This should never happen..
throw new Error('safeStorage.loadPaths returned with more than one object');
}
});
}

router.get('/projects', ensureAuthenticated, function (req, res, next) {
var userId = getUserId(req);
safeStorage.getProjects({username: userId})
Expand All @@ -540,7 +577,6 @@ function createAPI(app, mountPath, middlewareOpts) {
});
});


/**
* Creating project by seed
* Available body parameters:
Expand Down Expand Up @@ -606,6 +642,48 @@ function createAPI(app, mountPath, middlewareOpts) {
});
});

router.get('/projects/:ownerId/:projectName/commits/:commitHash', ensureAuthenticated, function (req, res, next) {
var userId = getUserId(req),
commitHash = StorageUtil.getHashTaggedHash(req.params.commitHash),
data = {
username: userId,
projectId: StorageUtil.getProjectIdFromOwnerIdAndProjectName(req.params.ownerId,
req.params.projectName),
before: commitHash,
number: 1
};

safeStorage.getCommits(data)
.then(function (result) {
res.json(result[0]);
})
.catch(function (err) {
if (err.message.indexOf('not exist') > -1 || err.message.indexOf('Not authorized to read') > -1 ) {
err.status = 404;
}
next(err);
});
});

router.get('/projects/:ownerId/:projectName/commits/:commitHash/tree/*', ensureAuthenticated,
function (req, res, next) {
var userId = getUserId(req),
projectId = StorageUtil.getProjectIdFromOwnerIdAndProjectName(req.params.ownerId,
req.params.projectName),
commitHash = StorageUtil.getHashTaggedHash(req.params.commitHash);

loadNodePathByCommitHash(userId, projectId, commitHash, '/' + req.params[0])
.then(function (nodeObj) {
res.json(nodeObj);
})
.catch(function (err) {
if (err.message.indexOf('not exist') > -1 || err.message.indexOf('Not authorized to read') > -1 ) {
err.status = 404;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not authorized status code is 403

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was requested that if the user doesn't have read access the return code should be 404.
For two reasons;
The storage just checks if the user has the project in his/her project list (with read access) and does't check whether the project exists or not.
If the user doesn't have read access, he shouldn't even be able to get information about whether the project exists or not.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks

}
next(err);
});
}
);

router.get('/projects/:ownerId/:projectName/compare/:branchOrCommitA...:branchOrCommitB',
ensureAuthenticated,
Expand Down Expand Up @@ -678,6 +756,36 @@ function createAPI(app, mountPath, middlewareOpts) {
});
});

router.get('/projects/:ownerId/:projectName/branches/:branchId/tree/*', ensureAuthenticated,
function (req, res, next) {
var userId = getUserId(req),
projectId = StorageUtil.getProjectIdFromOwnerIdAndProjectName(req.params.ownerId,
req.params.projectName),
data = {
username: userId,
projectId: projectId,
branchName: req.params.branchId
};

safeStorage.getBranchHash(data)
.then(function (branchHash) {
if (!branchHash) {
throw new Error('Branch does not exist ' + req.params.branchId);
}
return loadNodePathByCommitHash(userId, projectId, branchHash, '/' + req.params[0]);
})
.then(function (dataObj) {
res.json(dataObj);
})
.catch(function (err) {
if (err.message.indexOf('not exist') > -1 || err.message.indexOf('Not authorized to read') > -1 ) {
err.status = 404;
}
next(err);
});
}
);

router.patch('/projects/:ownerId/:projectName/branches/:branchId', function (req, res, next) {
var userId = getUserId(req),
data = {
Expand Down Expand Up @@ -855,7 +963,7 @@ function createAPI(app, mountPath, middlewareOpts) {
var pluginExecution = runningPlugins[req.params.resultId];
logger.debug('Plugin-result request for ', req.params.pluginId, req.params.resultId);
if (pluginExecution) {
if (pluginExecution.status === PLUGIN_CONSTANTS.RUNNING) {
if (pluginExecution.status === PLUGIN_CONSTANTS.RUNNING) {
res.send(pluginExecution);
} else {
// Remove the pluginExecution when it has finished or an error occurred.
Expand Down
8 changes: 4 additions & 4 deletions src/server/storage/safestorage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -977,7 +977,8 @@ SafeStorage.prototype.loadObjects = function (data, callback) {
* @param {string} data.projectId - identifier for project.
* @param {string} [data.username=gmeConfig.authentication.guestAccount]
* @param {object[]} data.pathsInfo - list of objects with parentHash and path.
* @param {string[]} data.excludes - list of known object hashes that should not be returned.
* @param {string[]} [data.excludes] - list of known object hashes that should not be returned.
* @param {boolean} [data.excludeParents] - if true will only return the data for the node at the path.
* @param {function} [callback]
* @returns {promise} //TODO: jsdocify this
*/
Expand All @@ -989,9 +990,8 @@ SafeStorage.prototype.loadPaths = function (data, callback) {
rejected = check(data !== null && typeof data === 'object', deferred, 'data is not an object.') ||
check(typeof data.projectId === 'string', deferred, 'data.projectId is not a string.') ||
check(REGEXP.PROJECT.test(data.projectId), deferred, 'data.projectId failed regexp: ' + data.projectId) ||
check(data.excludes instanceof Array,
deferred, 'data.excludes is not an array: ' + JSON.stringify(data.excludes)) ||
check(data.pathsInfo instanceof Array, deferred, 'data.pathsInfo is not an array: ' + JSON.stringify(data.hashes));
check(data.pathsInfo instanceof Array, deferred,
'data.pathsInfo is not an array: ' + JSON.stringify(data.hashes));

if (data.hasOwnProperty('username')) {
rejected = rejected || check(typeof data.username === 'string', deferred, 'data.username is not a string.');
Expand Down
77 changes: 53 additions & 24 deletions src/server/storage/storage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,9 +311,10 @@ Storage.prototype.loadObjects = function (data, callback) {
* @param {string} rootHash
* @param {Object<string, object>} loadedObjects
* @param {string} path
* @param {boolean} excludeParents - if true will only include the node at the path
* @returns {function|promise}
*/
function loadPath(dbProject, rootHash, loadedObjects, path) {
function loadPath(dbProject, rootHash, loadedObjects, path, excludeParents) {
var deferred = Q.defer(),
pathArray = path.split('/');

Expand All @@ -330,11 +331,14 @@ function loadPath(dbProject, rootHash, loadedObjects, path) {
} else {
dbProject.loadObject(parentHash)
.then(function (object) {
loadedObjects[parentHash] = object;
if (relPath) {
hash = object[relPath];
if (!excludeParents) {
loadedObjects[parentHash] = object;
}
loadParent(hash, pathArray.shift());
} else {
loadedObjects[parentHash] = object;
deferred.resolve();
}
})
Expand All @@ -357,35 +361,53 @@ Storage.prototype.loadPaths = function (data, callback) {

this.mongo.openProject(data.projectId)
.then(function (dbProject) {
var loadedObjects = {};

//self.logger.error('paths:', data.paths);
var loadedObjects = {},
throttleDeferred = Q.defer(),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what is saved here by having this 'throttle' function? shouldn't we use a library throttle (and for the loadObjects instead of the loadPath(s))?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

An example:
The root is loaded and the meta nodes are all requested to be loaded:
/languageNode/metaNodePath1
/languageNode/metaNodePath2
/languageNode/metaNodePathN

Without the throttle the languageNode will be loaded N-times from the mongo database. With the throttle it is only loaded once. The loadObject-calls are already throttled for a one path (or what do you mean?).

Q doesn't have a throttle and I wouldn't like to use another library.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I thought you were trying to throttle against too many paralel calls towards the db, but it okay then.

counter = data.pathsInfo.length;

Q.allSettled(data.pathsInfo.map(function (pathInfo) {
return loadPath(dbProject, pathInfo.parentHash, loadedObjects, pathInfo.path);
}))
.then(function (results) {
var keys = Object.keys(loadedObjects),
i;
function throttleLoad() {
var pathInfo;

for (i = 0; i < results.length; i += 1) {
if (results[i].state === 'rejected') {
self.logger.error('loadPaths failed, ignoring', data.pathsInfo[i].path, {
metadata: results[i].reason,
if (counter === 0) {
throttleDeferred.resolve();
} else {
counter -= 1;
pathInfo = data.pathsInfo[counter];
loadPath(dbProject, pathInfo.parentHash, loadedObjects, pathInfo.path, data.excludeParents)
.then(function () {
throttleLoad();
})
.catch(function (err) {
self.logger.error('loadPaths failed, ignoring', pathInfo.path, {
metadata: err,
});
}
}
throttleLoad();
});
}

for (i = 0; i < keys.length; i += 1) {
if (data.excludes.indexOf(keys[i]) > -1) {
// https://jsperf.com/delete-vs-setting-undefined-vs-new-object
// When sending the data these keys will be removed after JSON.stringify.
loadedObjects[keys[i]] = undefined;
return throttleDeferred.promise;
}

//Q.allSettled(data.pathsInfo.map(function (pathInfo) {
// return loadPath(dbProject, pathInfo.parentHash, loadedObjects, pathInfo.path, data.excludeParents);
//}))
throttleLoad()
.then(function () {
var keys = Object.keys(loadedObjects),
i;
if (data.excludes) {
for (i = 0; i < keys.length; i += 1) {
if (data.excludes.indexOf(keys[i]) > -1) {
// https://jsperf.com/delete-vs-setting-undefined-vs-new-object
// When sending the data these keys will be removed after JSON.stringify.
loadedObjects[keys[i]] = undefined;
}
}
}

deferred.resolve(loadedObjects);
});
})
.catch(deferred.reject);
})
.catch(function (err) {
deferred.reject(err);
Expand All @@ -407,7 +429,14 @@ Storage.prototype.getCommits = function (data, callback) {
self.logger.debug('commitHash was given will load commit', data.before);
project.loadObject(data.before)
.then(function (commitObject) {
return project.getCommits(commitObject.time + 1, data.number);
if (commitObject.type !== 'commit') {
throw new Error('Commit object does not exist ' + data.before);
}
if (data.number === 1) {
return [commitObject];
} else {
return project.getCommits(commitObject.time + 1, data.number);
}
})
.then(function (commits) {
deferred.resolve(commits);
Expand Down
Loading