From 390fba5d9d9cb1ac01cca689bccdb73d403734e4 Mon Sep 17 00:00:00 2001 From: VIJAYKUMARNINGANURE Date: Tue, 27 Sep 2016 17:35:45 +0530 Subject: [PATCH 01/59] Docker-Audit Trail Implimentation --- .../DockerContainerSync.js | 43 +++++- server/app/model/log-trail/containerLog.js | 141 ++++++++++++++++++ server/app/routes/v1.0/routes_audit_trails.js | 34 ++++- server/app/routes/v1.0/routes_instances.js | 38 +++-- server/install.js | 3 +- 5 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 server/app/model/log-trail/containerLog.js diff --git a/server/app/cronjobs/docker-container-sync/DockerContainerSync.js b/server/app/cronjobs/docker-container-sync/DockerContainerSync.js index 7a5df35cc..e06fb63b6 100644 --- a/server/app/cronjobs/docker-container-sync/DockerContainerSync.js +++ b/server/app/cronjobs/docker-container-sync/DockerContainerSync.js @@ -5,6 +5,7 @@ var MasterUtils = require('_pr/lib/utils/masterUtil.js'); var credentialCrpto = require('_pr/lib/credentialcryptography.js'); var instancesDao = require('_pr/model/classes/instance/instance'); var containerDao = require('_pr/model/container'); +var containerLogModel = require('_pr/model/log-trail/containerLog.js'); var SSH = require('_pr/lib/utils/sshexec'); var fileIo = require('_pr/lib/utils/fileio'); var toPairs = require('lodash.topairs'); @@ -203,7 +204,7 @@ function aggregateDockerContainerForInstance(instance){ if(containers.operation === 'delete'){ deleteContainerByInstanceId(containers.instanceId,next); }else if (containers.operation === 'create'){ - saveAndUpdateContainers(containers.containers,containers.containerIds,containers.instanceId,next) + saveAndUpdateContainers(containers.containers,containers.containerIds,containers.instanceId,instance,next) }else{ next(null,containers); } @@ -220,7 +221,7 @@ function aggregateDockerContainerForInstance(instance){ } }; -function saveAndUpdateContainers(containers,containerIds,instanceId,next){ +function saveAndUpdateContainers(containers,containerIds,instanceId,instance,next){ async.waterfall([ function(next){ containerDao.deleteContainersByContainerIds(instanceId,containerIds,next); @@ -238,8 +239,46 @@ function saveAndUpdateContainers(containers,containerIds,instanceId,next){ if(err){ logger.error(err); return; + }else{ count++; + var actionObj = 'Docker-Container Start' + ' : ' + container.Names; + var timestampStarted = new Date().getTime(); + var actionLog = instancesDao.insertDockerActionLog(instanceId, instance.catUser, actionObj, 1, timestampStarted); + var containerLogs ={ + actionId: actionLog._id, + containerId: container.Id, + orgName: instance.orgName, + orgId: instance.orgId, + bgName: instance.bgName, + bgId: instance.bgId, + projectName: instance.projectName, + envName: instance.envName, + envId: instance.envId, + status: container.Status, + actionStatus: "success", + platformId: instance.platformId, + containerName: container.Names, + Image: container.Image, + ImageID: container.ImageID, + platform: instance.hardware.platform, + os: instance.hardware.os, + user: instance.catUser, + createdOn: new Date().getTime(), + startedOn: new Date().getTime(), + providerType: instance.providerType ? instance.providerType:null, + action: actionObj, + logs: [{ + err: false, + log: "Started container", + timestamp: new Date().getTime() + }] + }; + containerLogModel.createOrUpdate(containerLogs, function(err, logData){ + if (err) { + logger.error("Failed to create or update containerLog: ", err); + } + }); if(count === containers.length){ next(null,containers); } diff --git a/server/app/model/log-trail/containerLog.js b/server/app/model/log-trail/containerLog.js new file mode 100644 index 000000000..234740872 --- /dev/null +++ b/server/app/model/log-trail/containerLog.js @@ -0,0 +1,141 @@ +/* +Copyright [2016] [Relevance Lab] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +var mongoose = require('mongoose'); +var mongoosePaginate = require('mongoose-paginate'); +var ObjectId = require('mongoose').Types.ObjectId; +var logger = require('_pr/logger')(module); + +var Schema = mongoose.Schema; +var ContainerLogSchema = new Schema({ + actionId: { + type: String, + unique: true + }, + containerId: String, + orgName: String, + orgId: String, + bgName: String, + bgId: String, + projectName: String, + envName: String, + envId: String, + status: String, + actionStatus: String, + platformId: String, + containerName: String, + Image: String, + ImageId: String, + platform: String, + os: String, + user: String, + createdOn: Number, + startedOn: Number, + endedOn: Number, + providerType: String, + action: String, + logs: [{ + err: Boolean, + log: String, + timestamp: Number + }] +}); + +ContainerLogSchema.plugin(mongoosePaginate); +var ContainerLogs = mongoose.model('containerlogs', ContainerLogSchema); + +var ContainerLog = function() { + this.createOrUpdate = function(logData, callback) { + ContainerLogs.find({ + actionId: logData.actionId, + containerId: logData.containerId, + orgId: logData.orgId, + bgId: logData.bgId, + envId: logData.envId + }, function(err, data) { + if (err) { + logger.debug("Failed to fetch ContainerLogs: ", err); + return callback(err, null); + } + if (data && data.length > 0) { + var logObj = { + status:logData.status, + action:logData.action, + actionStatus:logData.actionStatus, + user:logData.user, + platform:logData.platform, + containerName: logData.containerName, + image:logData.image, + os:logData.os, + logs:logData.logs + }; + if(logData.endedOn){ + logObj.endedOn = logData.endedOn; + }; + ContainerLogs.update({ + actionId: logData.actionId, + containerId: logData.containerId, + orgId: logData.orgId, + bgId: logData.bgId, + envId: logData.envId + }, { + $set: logObj + }, { + upsert: false + }, function(err, updatedData) { + if (err) { + logger.debug("Failed to update: ", err); + callback(err, null); + return; + } + callback(null, updatedData); + return; + }); + } else { + var log = new ContainerLogs(logData); + log.save(function(err, data) { + if (err) { + logger.error("Failed to insertLog", err); + if (typeof callback === 'function') { + callback(err, null); + } + return; + } + if (typeof callback === 'function') { + callback(null, data); + } + }); + } + + }); + }; + + this.getContainerActionLogs = function getContainerActionLogs(jsonData,callback){ + ContainerLogs.paginate(jsonData.queryObj, jsonData.options, function (err, containerLogList) { + if (err) { + var err = new Error('Internal server error'); + err.status = 500; + callback(err,null); + } + callback(null, containerLogList); + }); + } + +}; + + +module.exports = new ContainerLog(); diff --git a/server/app/routes/v1.0/routes_audit_trails.js b/server/app/routes/v1.0/routes_audit_trails.js index f1e6c3f4b..a70e640e8 100644 --- a/server/app/routes/v1.0/routes_audit_trails.js +++ b/server/app/routes/v1.0/routes_audit_trails.js @@ -20,6 +20,7 @@ var instanceService = require('_pr/services/instanceService'); var logger = require('_pr/logger')(module); var taskService = require('_pr/services/taskService'); var instanceLogModel = require('_pr/model/log-trail/instanceLog.js'); +var containerLogModel = require('_pr/model/log-trail/containerLog.js'); var apiUtil = require('_pr/lib/utils/apiUtil.js'); module.exports.setRoutes = function(app, sessionVerificationFunc) { @@ -75,7 +76,7 @@ module.exports.setRoutes = function(app, sessionVerificationFunc) { [ function(next) { - apiUtil.paginationRequest(req.query, 'taskLogs', next); + apiUtil.paginationRequest(req[0].query, 'taskLogs', next); }, function(paginationReq, next) { reqData = paginationReq; @@ -153,4 +154,35 @@ module.exports.setRoutes = function(app, sessionVerificationFunc) { return res.status(200).send(results); }); } + + + app.get('/audit-trail/container-action', getContainerActionList); + + function getContainerActionList(req, res, next) { + var reqData = {}; + async.waterfall( + [ + function(next) { + apiUtil.paginationRequest(req.query, 'containerLogs', next); + }, + function(paginationReq, next) { + paginationReq['searchColumns'] = ['platformId', 'status', 'action', 'user', 'actionStatus', 'orgName', 'bgName', 'projectName', 'envName', 'containerName', 'image']; + reqData = paginationReq; + apiUtil.databaseUtil(paginationReq, next); + }, + function(dataQuery, next) { + containerLogModel.getContainerActionLogs(dataQuery, next); + }, + function(instanceActions, next) { + apiUtil.paginationResponse(instanceActions, reqData, next); + } + ], + function(err, results) { + if (err) + return res.status(500).send(err); + else + return res.status(200).send(results); + }); + } + }; diff --git a/server/app/routes/v1.0/routes_instances.js b/server/app/routes/v1.0/routes_instances.js index e16f7c577..9f5fc8b04 100755 --- a/server/app/routes/v1.0/routes_instances.js +++ b/server/app/routes/v1.0/routes_instances.js @@ -56,6 +56,7 @@ var GCP = require('_pr/lib/gcp.js'); var async = require('async'); var apiUtil = require('_pr/lib/utils/apiUtil.js'); var instanceLogModel = require('_pr/model/log-trail/instanceLog.js'); +var containerLogModel = require('_pr/model/log-trail/containerLog.js'); var instanceService = require('_pr/services/instanceService'); var chefDao = require('_pr/model/dao/chefDao.js'); @@ -846,39 +847,44 @@ module.exports.setRoutes = function(app, sessionVerificationFunc) { log: "Docker-Container " + action.charAt(0).toUpperCase() + action.slice(1) + 'ing', timestamp: timestampStarted }); - var instanceLog = { + + var containerLog ={ actionId: actionLog._id, - instanceId: instance[0]._id, + containerId: containers[0].Id, orgName: instance[0].orgName, + orgId: instance[0].orgId, bgName: instance[0].bgName, + bgId: instance[0].bgId, projectName: instance[0].projectName, - envName: instance[0].environmentName, - status: instance[0].instanceState, + envName: instance[0].envName, + envId: instance[0].envId, + status: containers[0].Status, actionStatus: "pending", platformId: instance[0].platformId, - blueprintName: instance[0].blueprintData.blueprintName, - data: instance[0].runlist, + containerName: containers[0].Names, + Image: containers[0].Image, + ImageID: containers[0].ImageID, platform: instance[0].hardware.platform, os: instance[0].hardware.os, - size: instance[0].instanceType, - user: req.session.user.cn, + user: instance[0].catUser, createdOn: new Date().getTime(), startedOn: new Date().getTime(), - providerType: instance[0].providerType, + providerType: instance[0].providerType ? instance[0].providerType:null, action: actionObj, logs: [] }; + containerService.executeActionOnContainer(jsonData, function (err, containerResponse) { if (err) { - instanceLog.actionStatus = "failed"; - instanceLog.endedOn = new Date().getTime(); + containerLog.actionStatus = "failed"; + containerLog.endedOn = new Date().getTime(); logsDao.insertLog({ referenceId: logReferenceIds, err: true, log: 'Failed to Excute Docker command: ' + err, timestamp: timestampStarted }); - instanceLogModel.createOrUpdate(actionLog._id, instance[0]._id, instanceLog, function (err, logData) { + containerLogModel.createOrUpdate(containerLog, function(err, logData){ if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -888,8 +894,8 @@ module.exports.setRoutes = function(app, sessionVerificationFunc) { res.send(500); return; } - instanceLog.actionStatus = "success"; - instanceLog.endedOn = new Date().getTime(); + containerLog.actionStatus = "success"; + containerLog.endedOn = new Date().getTime(); if(action === 'stop'){ logsDao.insertLog({ referenceId: logReferenceIds, @@ -905,12 +911,12 @@ module.exports.setRoutes = function(app, sessionVerificationFunc) { timestamp: timestampStarted }); } - instancesDao.updateActionLog(instance[0]._id, actionLog._id, true, new Date().getTime()); - instanceLogModel.createOrUpdate(actionLog._id, instance[0]._id, instanceLog, function (err, logData) { + containerLogModel.createOrUpdate(containerLog, function(err, logData){ if (err) { logger.error("Failed to create or update instanceLog: ", err); } }); + instancesDao.updateActionLog(instance[0]._id, actionLog._id, true, new Date().getTime()); res.status(200).send(containerResponse); }); diff --git a/server/install.js b/server/install.js index 370278b28..7fd77c1ae 100755 --- a/server/install.js +++ b/server/install.js @@ -101,7 +101,8 @@ function getDefaultsConfig() { "unassignedInstances":"state", "chefNodes":"createdOn", "blueprints":"name", - 'compositeBlueprints':"name" + "compositeBlueprints":"name", + "containerLogs":"createdOn" }, skip_Records : 1, max_record_limit : 200, From 13545bf7e964e2b8396e255c26060e6d922ed1a3 Mon Sep 17 00:00:00 2001 From: Divakar Konakalla Date: Wed, 28 Sep 2016 16:09:48 +0530 Subject: [PATCH 02/59] Added new HTML for Container Logs audit trail, modified CSS for container Logs. --- client/htmls/public/containerLog.html | 295 +++++++++++++++++++++ client/htmls/public/css/auditTrailLogs.css | 2 +- 2 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 client/htmls/public/containerLog.html diff --git a/client/htmls/public/containerLog.html b/client/htmls/public/containerLog.html new file mode 100644 index 000000000..d5fe96896 --- /dev/null +++ b/client/htmls/public/containerLog.html @@ -0,0 +1,295 @@ + + + + + + + + + + + + + +
+

Container Audit Trail Logs

+
+ +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + +
Time + containerInstanceStatusImageActionStatusInstanceIPOSUserOrgBUProjectEnvActionLog
+
+
+ + + + + + + \ No newline at end of file diff --git a/client/htmls/public/css/auditTrailLogs.css b/client/htmls/public/css/auditTrailLogs.css index 7e6711e95..35621fd3c 100755 --- a/client/htmls/public/css/auditTrailLogs.css +++ b/client/htmls/public/css/auditTrailLogs.css @@ -184,7 +184,7 @@ table { background-image: url("/private/img/sort_desc.png"); } -#instanceActionListLoader, #taskActionListLoader { +#instanceActionListLoader, #taskActionListLoader, #containerActionListLoader { opacity: 0.6; height: 100%; width: 100%; From 35225a0b0349f1add4502f23223107ee084532a6 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Thu, 29 Sep 2016 10:40:31 +0530 Subject: [PATCH 03/59] Add category and short description for creating a orchestation job --- .../orchestration/popups/newTask.html | 34 +++++++++++++++++-- .../orchestration/popups/newTaskCtrl.js | 2 ++ server/app/model/classes/tasks/tasks.js | 8 +++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html index 89499be44..4c4f5c103 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html @@ -15,7 +15,7 @@