From b6c43aed1508b8f890e56516f2bd26371446c485 Mon Sep 17 00:00:00 2001 From: Gautam Jethwani Date: Tue, 28 May 2019 11:57:31 +0800 Subject: [PATCH 01/52] moved process.exit --- tools/wrench/dashboard/parser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/wrench/dashboard/parser.js b/tools/wrench/dashboard/parser.js index 824114ca6c..d6eb3b304d 100644 --- a/tools/wrench/dashboard/parser.js +++ b/tools/wrench/dashboard/parser.js @@ -50,12 +50,12 @@ else if (numProgramArguments == 2) { parseFile(process.argv[2]) .then(function(content) { opn('index.html') - process.exit() energyFilePath = process.argv[3]; energyData = JSON.parse(fs.readFileSync(energyFilePath)); addToHTMLFile(content, energyData); + process.exit() }) .catch(function(err) { console.log(err); From 13d52da0fe3f5e63804c2a4d0fce6277c767ee21 Mon Sep 17 00:00:00 2001 From: Gautam Jethwani Date: Tue, 28 May 2019 11:58:24 +0800 Subject: [PATCH 02/52] Remove unnecesary d3 incoude --- tools/wrench/dashboard/index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/wrench/dashboard/index.html b/tools/wrench/dashboard/index.html index 5f36ef0bf8..2f09417564 100644 --- a/tools/wrench/dashboard/index.html +++ b/tools/wrench/dashboard/index.html @@ -15,7 +15,6 @@ - From 0d5619b0ec378477c1e56075cfdab2141e6b4348 Mon Sep 17 00:00:00 2001 From: gjethwani Date: Mon, 20 May 2019 18:17:03 +0800 Subject: [PATCH 03/52] Add 3d file --- tools/wrench/dashboard/index.html | 3 +- tools/wrench/dashboard/public/d3-3d.js | 412 +++++++++++++++++++++++++ 2 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 tools/wrench/dashboard/public/d3-3d.js diff --git a/tools/wrench/dashboard/index.html b/tools/wrench/dashboard/index.html index 2f09417564..fdb3b8cb00 100644 --- a/tools/wrench/dashboard/index.html +++ b/tools/wrench/dashboard/index.html @@ -13,11 +13,10 @@ - + - diff --git a/tools/wrench/dashboard/public/d3-3d.js b/tools/wrench/dashboard/public/d3-3d.js new file mode 100644 index 0000000000..fdf10b5fd0 --- /dev/null +++ b/tools/wrench/dashboard/public/d3-3d.js @@ -0,0 +1,412 @@ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (factory((global.d3 = global.d3 || {}))); +}(this, (function (exports) { 'use strict'; + +function ccw(polygon) { + + var _p = polygon.slice(0), sum = 0; + + _p.push(_p[0]); + + for (var i = 0; i <= polygon.length - 1; i++) { + + var j = i + 1; + var p1 = _p[i].rotated; + var p2 = _p[j].rotated; + + sum += (p2.x - p1.x) * (p2.y + p1.y); + } + // if the area is positive + // the curve is counter-clockwise + // because of the flipped y-Axis in the browser + return sum > 0 ? true : false; +} + +function centroid(polygon){ + var _x = 0, _y = 0, _z = 0, _n = polygon.length; + + for (var i = _n - 1; i >= 0; i--) { + _x += polygon[i].rotated.x; + _y += polygon[i].rotated.y; + _z += polygon[i].rotated.z; + } + return { + x: _x / _n, + y: _y / _n, + z: _z / _n, + }; +} + +function rotateRzRyRx(po, angles){ + + var rc = angles.rotateCenter; + + po.x -= rc[0]; + po.y -= rc[1]; + po.z -= rc[2]; + + var rz = rotateZ(po, angles.z); + var ry = rotateY(rz, angles.y); + var rx = rotateX(ry, angles.x); + + rx.x += rc[0]; + rx.y += rc[1]; + rx.z += rc[2]; + + return rx; +} + +function rotateX(p, a){ + var sa = Math.sin(a), ca = Math.cos(a); + return { + x: p.x, + y: p.y * ca - p.z * sa, + z: p.y * sa + p.z * ca + }; +} + +function rotateY(p, a){ + var sa = Math.sin(a), ca = Math.cos(a); + return { + x: p.z * sa + p.x * ca, + y: p.y, + z: p.z * ca - p.x * sa + }; +} + +function rotateZ(p, a){ + var sa = Math.sin(a), ca = Math.cos(a); + return { + x: p.x * ca - p.y * sa, + y: p.x * sa + p.y * ca, + z: p.z + }; +} + +function point(points, options, point, angles){ + + for (var i = points.length - 1; i >= 0; i--) { + + var p = points[i]; + + p.rotated = rotateRzRyRx({x : point.x(p), y : point.y(p), z : point.z(p)}, angles); + p.centroid = p.rotated; + p.projected = options.project(p.rotated, options); + } + return points; +} + +function cube(cubes, options, point$$1, angles){ + for (var i = cubes.length - 1; i >= 0; i--) { + + var cube = cubes[i]; + + var vertices = point([ + cube[0], + cube[1], + cube[2], + cube[3], + cube[4], + cube[5], + cube[6], + cube[7] + ], options, point$$1, angles); + + var v1 = vertices[0]; + var v2 = vertices[1]; + var v3 = vertices[2]; + var v4 = vertices[3]; + var v5 = vertices[4]; + var v6 = vertices[5]; + var v7 = vertices[6]; + var v8 = vertices[7]; + + var front = [v1, v2, v3, v4]; + var back = [v8, v7, v6, v5]; + var left = [v5, v6, v2, v1]; + var right = [v4, v3, v7, v8]; + var top = [v5, v1, v4, v8]; + var bottom = [v2, v6, v7, v3]; + + front.centroid = centroid(front); + back.centroid = centroid(back); + left.centroid = centroid(left); + right.centroid = centroid(right); + top.centroid = centroid(top); + bottom.centroid = centroid(bottom); + + front.ccw = ccw(front); + back.ccw = ccw(back); + left.ccw = ccw(left); + right.ccw = ccw(right); + top.ccw = ccw(top); + bottom.ccw = ccw(bottom); + + front.face = 'front'; + back.face = 'back'; + left.face = 'left'; + right.face = 'right'; + top.face = 'top'; + bottom.face = 'bottom'; + + var faces = [front, back, left, right, top, bottom]; + + cube.faces = faces; + cube.centroid = {x: (left.centroid.x + right.centroid.x)/2, y: (top.centroid.y + bottom.centroid.y)/2, z: (front.centroid.z + back.centroid.z/2)}; + } + return cubes; +} + +function gridPlane(grid, options, point$$1, angles){ + + var points = point(grid, options, point$$1, angles); + var cnt = 0, planes = []; + var numPts = options.row; + var numRow = points.length/numPts; + + for (var i = numRow - 1; i > 0; i--) { + for (var j = numPts - 1; j > 0; j--) { + + var p1 = j + i * numPts, p4 = p1 - 1, p2 = p4 - numPts + 1, p3 = p2 - 1; + var pl = [points[p1], points[p2], points[p3], points[p4]]; + + pl.plane = 'plane_' + cnt++; + pl.ccw = ccw(pl); + pl.centroid = centroid(pl); + planes.push(pl); + } + } + + return planes; +} + +function lineStrip(lineStrip, options, point, angles){ + + for (var i = lineStrip.length - 1; i >= 0; i--) { + + var l = lineStrip[i], m = l.length/2, t = parseInt(m); + + for (var j = l.length - 1; j >= 0; j--) { + var p = l[j]; + p.rotated = rotateRzRyRx({x : point.x(p), y : point.y(p), z : point.z(p)}, angles); + p.projected = options.project(p.rotated, options); + } + + l.centroid = t === m ? centroid([ l[m - 1], l[m] ]) : { x: l[t].rotated.x, y: l[t].rotated.y, z: l[t].rotated.z }; + } + return lineStrip; +} + +function line(lines, options, point, angles){ + + for (var i = lines.length - 1; i >= 0; i--) { + + var line = lines[i]; + + var p1 = line[0]; + var p2 = line[1]; + + p1.rotated = rotateRzRyRx({x : point.x(p1), y : point.y(p1), z : point.z(p1)}, angles); + p2.rotated = rotateRzRyRx({x : point.x(p2), y : point.y(p2), z : point.z(p2)}, angles); + + p1.projected = options.project(p1.rotated, options); + p2.projected = options.project(p2.rotated, options); + + line.centroid = centroid(line); + } + return lines; +} + +function plane(planes, options, point, angles){ + + for (var i = planes.length - 1; i >= 0; i--) { + + var plane = planes[i]; + + var p1 = plane[0]; + var p2 = plane[1]; + var p3 = plane[2]; + var p4 = plane[3]; + + p1.rotated = rotateRzRyRx({x : point.x(p1), y : point.y(p1), z : point.z(p1)}, angles); + p2.rotated = rotateRzRyRx({x : point.x(p2), y : point.y(p2), z : point.z(p2)}, angles); + p3.rotated = rotateRzRyRx({x : point.x(p3), y : point.y(p3), z : point.z(p3)}, angles); + p4.rotated = rotateRzRyRx({x : point.x(p4), y : point.y(p4), z : point.z(p4)}, angles); + + p1.projected = options.project(p1.rotated, options); + p2.projected = options.project(p2.rotated, options); + p3.projected = options.project(p3.rotated, options); + p4.projected = options.project(p4.rotated, options); + + plane.ccw = ccw(plane); + plane.centroid = centroid(plane); + } + return planes; +} + +function triangle(triangles, options, point, angles){ + + for (var i = triangles.length - 1; i >= 0; i--) { + var tri = triangles[i]; + var p1 = tri[0]; + var p2 = tri[1]; + var p3 = tri[2]; + + p1.rotated = rotateRzRyRx({x : point.x(p1), y : point.y(p1), z : point.z(p1)}, angles); + p2.rotated = rotateRzRyRx({x : point.x(p2), y : point.y(p2), z : point.z(p2)}, angles); + p3.rotated = rotateRzRyRx({x : point.x(p3), y : point.y(p3), z : point.z(p3)}, angles); + + p1.projected = options.project(p1.rotated, options); + p2.projected = options.project(p2.rotated, options); + p3.projected = options.project(p3.rotated, options); + + tri.ccw = ccw(tri); + tri.centroid = centroid(tri); + } + return triangles; +} + +function drawLineStrip(lineStrip){ + var lastPoint = lineStrip[lineStrip.length - 1]; + var path = 'M' + lastPoint.projected.x + ',' + lastPoint.projected.y; + for (var i = lineStrip.length - 2; i >= 0; i--) { + var p = lineStrip[i].projected; + path += 'L' + p.x + ',' + p.y; + } + return path; +} + +function drawPlane(d){ + return 'M' + d[0].projected.x + ',' + d[0].projected.y + 'L' + d[1].projected.x + ',' + d[1].projected.y + 'L' + d[2].projected.x + ',' + d[2].projected.y + 'L' + d[3].projected.x + ',' + d[3].projected.y + 'Z'; +} + +function drawTriangle(d){ + return 'M' + d[0].projected.x + ',' + d[0].projected.y + 'L' + d[1].projected.x + ',' + d[1].projected.y + 'L' + d[2].projected.x + ',' + d[2].projected.y + 'Z'; +} + +function orthographic(d, options){ + return { + x: options.origin[0] + options.scale * d.x, + y: options.origin[1] + options.scale * d.y + }; +} + +function x(p) { + return p[0]; +} + +function y(p) { + return p[1]; +} + +function z(p) { + return p[2]; +} + +/** +* @author Stefan Nieke / http://niekes.com/ +*/ + +var _3d = function() { + + var origin = [0, 0], + scale = 1, + projection = orthographic, + angleX = 0, + angleY = 0, + angleZ = 0, + rotateCenter = [0,0,0], + x$$1 = x, + y$$1 = y, + z$$1 = z, + row = undefined, + shape = 'POINT', + processData = { + 'CUBE' : cube, + 'GRID' : gridPlane, + 'LINE' : line, + 'LINE_STRIP' : lineStrip, + 'PLANE' : plane, + 'POINT' : point, + 'SURFACE' : gridPlane, + 'TRIANGLE' : triangle, + }, + draw = { + 'CUBE' : drawPlane, + 'GRID' : drawPlane, + 'LINE_STRIP' : drawLineStrip, + 'PLANE' : drawPlane, + 'SURFACE' : drawPlane, + 'TRIANGLE' : drawTriangle, + }; + + function _3d(data){ + return processData[shape]( + data, + { scale: scale, origin: origin, project: projection, row: row }, + { x: x$$1, y: y$$1, z: z$$1 }, + { x: angleX, y: angleY, z: angleZ, rotateCenter: rotateCenter } + ); + } + + _3d.origin = function(_){ + return arguments.length ? (origin = _, _3d) : origin; + }; + + _3d.scale = function(_){ + return arguments.length ? (scale = _, _3d) : scale; + }; + + _3d.rotateX = function(_){ + return arguments.length ? (angleX = _, _3d) : angleX; + }; + + _3d.rotateY = function(_){ + return arguments.length ? (angleY = _, _3d) : angleY; + }; + + _3d.rotateZ = function(_){ + return arguments.length ? (angleZ = _, _3d) : angleZ; + }; + + _3d.shape = function(_, r){ + return arguments.length ? (shape = _, row = r, _3d) : shape; + }; + + _3d.rotateCenter = function(_){ + return arguments.length ? (rotateCenter = _, _3d) : rotateCenter; + }; + + _3d.x = function(_){ + return arguments.length ? (x$$1 = typeof _ === 'function' ? _ : +_, _3d) : x$$1; + }; + + _3d.y = function(_){ + return arguments.length ? (y$$1 = typeof _ === 'function' ? _ : +_, _3d) : y$$1; + }; + + _3d.z = function(_){ + return arguments.length ? (z$$1 = typeof _ === 'function' ? _ : +_, _3d) : z$$1; + }; + + _3d.sort = function(a, b){ + var _a = a.centroid.z, _b = b.centroid.z; + return _a < _b ? -1 : _a > _b ? 1 : _a >= _b ? 0 : NaN; + }; + + _3d.draw = function(d){ + if(!((shape === 'POINT') || (shape === 'LINE'))){ + return draw[shape](d); + } + }; + + return _3d; +}; + +exports._3d = _3d; + +Object.defineProperty(exports, '__esModule', { value: true }); + +}))); From eb6ea0b015fe309ecafe1eacfb93175d1aec687e Mon Sep 17 00:00:00 2001 From: Gautam Jethwani Date: Thu, 23 May 2019 15:20:11 +0800 Subject: [PATCH 04/52] Added grid and y axis --- tools/wrench/dashboard/index.html | 12 ++ tools/wrench/dashboard/playground/grid.html | 118 ++++++++++++ tools/wrench/dashboard/playground/index.html | 178 +++++++++++++++++++ tools/wrench/dashboard/public/scripts.js | 173 ++++++++++++++++++ 4 files changed, 481 insertions(+) create mode 100644 tools/wrench/dashboard/playground/grid.html create mode 100644 tools/wrench/dashboard/playground/index.html diff --git a/tools/wrench/dashboard/index.html b/tools/wrench/dashboard/index.html index fdb3b8cb00..88871096d9 100644 --- a/tools/wrench/dashboard/index.html +++ b/tools/wrench/dashboard/index.html @@ -150,6 +150,18 @@
Energy Graphs
+ +
+ +
+ + + + + + + + \ No newline at end of file diff --git a/tools/wrench/dashboard/playground/index.html b/tools/wrench/dashboard/playground/index.html new file mode 100644 index 0000000000..a3785530cb --- /dev/null +++ b/tools/wrench/dashboard/playground/index.html @@ -0,0 +1,178 @@ + + + + + + + + + \ No newline at end of file diff --git a/tools/wrench/dashboard/public/scripts.js b/tools/wrench/dashboard/public/scripts.js index 77034cbd1d..cf190602c3 100644 --- a/tools/wrench/dashboard/public/scripts.js +++ b/tools/wrench/dashboard/public/scripts.js @@ -22,6 +22,7 @@ function initialise() { populateLegend("taskView") populateWorkflowTaskDataTable(data.contents) getOverallWorkflowMetrics(data.contents) + generate3dGraph(data.contents) } } @@ -815,4 +816,176 @@ function toggleView() { informationImg.style.display = "none" currGraphState = "taskView" } +} + +function determineTaskEnd(d) { + var taskEnd + if (d.terminated !== -1) { + taskEnd = d.terminated + } else if (d.failed !== -1) { + taskEnd = d.failed + } else { + taskEnd = d.whole_task.end + } + return taskEnd +} + +function determineTaskOverlap(data) { + var taskOverlap = {} + data.forEach(function(d) { + var taskStart = d.whole_task.start + var taskEnd = determineTaskEnd(d) + if (Object.keys(taskOverlap).length === 0) { + taskOverlap[0] = [] + taskOverlap[0].push(d) + } else { + var i = 0 + var placed = false + while (!placed) { + if (taskOverlap[i] === undefined) { + taskOverlap[i] = [] + } + var overlap = false + for (var j = 0; j < taskOverlap[i].length; j++) { + var t = taskOverlap[i][j] + var currTaskStart = t.whole_task.start + var currTaskEnd = determineTaskEnd(t) + if ((taskStart >= currTaskStart && taskStart <= currTaskEnd) || (taskEnd >= currTaskStart && taskEnd <= currTaskEnd)) { + i++ + overlap = true + break + } + } + if (!overlap) { + taskOverlap[i].push(d) + placed = true + } + } + } + }) + return taskOverlap +} + +function determineMaxNumCoresAllocated(data) { + var max = 0 + data.forEach(function(d) { + if (d.num_cores_allocated >= max) { + max = d.num_cores_allocated + } + }) + return max +} + +var origin = [0, 400] +startAngle = Math.PI/4 +var scale = 20 +var key = function(d) { return d.task_id; } + +function processData(data, tt){ + + var grid3d = d3._3d() + .shape('GRID', 20) + .origin(origin) + .rotateY( startAngle) + .rotateX(-startAngle) + .scale(scale); + + var yScale3d = d3._3d() + .shape('LINE_STRIP') + .origin(origin) + .rotateY( startAngle) + .rotateX(-startAngle) + .scale(scale); + + var svg = d3.select('#three-d-graph-svg').append('g'); + + /* ----------- GRID ----------- */ + + var xGrid = svg.selectAll('path.grid').data(data[0], key); + + xGrid + .enter() + .append('path') + .attr('class', '_3d grid') + .merge(xGrid) + .attr('stroke', 'black') + .attr('stroke-width', 0.3) + .attr('fill', function(d){ return d.ccw ? 'lightgrey' : '#717171'; }) + .attr('fill-opacity', 0.9) + .attr('d', grid3d.draw); + + xGrid.exit().remove(); + + /* ----------- y-Scale ----------- */ + + var yScale = svg.selectAll('path.yScale').data(data[1]); + + console.log(yScale) + + yScale + .enter() + .append('path') + .attr('class', '_3d yScale') + .merge(yScale) + .attr('stroke', 'black') + .attr('stroke-width', .5) + .attr('d', yScale3d.draw); + + yScale.exit().remove(); + + /* ----------- y-Scale Text ----------- */ + + var yText = svg.selectAll('text.yText').data(data[1][0]); + + yText + .enter() + .append('text') + .attr('class', '_3d yText') + .attr('dx', '.3em') + .merge(yText) + .each(function(d){ + d.centroid = {x: d.rotated.x, y: d.rotated.y, z: d.rotated.z}; + }) + .attr('x', function(d){ return d.projected.x; }) + .attr('y', function(d){ return d.projected.y; }) + .text(function(d){ return d[1] <= 0 ? d[1] : ''; }); + + yText.exit().remove(); +} + +function generate3dGraph(data) { + var grid3d = d3._3d() + .shape('GRID', 20) + .origin(origin) + .rotateY( startAngle) + .rotateX(-startAngle) + .scale(scale); + + var yScale3d = d3._3d() + .shape('LINE_STRIP') + .origin(origin) + .rotateY( startAngle) + .rotateX(-startAngle) + .scale(scale); + var j = 10 + var maxTime = d3.max(data, function(d) { + return Math.max(d['whole_task'].end, d['failed'], d['terminated']) + }) + var taskOverlap = determineTaskOverlap(data) + var maxNumCoresAllocated = determineMaxNumCoresAllocated(data) + xGrid = [], scatter = [], yLine = []; + for(var z = 0; z < Object.keys(taskOverlap).length; z++){ + for(var x = 0; x < maxTime; x++) { + xGrid.push([x, 1, z]); + } + } + + d3.range(-1, maxNumCoresAllocated + 1, 1).forEach(function(d) { yLine.push([0, -d, 0]); }); + + var data = [ + grid3d(xGrid), + yScale3d([yLine]) + ]; + processData(data, 1000); + } \ No newline at end of file From 29a595064cf37f4002b81cdffc8e39b9d4f1403c Mon Sep 17 00:00:00 2001 From: gjethwani Date: Fri, 24 May 2019 10:39:32 +0800 Subject: [PATCH 05/52] Cubes dont work --- tools/wrench/dashboard/playground/index.html | 73 ++++++++++++- tools/wrench/dashboard/public/scripts.js | 107 +++++++++++++++++-- 2 files changed, 168 insertions(+), 12 deletions(-) diff --git a/tools/wrench/dashboard/playground/index.html b/tools/wrench/dashboard/playground/index.html index a3785530cb..2688ff7275 100644 --- a/tools/wrench/dashboard/playground/index.html +++ b/tools/wrench/dashboard/playground/index.html @@ -14,6 +14,7 @@ @@ -164,6 +163,7 @@
3D Visualisation
+ diff --git a/tools/wrench/dashboard/public/scripts.js b/tools/wrench/dashboard/public/scripts.js index 48c3ea7cec..85b6f8c39b 100644 --- a/tools/wrench/dashboard/public/scripts.js +++ b/tools/wrench/dashboard/public/scripts.js @@ -23,7 +23,7 @@ function initialise() { populateLegend("taskView") populateWorkflowTaskDataTable(data.contents) getOverallWorkflowMetrics(data.contents) - generate3dGraph(data.contents) + generate3dGraph(data.contents, true) } } @@ -872,7 +872,7 @@ function determineMaxNumCoresAllocated(data) { return max } -var origin = [0, 500] +var origin = [0, 200] var startAngle = Math.PI/4 var yAngle = startAngle var xAngle = -startAngle @@ -920,32 +920,6 @@ var cubes3d = d3._3d() .origin(origin) .scale(scale) -var originXBox = document.getElementById('origin-x') -var originYBox = document.getElementById('origin-y') -var timeIntervalBox = document.getElementById('time-interval') -var scaleBox = document.getElementById('scale-input') - -originXBox.value = origin[0] -originYBox.value = origin[1] -timeIntervalBox.value = timeScalingFactor -scaleBox.value = scale - -originXBox.onchange = function(e) { - changeOriginOrScale([parseInt(e.target.value), origin[1]], scale) -} - -originYBox.onchange = function(e) { - changeOriginOrScale([origin[0], parseInt(e.target.value)], scale) -} - -timeIntervalBox.onchange = function(e) { - changeTimeScalingFactor(parseInt(e.target.value)) -} - -scaleBox.onchange = function(e) { - changeOriginOrScale(origin, e.target.value) -} - function searchOverlap(taskId, taskOverlap) { for (var key in taskOverlap) { if (taskOverlap.hasOwnProperty(key)) { @@ -989,7 +963,7 @@ function dragged(){ data.contents, cubes3d.rotateY(beta + startAngle).rotateX(alpha - startAngle)(ftCubesData) ]; - processData(rotatedData, 0); + processData(rotatedData, 0, false); } function dragStart(){ @@ -1088,7 +1062,7 @@ function deselectCube(hover) { hideTooltip() } -function processData(data, tt){ +function processData(data, tt, populateLegend){ /* ----------- GRID ----------- */ var xGrid = svg.selectAll('path.grid').data(data[0], key); @@ -1190,15 +1164,17 @@ function processData(data, tt){ var legend = d3.select('#three-d-legend') - legend.append("small") - .attr("class", "inline-block") - .style("border-left", "15px solid orange") - .text("Terminated by User") + if (populateLegend) { + legend.append("small") + .attr("class", "inline-block") + .style("border-left", "15px solid orange") + .text("Terminated by User") - legend.append("small") - .attr("class", "inline-block") - .style("border-left", "15px solid red") - .text("Failed During Execution") + legend.append("small") + .attr("class", "inline-block") + .style("border-left", "15px solid red") + .text("Failed During Execution") + } var ce = cubes .enter() @@ -1289,7 +1265,37 @@ function processData(data, tt){ } -function generate3dGraph(data) { +function generate3dGraph(data, populateLegend) { + var originXBox = document.getElementById('origin-x') + var originYBox = document.getElementById('origin-y') + var timeIntervalBox = document.getElementById('time-interval') + var scaleBox = document.getElementById('scale-input') + + originXBox.value = origin[0] + originYBox.value = origin[1] + timeIntervalBox.value = timeScalingFactor + scaleBox.value = scale + + originXBox.onchange = function(e) { + changeOriginOrScale([parseInt(e.target.value), origin[1]], scale) + } + + originYBox.onchange = function(e) { + changeOriginOrScale([origin[0], parseInt(e.target.value)], scale) + } + + timeIntervalBox.onchange = function(e) { + changeTimeScalingFactor(parseInt(e.target.value)) + } + + scaleBox.onchange = function(e) { + changeOriginOrScale(origin, e.target.value) + } + + if (firstVisit) { + showInstructions("three-d-instructions", "three-d-information-img") + } + xGrid = [], scatter = [], yLine = [], xLine = [] for(var z = 0; z <= maxTime + timeScalingFactor; z+=timeScalingFactor){ for(var x = 0; x < maxTaskOverlap; x++) { @@ -1332,7 +1338,7 @@ function generate3dGraph(data) { data, cubes3d(ftCubesData) ]; - processData(threeDdata, 1000); + processData(threeDdata, 1000, populateLegend); } @@ -1371,10 +1377,10 @@ function changeOriginOrScale(newOrigin, newScale) { data.contents, cubes3d(ftCubesData) ] - processData(newOriginData, 1000) + processData(newOriginData, 1000, false) } function changeTimeScalingFactor(factor) { timeScalingFactor = factor - generate3dGraph(data.contents) + generate3dGraph(data.contents, false) } \ No newline at end of file From 51a522c64320b46463df4d55b42d6e0cfec480c0 Mon Sep 17 00:00:00 2001 From: gjethwani Date: Sun, 2 Jun 2019 22:57:09 +0800 Subject: [PATCH 36/52] empty out data --- tools/wrench/dashboard/public/scripts.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/wrench/dashboard/public/scripts.js b/tools/wrench/dashboard/public/scripts.js index 85b6f8c39b..8db27b5934 100644 --- a/tools/wrench/dashboard/public/scripts.js +++ b/tools/wrench/dashboard/public/scripts.js @@ -1,5 +1,5 @@ -var data={"modified":"2019-06-02T03:09:44.509Z","file":"test_data/workflow_data.json","contents":[{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task0","terminated":-1,"vertical_position":0,"whole_task":{"end":3827.6378817372047,"start":0.007199838144329895},"write":{"end":3827.6378817372047,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task1","terminated":-1,"vertical_position":1,"whole_task":{"end":3827.635010247309,"start":0.007199838144329895},"write":{"end":3827.635010247309,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task10","terminated":-1,"vertical_position":2,"whole_task":{"end":3827.639191182566,"start":0.007199838144329895},"write":{"end":3827.639191182566,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_2","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task11","terminated":-1,"vertical_position":0,"whole_task":{"end":3827.6411369314314,"start":0.007199838144329895},"write":{"end":3827.6411369314314,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_2","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task12","terminated":-1,"vertical_position":1,"whole_task":{"end":3827.641331233935,"start":0.007199838144329895},"write":{"end":3827.641331233935,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_2","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task13","terminated":-1,"vertical_position":2,"whole_task":{"end":3827.6414647120796,"start":0.007199838144329895},"write":{"end":3827.6414647120796,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_3","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task14","terminated":-1,"vertical_position":0,"whole_task":{"end":3827.641553979021,"start":0.007199838144329895},"write":{"end":3827.641553979021,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_3","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task15","terminated":-1,"vertical_position":1,"whole_task":{"end":3827.641607708093,"start":0.007199838144329895},"write":{"end":3827.641607708093,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_3","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task16","terminated":-1,"vertical_position":2,"whole_task":{"end":3827.6416322071746,"start":0.007199838144329895},"write":{"end":3827.6416322071746,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_4","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task17","terminated":-1,"vertical_position":0,"whole_task":{"end":3827.640506539679,"start":0.007199838144329895},"write":{"end":3827.640506539679,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_4","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task18","terminated":-1,"vertical_position":1,"whole_task":{"end":3827.6399793833903,"start":0.007199838144329895},"write":{"end":3827.6399793833903,"start":3807.8470265752576}},{"compute":{"end":3807.8470265752576,"start":207.84676637525777},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_4","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":207.84676637525777,"start":0.007199838144329895},"task_id":"task19","terminated":-1,"vertical_position":2,"whole_task":{"end":3827.6408767314315,"start":0.007199838144329895},"write":{"end":3827.6408767314315,"start":3807.8470265752576}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task2","terminated":-1,"vertical_position":0,"whole_task":{"end":7579.408645646651,"start":3827.6507028638657},"write":{"end":7579.408645646651,"start":7566.212108144485}},{"compute":{"end":7912.423338350066,"start":7612.4230781500655},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":7612.4230781500655,"start":7579.417740802427},"task_id":"task20","terminated":-1,"vertical_position":0,"whole_task":{"end":7929.747237014191,"start":7579.417740802427},"write":{"end":7929.747237014191,"start":7912.423338350066}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task3","terminated":-1,"vertical_position":1,"whole_task":{"end":7579.408772085021,"start":3827.6507028638657},"write":{"end":7579.408772085021,"start":7566.212108144485}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_1","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task4","terminated":-1,"vertical_position":2,"whole_task":{"end":7579.408734069057,"start":3827.6507028638657},"write":{"end":7579.408734069057,"start":7566.212108144485}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_2","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task5","terminated":-1,"vertical_position":0,"whole_task":{"end":7579.405173391115,"start":3827.6507028638657},"write":{"end":7579.405173391115,"start":7566.212108144485}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_2","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task6","terminated":-1,"vertical_position":1,"whole_task":{"end":7579.4070007028695,"start":3827.6507028638657},"write":{"end":7579.4070007028695,"start":7566.212108144485}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_2","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task7","terminated":-1,"vertical_position":2,"whole_task":{"end":7579.407786370086,"start":3827.6507028638657},"write":{"end":7579.407786370086,"start":7566.212108144485}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_3","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task8","terminated":-1,"vertical_position":0,"whole_task":{"end":7579.408224259434,"start":3827.6507028638657},"write":{"end":7579.408224259434,"start":7566.212108144485}},{"compute":{"end":7566.212108144485,"start":3966.2118479444844},"execution_host":{"cores":12,"flop_rate":1000000000000,"hostname":"hpc.edu/node_3","memory":32000000000},"failed":-1,"num_cores_allocated":1,"read":{"end":3966.2118479444844,"start":3827.6507028638657},"task_id":"task9","terminated":-1,"vertical_position":1,"whole_task":{"end":7579.4084844594345,"start":3827.6507028638657},"write":{"end":7579.4084844594345,"start":7566.212108144485}}]} -var energyData=[{"consumed_energy_trace":[{"time":0,"joules":0},{"time":2,"joules":400},{"time":4,"joules":800},{"time":6,"joules":1200}],"hostname":"host1","pstate_trace":[{"pstate":1,"time":0},{"pstate":0,"time":2}],"pstates":[{"idle":"100.0","pstate":0,"running":"200.0","speed":100000000},{"idle":" 93.0","pstate":1,"running":"170.0","speed":50000000},{"idle":" 90.0","pstate":2,"running":"150.0","speed":20000000}],"watt_off":"10"},{"consumed_energy_trace":[{"time":0,"joules":0},{"time":2,"joules":200},{"time":4,"joules":400},{"time":6,"joules":600}],"hostname":"host2","pstate_trace":[{"pstate":0,"time":0},{"pstate":1,"time":2}],"pstates":[{"idle":"100.0","pstate":0,"running":"200.0","speed":100000000},{"idle":" 93.0","pstate":1,"running":"170.0","speed":50000000},{"idle":" 90.0","pstate":2,"running":"150.0","speed":20000000}],"watt_off":"10"}] +var data={} +var energyData=[] var currGraphState = "taskView" var hostColours = {} var currentlySelectedHost = {hostName: "", id: ""} From eb17d9fd783a5d235ce9d7faf807c0abf4998014 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Tue, 4 Jun 2019 17:31:17 +0200 Subject: [PATCH 37/52] fix for the bug reported in Issue #104 --- src/wrench/logging/TerminalOutput.cpp | 4 ++-- src/wrench/wms/WMS.cpp | 1 - test/main.cpp | 2 +- test/simulated_failures/CloudComputeServiceFailuresTest.cpp | 1 - 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/wrench/logging/TerminalOutput.cpp b/src/wrench/logging/TerminalOutput.cpp index b2f7e27096..f26ab35bc9 100644 --- a/src/wrench/logging/TerminalOutput.cpp +++ b/src/wrench/logging/TerminalOutput.cpp @@ -27,7 +27,7 @@ namespace wrench { std::map TerminalOutput::colormap; bool TerminalOutput::color_enabled = true; - bool TerminalOutput::wrench_no_log = true; + bool TerminalOutput::wrench_no_log = false; /** * @brief Set the color of log messages printed to the terminal @@ -67,7 +67,7 @@ namespace wrench { * @brief Disable log altogether */ void TerminalOutput::disableLog() { - TerminalOutput::wrench_no_log = false; + TerminalOutput::wrench_no_log = true; } /** diff --git a/src/wrench/wms/WMS.cpp b/src/wrench/wms/WMS.cpp index 84793e624f..7738801e32 100644 --- a/src/wrench/wms/WMS.cpp +++ b/src/wrench/wms/WMS.cpp @@ -294,7 +294,6 @@ namespace wrench { std::shared_ptr WMS::createJobManager() { std::shared_ptr job_manager = std::shared_ptr( new JobManager(this->getSharedPtr())); - WRENCH_INFO("ASDASDA"); job_manager->simulation = this->simulation; job_manager->start(job_manager, true, false); // Always daemonize, no auto-restart diff --git a/test/main.cpp b/test/main.cpp index 72facfbc3d..0d4b4f2cb5 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -12,7 +12,7 @@ int main(int argc, char **argv) { // disable log - xbt_log_control_set("root.thresh:critical"); +// xbt_log_control_set("root.thresh:critical"); // xbt_log_control_set("simulation_timestamps.thresh:debug"); // Example selective log enabling diff --git a/test/simulated_failures/CloudComputeServiceFailuresTest.cpp b/test/simulated_failures/CloudComputeServiceFailuresTest.cpp index b2c3ada477..87a1740001 100644 --- a/test/simulated_failures/CloudComputeServiceFailuresTest.cpp +++ b/test/simulated_failures/CloudComputeServiceFailuresTest.cpp @@ -245,7 +245,6 @@ class CloudServiceFailureOfAVMAndRestartTestWMS : public wrench::WMS { resurector->simulation = this->simulation; resurector->start(resurector, true, false); // Daemonized, no auto-restart - wrench::Simulation::sleep(10); // Create a VM on the Cloud Service From 9393c74237baeddfe88cf2903cd4c268aabc5dc0 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Wed, 5 Jun 2019 11:34:46 +0200 Subject: [PATCH 38/52] Added the option to supply a desired VM name when creating a new VM in a Cloud/VC Compute Service Added a corresponding test --- .../compute/cloud/CloudComputeService.h | 7 + src/wrench/services/Service.cpp | 2 +- .../compute/cloud/CloudComputeService.cpp | 80 ++++++++--- .../cloud/CloudComputeServiceMessage.cpp | 4 +- .../cloud/CloudComputeServiceMessage.h | 3 + .../VirtualizedClusterComputeService.cpp | 2 +- .../VirtualizedClusterServiceTest.cpp | 131 ++++++++++++++++++ .../MessageConstructorTest.cpp | 4 +- test/main.cpp | 5 +- 9 files changed, 213 insertions(+), 25 deletions(-) diff --git a/include/wrench/services/compute/cloud/CloudComputeService.h b/include/wrench/services/compute/cloud/CloudComputeService.h index 7540899eef..5a9ab93244 100644 --- a/include/wrench/services/compute/cloud/CloudComputeService.h +++ b/include/wrench/services/compute/cloud/CloudComputeService.h @@ -85,6 +85,12 @@ namespace wrench { std::map property_list = {}, std::map messagepayload_list = {}); + virtual std::string createVM(unsigned long num_cores, + double ram_memory, + std::string desired_vm_name, + std::map property_list = {}, + std::map messagepayload_list = {}); + virtual void shutdownVM(const std::string &vm_name); virtual std::shared_ptr startVM(const std::string &vm_name); @@ -145,6 +151,7 @@ namespace wrench { virtual void processCreateVM(const std::string &answer_mailbox, unsigned long requested_num_cores, double requested_ram, + std::string desired_vm_name, std::map property_list, std::map messagepayload_list ); diff --git a/src/wrench/services/Service.cpp b/src/wrench/services/Service.cpp index 7aa52a54ae..cd78a8636b 100644 --- a/src/wrench/services/Service.cpp +++ b/src/wrench/services/Service.cpp @@ -250,7 +250,7 @@ namespace wrench { // Start the daemon for the service this->startDaemon(daemonize, auto_restart); // Print some information a out the currently tracked daemons - WRENCH_INFO("MAP SIZE = %ld NUM_TERMINATED_SERVICES = %ld", + WRENCH_DEBUG("MAP SIZE = %ld NUM_TERMINATED_SERVICES = %ld", Service::service_shared_ptr_map.size(), Service::num_terminated_services); if ((Service::service_shared_ptr_map.size() > 1000) or (Service::num_terminated_services > Service::service_shared_ptr_map.size() / 2)) { diff --git a/src/wrench/services/compute/cloud/CloudComputeService.cpp b/src/wrench/services/compute/cloud/CloudComputeService.cpp index 61c6704367..4d3a2ec371 100644 --- a/src/wrench/services/compute/cloud/CloudComputeService.cpp +++ b/src/wrench/services/compute/cloud/CloudComputeService.cpp @@ -124,6 +124,28 @@ namespace wrench { double ram_memory, std::map property_list, std::map messagepayload_list) { + return this->createVM(num_cores, ram_memory, "", property_list, messagepayload_list); + } + + /** + * @brief Create a BareMetalComputeService VM (balances load on execution hosts) + * + * @param num_cores: the number of cores for the VM + * @param ram_memory: the VM's RAM memory capacity + * @param desired_vm_name: the VM's desired name ("" means "pick a name for me") + * @param property_list: a property list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") + * @param messagepayload_list: a message payload list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") + * + * @return A VM name + * + * @throw WorkflowExecutionException + * @throw std::runtime_error + */ + std::string CloudComputeService::createVM(unsigned long num_cores, + double ram_memory, + std::string desired_vm_name, + std::map property_list, + std::map messagepayload_list) { if (num_cores == ComputeService::ALL_CORES) { @@ -144,7 +166,7 @@ namespace wrench { answer_mailbox, new CloudComputeServiceCreateVMRequestMessage( answer_mailbox, - num_cores, ram_memory, property_list, messagepayload_list, + num_cores, ram_memory, desired_vm_name, property_list, messagepayload_list, this->getMessagePayloadValue( CloudComputeServiceMessagePayload::CREATE_VM_REQUEST_MESSAGE_PAYLOAD))); @@ -628,7 +650,7 @@ namespace wrench { return true; } else if (auto msg = std::dynamic_pointer_cast(message)) { - processCreateVM(msg->answer_mailbox, msg->num_cores, msg->ram_memory, msg->property_list, + processCreateVM(msg->answer_mailbox, msg->num_cores, msg->ram_memory, msg->desired_vm_name, msg->property_list, msg->messagepayload_list); return true; @@ -700,6 +722,7 @@ namespace wrench { * @param answer_mailbox: the mailbox to which the answer message should be sent * @param requested_num_cores: the number of cores the service can use * @param requested_ram: the VM's RAM memory capacity + * @param desired_vm_name: the desired VM name ("" means "pick a name for me") * @param property_list: a property list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") * @param messagepayload_list: a message payload list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") * @@ -708,6 +731,7 @@ namespace wrench { void CloudComputeService::processCreateVM(const std::string &answer_mailbox, unsigned long requested_num_cores, double requested_ram, + std::string desired_vm_name, std::map property_list, std::map messagepayload_list) { @@ -744,25 +768,45 @@ namespace wrench { } else { // Pick a VM name (and being paranoid about mistakenly picking an actual hostname!) - std::string vm_name; - do { - vm_name = this->getName() + "_vm" + std::to_string(CloudComputeService::VM_ID++); - } while (S4U_Simulation::hostExists(vm_name)); + std::string vm_name = ""; - // Create the VM - auto vm = std::shared_ptr( - new S4U_VirtualMachine(vm_name, requested_num_cores, requested_ram, property_list, - messagepayload_list)); + if (desired_vm_name.empty()) { + do { + vm_name = this->getName() + "_vm" + std::to_string(CloudComputeService::VM_ID++); + } while (S4U_Simulation::hostExists(vm_name)); + } else { + if (this->vm_list.find(desired_vm_name) == this->vm_list.end()) { + vm_name = desired_vm_name; + } + } - // Add the VM to the list of VMs, with (for now) a nullptr compute service - this->vm_list[vm_name] = std::make_pair(vm, nullptr); + if (vm_name.empty()) { + std::string empty = std::string(); + std::string error_msg = "Invalid requested VM name"; + msg_to_send_back = new CloudComputeServiceCreateVMAnswerMessage( + false, + empty, + std::shared_ptr( + new NotAllowed(this->getSharedPtr(), error_msg)), + this->getMessagePayloadValue( + CloudComputeServiceMessagePayload::CREATE_VM_ANSWER_MESSAGE_PAYLOAD)); + } else { - msg_to_send_back = new CloudComputeServiceCreateVMAnswerMessage( - true, - vm_name, - nullptr, - this->getMessagePayloadValue( - CloudComputeServiceMessagePayload::CREATE_VM_ANSWER_MESSAGE_PAYLOAD)); + // Create the VM + auto vm = std::shared_ptr( + new S4U_VirtualMachine(vm_name, requested_num_cores, requested_ram, property_list, + messagepayload_list)); + + // Add the VM to the list of VMs, with (for now) a nullptr compute service + this->vm_list[vm_name] = std::make_pair(vm, nullptr); + + msg_to_send_back = new CloudComputeServiceCreateVMAnswerMessage( + true, + vm_name, + nullptr, + this->getMessagePayloadValue( + CloudComputeServiceMessagePayload::CREATE_VM_ANSWER_MESSAGE_PAYLOAD)); + } } // Send reply diff --git a/src/wrench/services/compute/cloud/CloudComputeServiceMessage.cpp b/src/wrench/services/compute/cloud/CloudComputeServiceMessage.cpp index d07bfe37fb..e18b57691c 100644 --- a/src/wrench/services/compute/cloud/CloudComputeServiceMessage.cpp +++ b/src/wrench/services/compute/cloud/CloudComputeServiceMessage.cpp @@ -62,6 +62,7 @@ namespace wrench { * available on the host) * @param ram_memory: the VM's RAM memory capacity (use ComputeService::ALL_RAM to use all RAM available on the * host, this can be lead to an out of memory issue) + * @param desired_vm_name: the desired VM name ("" means "pick a name for me") * @param property_list: a property list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") * @param messagepayload_list: a message payload list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") * @param payload: the message size in bytes @@ -72,11 +73,12 @@ namespace wrench { const std::string &answer_mailbox, unsigned long num_cores, double ram_memory, + std::string desired_vm_name, std::map property_list, std::map messagepayload_list, double payload) : CloudComputeServiceMessage("CREATE_VM_REQUEST", payload), - num_cores(num_cores), ram_memory(ram_memory), property_list(property_list), + num_cores(num_cores), ram_memory(ram_memory), desired_vm_name(desired_vm_name), property_list(property_list), messagepayload_list(messagepayload_list) { if (answer_mailbox.empty() || (ram_memory < 0.0)) { diff --git a/src/wrench/services/compute/cloud/CloudComputeServiceMessage.h b/src/wrench/services/compute/cloud/CloudComputeServiceMessage.h index 3397431b93..23b16f20d3 100644 --- a/src/wrench/services/compute/cloud/CloudComputeServiceMessage.h +++ b/src/wrench/services/compute/cloud/CloudComputeServiceMessage.h @@ -62,6 +62,7 @@ namespace wrench { CloudComputeServiceCreateVMRequestMessage(const std::string &answer_mailbox, unsigned long num_cores, double ram_memory, + std::string desired_vm_name, std::map property_list, std::map messagepayload_list, double payload); @@ -73,6 +74,8 @@ namespace wrench { unsigned long num_cores; /** @brief The VM RAM memory capacity (0 means "use all memory available on the host", this can be lead to out of memory issue) */ double ram_memory; + /** @brief The desired name for the VM ("" means "pick for me") */ + std::string desired_vm_name; /** @brief A property list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") */ std::map property_list; /** @brief A message payload list for the BareMetalComputeService that will run on the VM ({} means "use all defaults") */ diff --git a/src/wrench/services/compute/virtualized_cluster/VirtualizedClusterComputeService.cpp b/src/wrench/services/compute/virtualized_cluster/VirtualizedClusterComputeService.cpp index e110e3e1b3..c0fe7a51a1 100644 --- a/src/wrench/services/compute/virtualized_cluster/VirtualizedClusterComputeService.cpp +++ b/src/wrench/services/compute/virtualized_cluster/VirtualizedClusterComputeService.cpp @@ -202,7 +202,7 @@ namespace wrench { return true; } else if (auto msg = std::dynamic_pointer_cast(message)) { - processCreateVM(msg->answer_mailbox, msg->num_cores, msg->ram_memory, msg->property_list, + processCreateVM(msg->answer_mailbox, msg->num_cores, msg->ram_memory, msg->desired_vm_name, msg->property_list, msg->messagepayload_list); return true; diff --git a/test/compute_services/VirtualizedClusterServiceTest.cpp b/test/compute_services/VirtualizedClusterServiceTest.cpp index ccc96a1fd6..817536c6db 100644 --- a/test/compute_services/VirtualizedClusterServiceTest.cpp +++ b/test/compute_services/VirtualizedClusterServiceTest.cpp @@ -39,6 +39,8 @@ class VirtualizedClusterServiceTest : public ::testing::Test { void do_StandardJobTaskTest_test(); + void do_StandardJobTaskWithCustomVMNameTest_test(); + void do_VMMigrationTest_test(); void do_NumCoresTest_test(); @@ -268,6 +270,135 @@ void VirtualizedClusterServiceTest::do_StandardJobTaskTest_test() { free(argv); } + +/***********************************************************************************/ +/** STANDARD JOB SUBMISSION TASK SIMULATION TEST WITH CUSTOM VM NAME ON ONE HOST **/ +/***********************************************************************************/ + +class CloudStandardJobWithCustomVMNameTestWMS : public wrench::WMS { + +public: + CloudStandardJobWithCustomVMNameTestWMS(VirtualizedClusterServiceTest *test, + const std::set> &compute_services, + const std::set> &storage_services, + std::string &hostname) : + wrench::WMS(nullptr, nullptr, compute_services, storage_services, {}, nullptr, hostname, "test") { + this->test = test; + } + +private: + + VirtualizedClusterServiceTest *test; + + int main() { + auto cs = *(this->getAvailableComputeServices().begin()); + + // Create a data movement manager + auto data_movement_manager = this->createDataMovementManager(); + + // Create a job manager + auto job_manager = this->createJobManager(); + + // Create a 2-task job + wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task1, this->test->task2}, {}, + {std::make_tuple(this->test->input_file, + this->test->storage_service, + wrench::ComputeService::SCRATCH)}, + {}, {}); + + // Create and start a VM + auto vm_name = cs->createVM(2, 10, "my_custom_name"); + + if (vm_name != "my_custom_name") { + throw std::runtime_error("Could not create VM with the desired name"); + } + + // Try to create a VM with the same name + try { + auto bogus_vm_name = cs->createVM(2, 10, "my_custom_name"); + throw std::runtime_error("Should not be able to create a VM with an existing name!"); + } catch (wrench::WorkflowExecutionException &e) { + } + + + // Start the VM + auto vm_cs = cs->startVM(vm_name); + + // Submit the 2-task job for execution + try { + job_manager->submitJob(two_task_job, vm_cs); + } catch (wrench::WorkflowExecutionException &e) { + throw std::runtime_error(e.what()); + } + + // Wait for a workflow execution event + std::shared_ptr event; + try { + event = this->getWorkflow()->waitForNextExecutionEvent(); + } catch (wrench::WorkflowExecutionException &e) { + throw std::runtime_error("Error while getting and execution event: " + e.getCause()->toString()); + } + if (not std::dynamic_pointer_cast(event)) { + throw std::runtime_error("Unexpected workflow execution event: " + event->toString()); + } + + return 0; + } +}; + +TEST_F(VirtualizedClusterServiceTest, CloudStandardJobWithCustomVMNameTestWMS) { + DO_TEST_WITH_FORK(do_StandardJobTaskWithCustomVMNameTest_test); +} + +void VirtualizedClusterServiceTest::do_StandardJobTaskWithCustomVMNameTest_test() { + + // Create and initialize a simulation + auto *simulation = new wrench::Simulation(); + int argc = 1; + auto argv = (char **) calloc(1, sizeof(char *)); + argv[0] = strdup("cloud_service_test"); + + ASSERT_NO_THROW(simulation->init(&argc, argv)); + + // Setting up the platform + ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); + + // Get a hostname + std::string hostname = simulation->getHostnameList()[0]; + + // Create a Storage Service + ASSERT_NO_THROW(storage_service = simulation->add( + new wrench::SimpleStorageService(hostname, 100.0))); + + // Create a Cloud Service + std::vector execution_hosts = {simulation->getHostnameList()[1]}; + ASSERT_NO_THROW(compute_service = simulation->add( + new wrench::CloudComputeService(hostname, execution_hosts, 100, + {{wrench::BareMetalComputeServiceProperty::SUPPORTS_PILOT_JOBS, "false"}}))); + + // Create a WMS + std::shared_ptr wms = nullptr;; + ASSERT_NO_THROW(wms = simulation->add( + new CloudStandardJobWithCustomVMNameTestWMS(this, {compute_service}, {storage_service}, hostname))); + + ASSERT_NO_THROW(wms->addWorkflow(workflow)); + + // Create a file registry + ASSERT_NO_THROW(simulation->add(new wrench::FileRegistryService(hostname))); + + // Staging the input_file on the storage service + ASSERT_NO_THROW(simulation->stageFile(input_file, storage_service)); + + // Running a "run a single task" simulation + ASSERT_NO_THROW(simulation->launch()); + + delete simulation; + free(argv[0]); + free(argv); +} + + + /**********************************************************************/ /** VM MIGRATION SIMULATION TEST **/ /**********************************************************************/ diff --git a/test/constructors/simulation_message_constructors/MessageConstructorTest.cpp b/test/constructors/simulation_message_constructors/MessageConstructorTest.cpp index 2f69bf8387..f8eb42e711 100644 --- a/test/constructors/simulation_message_constructors/MessageConstructorTest.cpp +++ b/test/constructors/simulation_message_constructors/MessageConstructorTest.cpp @@ -209,9 +209,9 @@ TEST_F(MessageConstructorTest, CloudComputeServiceMessages) { std::map property_list; std::map messagepayload_list; - ASSERT_NO_THROW(new wrench::CloudComputeServiceCreateVMRequestMessage("mailbox", 42, 10, {}, {}, 666)); + ASSERT_NO_THROW(new wrench::CloudComputeServiceCreateVMRequestMessage("mailbox", 42, 10, "stuff", {}, {}, 666)); ASSERT_THROW( - new wrench::CloudComputeServiceCreateVMRequestMessage("", 42, 0, {}, {}, 666), std::invalid_argument); + new wrench::CloudComputeServiceCreateVMRequestMessage("", 42, 0, "stuff", {}, {}, 666), std::invalid_argument); ASSERT_NO_THROW(new wrench::VirtualizedClusterComputeServiceMigrateVMRequestMessage("mailbox", "host", "host", 666)); ASSERT_THROW(new wrench::VirtualizedClusterComputeServiceMigrateVMRequestMessage("", "host", "host", 666), diff --git a/test/main.cpp b/test/main.cpp index 0d4b4f2cb5..43ed469c55 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -12,10 +12,11 @@ int main(int argc, char **argv) { // disable log -// xbt_log_control_set("root.thresh:critical"); -// xbt_log_control_set("simulation_timestamps.thresh:debug"); + xbt_log_control_set("root.thresh:critical"); + // Example selective log enabling +// xbt_log_control_set("simulation_timestamps.thresh:debug"); // xbt_log_control_set("mailbox.thresh:debug"); // xbt_log_control_set("comprehensive_failure_integration_test.thresh:info"); // xbt_log_control_set("s4u_daemon.thresh:info"); From 66b48de1dfde63b374a105ae7d23bffd5b1217ee Mon Sep 17 00:00:00 2001 From: henricasanova Date: Wed, 5 Jun 2019 17:50:18 +0200 Subject: [PATCH 39/52] Added a new WMS feature: set a timer This allows a WMS to set a timer in the future, and then get a TimerEvent when the timer goes off. This was requested by a user, and seems generally useful. --- include/wrench/wms/WMS.h | 4 + .../execution_events/WorkflowExecutionEvent.h | 52 +- src/wrench/services/helpers/alarm/Alarm.cpp | 14 +- src/wrench/wms/WMS.cpp | 53 +- src/wrench/wms/WMSMessage.cpp | 11 + src/wrench/wms/WMSMessage.h | 11 + src/wrench/workflow/Workflow.cpp | 2 +- src/wrench/workflow/WorkflowTask.cpp | 8 + .../WorkflowExecutionEvent.cpp | 3 + test/wms/WMSTest.cpp | 484 +++++++++--------- 10 files changed, 366 insertions(+), 276 deletions(-) diff --git a/include/wrench/wms/WMS.h b/include/wrench/wms/WMS.h index 607ac2556e..ee8efcfcc2 100644 --- a/include/wrench/wms/WMS.h +++ b/include/wrench/wms/WMS.h @@ -63,6 +63,8 @@ namespace wrench { void checkDeferredStart(); + void setTimer(double date, std::string message); + std::shared_ptr createJobManager(); std::shared_ptr createDataMovementManager(); std::shared_ptr createEnergyMeter(const std::map &measurement_periods); @@ -139,6 +141,8 @@ namespace wrench { virtual void processEventFileCopyFailure(std::shared_ptr); + virtual void processEventTimer(std::shared_ptr); + /***********************/ /** \endcond */ /***********************/ diff --git a/include/wrench/workflow/execution_events/WorkflowExecutionEvent.h b/include/wrench/workflow/execution_events/WorkflowExecutionEvent.h index 16ced94dca..0ea2a12b97 100644 --- a/include/wrench/workflow/execution_events/WorkflowExecutionEvent.h +++ b/include/wrench/workflow/execution_events/WorkflowExecutionEvent.h @@ -44,30 +44,6 @@ namespace wrench { public: -#if 0 - /** @brief Workflow execution event types */ - enum EventType { - /** @brief An error type */ - UNDEFINED, - /** @brief A standard job successfully completed */ - STANDARD_JOB_COMPLETION, - /** @brief A standard job failed */ - STANDARD_JOB_FAILURE, - /** @brief A pilot job started */ - PILOT_JOB_START, - /** @brief A pilot job expired */ - PILOT_JOB_EXPIRATION, - /** @brief A file copy operation completed */ - FILE_COPY_COMPLETION, - /** @brief A file copy operation failed */ - FILE_COPY_FAILURE, - }; -#endif - -// /** @brief The event type */ -// WorkflowExecutionEvent::EventType type; - - /***********************/ /** \cond INTERNAL */ /***********************/ @@ -311,6 +287,34 @@ namespace wrench { }; + /** + * @brief A "timer went off" WorkflowExecutionEvent + */ + class TimerEvent : public WorkflowExecutionEvent { + + private: + + friend class WorkflowExecutionEvent; + /** + * @brief Constructor + * @param content: some arbitrary message + */ + TimerEvent(std::string message) + : message(message) {} + + public: + + /** @brief The message */ + std::string message; + + /** + * @brief Get a textual description of the event + * @return a text string + */ + std::string toString() override { return "TimerEvent (message: " + this->message + ")";} + + }; + }; /***********************/ diff --git a/src/wrench/services/helpers/alarm/Alarm.cpp b/src/wrench/services/helpers/alarm/Alarm.cpp index 2705037239..8b94f00edd 100644 --- a/src/wrench/services/helpers/alarm/Alarm.cpp +++ b/src/wrench/services/helpers/alarm/Alarm.cpp @@ -54,13 +54,13 @@ namespace wrench { if (time_to_sleep > 0) { S4U_Simulation::sleep(time_to_sleep); - WRENCH_INFO("Alarm Service Sending a message to %s", this->reply_mailbox_name.c_str()); - try { - S4U_Mailbox::putMessage(this->reply_mailbox_name, - msg); - } catch (std::shared_ptr &cause) { - WRENCH_WARN("AlarmService was not able to send the trigger to its upper service"); - } + } + + WRENCH_INFO("Alarm Service Sending a message to %s", this->reply_mailbox_name.c_str()); + try { + S4U_Mailbox::putMessage(this->reply_mailbox_name, msg); + } catch (std::shared_ptr &cause) { + WRENCH_WARN("AlarmService was not able to send the trigger to its upper service"); } this->setStateToDown(); diff --git a/src/wrench/wms/WMS.cpp b/src/wrench/wms/WMS.cpp index 7738801e32..5b196807dd 100644 --- a/src/wrench/wms/WMS.cpp +++ b/src/wrench/wms/WMS.cpp @@ -189,6 +189,8 @@ namespace wrench { processEventFileCopyCompletion(real_event); } else if (auto real_event = std::dynamic_pointer_cast(event)) { processEventFileCopyFailure(real_event); + } else if (auto real_event = std::dynamic_pointer_cast(event)) { + processEventTimer(real_event); } else { throw std::runtime_error("SimpleWMS::main(): Unknown workflow execution event: " + event->toString()); } @@ -197,9 +199,9 @@ namespace wrench { } /** - * @brief Process a WorkflowExecutionEvent::STANDARD_JOB_COMPLETION + * @brief Process a standard job completion event * - * @param event: a workflow execution event + * @param event: a StandardJobCompletedEvent */ void WMS::processEventStandardJobCompletion(std::shared_ptr event) { auto standard_job = event->standard_job; @@ -207,50 +209,59 @@ namespace wrench { } /** - * @brief Process a WorkflowExecutionEvent::STANDARD_JOB_FAILURE event + * @brief Process a standard job failure event * - * @param event: a workflow execution event + * @param event: a StandardJobFailedEvent */ void WMS::processEventStandardJobFailure(std::shared_ptr event) { WRENCH_INFO("Notified that a standard job has failed (all its tasks are back in the ready state)"); } /** - * @brief Process a WorkflowExecutionEvent::PILOT_JOB_START event + * @brief Process a pilot job start event * - * @param event: a workflow execution event + * @param event: a PilotJobStartedEvent */ void WMS::processEventPilotJobStart(std::shared_ptr event) { WRENCH_INFO("Notified that a pilot job has started!"); } /** - * @brief Process a WorkflowExecutionEvent::PILOT_JOB_EXPIRATION event + * @brief Process a pilot job expiration event * - * @param event: a workflow execution event + * @param event: a PilotJobExpiredEvent */ void WMS::processEventPilotJobExpiration(std::shared_ptr event) { WRENCH_INFO("Notified that a pilot job has expired!"); } /** - * @brief Process a WorkflowExecutionEvent::FILE_COPY_COMPLETION event + * @brief Process a file copy completion event * - * @param event: a workflow execution event + * @param event: a FileCopyCompletedEvent */ void WMS::processEventFileCopyCompletion(std::shared_ptr event) { WRENCH_INFO("Notified that a file copy is completed!"); } /** - * @brief Process a WorkflowExecutionEvent::FILE_COPY_FAILURE event + * @brief Process a file copy failure event * - * @param event: a workflow execution event + * @param event: a FileCopyFailedEvent */ void WMS::processEventFileCopyFailure(std::shared_ptr event) { WRENCH_INFO("Notified that a file copy has failed!"); } + /** + * @brief Process a timer event + * + * @param event: a TimerEvent + */ + void WMS::processEventTimer(std::shared_ptr event) { + WRENCH_INFO("Notified that a time has gone off!"); + } + /** * @brief Assign a workflow to the WMS * @param workflow: a workflow to execute @@ -362,7 +373,8 @@ namespace wrench { } - /** @brief Get the WMS's pilot scheduler + /** + * @brief Get the WMS's pilot scheduler * * @return the pilot scheduler, or nullptr if none */ @@ -370,7 +382,8 @@ namespace wrench { return (this->pilot_job_scheduler).get(); } - /** @brief Get the WMS's pilot scheduler + /** + * @brief Get the WMS's pilot scheduler * * @return the pilot scheduler, or nullptr if none */ @@ -378,4 +391,16 @@ namespace wrench { return (this->standard_job_scheduler).get(); } + /** + * @brief Sets a timer (which, when it goes off, will generate a TimerEvent) + * @param date: the date at which the timer should go off + * @param message: a string message that will be in the generated TimerEvent + */ + void WMS::setTimer(double date, std::string message) { + + Alarm::createAndStartAlarm(this->simulation, date, this->hostname, this->getWorkflow()->callback_mailbox, + new AlarmWMSTimerMessage(message, 0), "wms_timer"); + + } + }; diff --git a/src/wrench/wms/WMSMessage.cpp b/src/wrench/wms/WMSMessage.cpp index b620bbd625..184300b38e 100644 --- a/src/wrench/wms/WMSMessage.cpp +++ b/src/wrench/wms/WMSMessage.cpp @@ -29,4 +29,15 @@ namespace wrench { AlarmWMSDeferredStartMessage::AlarmWMSDeferredStartMessage(double payload) : WMSMessage("WMS_START_TIME", payload) {} + + /** + * @brief Constructor + * @param payload: message size in bytes + * + * @throw std::invalid_argument + */ + AlarmWMSTimerMessage::AlarmWMSTimerMessage(std::string message, double payload) : WMSMessage("WMS_START_TIME", + payload), message(message) {} + + }; diff --git a/src/wrench/wms/WMSMessage.h b/src/wrench/wms/WMSMessage.h index 99e57cf9d0..ba30ad3490 100644 --- a/src/wrench/wms/WMSMessage.h +++ b/src/wrench/wms/WMSMessage.h @@ -36,6 +36,17 @@ namespace wrench { }; + /** + * @brief Message sent when a timer set by a WMS goes off + */ + class AlarmWMSTimerMessage : public WMSMessage { + public: + explicit AlarmWMSTimerMessage(std::string message, double payload); + std::string message; + + }; + + /***********************/ /** \endcond */ /***********************/ diff --git a/src/wrench/workflow/Workflow.cpp b/src/wrench/workflow/Workflow.cpp index 4fbcaf9375..51c12742ee 100644 --- a/src/wrench/workflow/Workflow.cpp +++ b/src/wrench/workflow/Workflow.cpp @@ -582,7 +582,6 @@ namespace wrench { std::string link = uses.attribute("link").value(); // Check whether the file already exists WorkflowFile *file = nullptr; - try { file = this->getFileByID(id); } catch (std::invalid_argument &e) { @@ -713,6 +712,7 @@ namespace wrench { std::string link = f.at("link"); std::string id = f.at("name"); wrench::WorkflowFile *workflow_file = nullptr; + // Check whether the file already exists try { workflow_file = this->getFileByID(id); } catch (const std::invalid_argument &ia) { diff --git a/src/wrench/workflow/WorkflowTask.cpp b/src/wrench/workflow/WorkflowTask.cpp index 892612c4ce..6b83133cc1 100644 --- a/src/wrench/workflow/WorkflowTask.cpp +++ b/src/wrench/workflow/WorkflowTask.cpp @@ -75,6 +75,14 @@ namespace wrench { WRENCH_DEBUG("Adding file '%s' as output t task %s", file->getID().c_str(), this->getID().c_str()); +// if (file->getOutputOf() != nullptr) { +// std::cerr << "Trying to set file '" + file->getID() + "' as output of task '" + this->getID() + +// "', but this file is already the output of task '" + file->getOutputOf()->getID()+ "'\n"; +// throw std::invalid_argument("Trying to set file '" + file->getID() + "' as output of task '" + this->getID() + +// "', but this file is already the output of task '" + file->getOutputOf()->getID()+ "'"); +// +// } + addFileToMap(output_files, input_files, file); file->setOutputOf(this); diff --git a/src/wrench/workflow/execution_events/WorkflowExecutionEvent.cpp b/src/wrench/workflow/execution_events/WorkflowExecutionEvent.cpp index 630595d7bf..14836ef850 100644 --- a/src/wrench/workflow/execution_events/WorkflowExecutionEvent.cpp +++ b/src/wrench/workflow/execution_events/WorkflowExecutionEvent.cpp @@ -11,6 +11,7 @@ #include #include +#include #include "wrench/simulation/SimulationMessage.h" #include "wrench/services/compute/ComputeServiceMessage.h" @@ -101,6 +102,8 @@ namespace wrench { return std::shared_ptr( new FileCopyFailedEvent(m->file, m->storage_service, m->failure_cause)); } + } else if (auto m = std::dynamic_pointer_cast(message)) { + return std::shared_ptr(new TimerEvent(m->message)); } else { throw std::runtime_error( "WorkflowExecutionEvent::waitForNextExecutionEvent(): Non-handled message type when generating execution event"); diff --git a/test/wms/WMSTest.cpp b/test/wms/WMSTest.cpp index 36f99d5353..a552d6c7ed 100644 --- a/test/wms/WMSTest.cpp +++ b/test/wms/WMSTest.cpp @@ -29,20 +29,20 @@ class WMSTest : public ::testing::Test { protected: WMSTest() { - // Create a platform file - std::string xml = "" - "" - " " - " " - " " - " " - " " - " " - " " - ""; - FILE *platform_file = fopen(platform_file_path.c_str(), "w"); - fprintf(platform_file, "%s", xml.c_str()); - fclose(platform_file); + // Create a platform file + std::string xml = "" + "" + " " + " " + " " + " " + " " + " " + " " + ""; + FILE *platform_file = fopen(platform_file_path.c_str(), "w"); + fprintf(platform_file, "%s", xml.c_str()); + fclose(platform_file); } @@ -57,12 +57,12 @@ class TestDefaultHandlerWMS : public wrench::WMS { public: TestDefaultHandlerWMS(WMSTest *test, - const std::set> &compute_services, - const std::set> &storage_services, - std::string &hostname) : + const std::set> &compute_services, + const std::set> &storage_services, + std::string &hostname) : wrench::WMS(nullptr, nullptr, compute_services, storage_services, {}, nullptr, hostname, "test" ) { - this->test = test; + this->test = test; } private: @@ -71,122 +71,130 @@ class TestDefaultHandlerWMS : public wrench::WMS { int main() { - // Create a data movement manager - auto data_movement_manager = this->createDataMovementManager(); - - // Create a job manager - auto job_manager = this->createJobManager(); - - // Get the file registry service - auto file_registry_service = this->getAvailableFileRegistryService(); - - // Create and start a VM on the cloud service - auto cloud = *(this->getAvailableComputeServices().begin()); - auto vm_name = cloud->createVM(4, 0.0); - auto vm_cs = cloud->startVM(vm_name); - - // Get a "STANDARD JOB COMPLETION" event (default handler) - wrench::WorkflowTask *task1 = this->getWorkflow()->addTask("task1", 10.0, 1, 1, 1.0, 0); - wrench::StandardJob *job1 = job_manager->createStandardJob(task1, {}); - job_manager->submitJob(job1, vm_cs); - this->waitForAndProcessNextEvent(); - - auto batch = this->test->cs_batch; - - // Get a "PILOT JOB STARTED" event (default handler) - wrench::PilotJob *job2 = job_manager->createPilotJob(); - job_manager->submitJob(job2, batch, {{"-N", "1"}, {"-t", "50"}, {"-c", "1"}}); - this->waitForAndProcessNextEvent(); - - // Get a "STANDARD JOB FAILED" and "PILOT JOB EXPIRED" event (default handler) - wrench::WorkflowTask *task2 = this->getWorkflow()->addTask("task2", 100.0, 1, 1, 1.0, 0); - wrench::StandardJob *job3 = job_manager->createStandardJob(task2, {}); - job_manager->submitJob(job3, job2->getComputeService()); - this->waitForAndProcessNextEvent(); - this->waitForAndProcessNextEvent(); - - // Get a "FILE COPY COMPLETION" event (default handler) - data_movement_manager->initiateAsynchronousFileCopy(this->test->small_file, - this->test->storage_service1, this->test->storage_service2, nullptr); - this->waitForAndProcessNextEvent(); - - // Get a "FILE COPY FAILURE" event (default handler) - data_movement_manager->initiateAsynchronousFileCopy(this->test->big_file, - this->test->storage_service1, this->test->storage_service2, nullptr); - this->waitForAndProcessNextEvent(); - - return 0; + // Create a data movement manager + auto data_movement_manager = this->createDataMovementManager(); + + // Create a job manager + auto job_manager = this->createJobManager(); + + // Get the file registry service + auto file_registry_service = this->getAvailableFileRegistryService(); + + // Create and start a VM on the cloud service + auto cloud = *(this->getAvailableComputeServices().begin()); + auto vm_name = cloud->createVM(4, 0.0); + auto vm_cs = cloud->startVM(vm_name); + + // Get a "STANDARD JOB COMPLETION" event (default handler) + wrench::WorkflowTask *task1 = this->getWorkflow()->addTask("task1", 10.0, 1, 1, 1.0, 0); + wrench::StandardJob *job1 = job_manager->createStandardJob(task1, {}); + job_manager->submitJob(job1, vm_cs); + this->waitForAndProcessNextEvent(); + + auto batch = this->test->cs_batch; + + // Get a "PILOT JOB STARTED" event (default handler) + wrench::PilotJob *job2 = job_manager->createPilotJob(); + job_manager->submitJob(job2, batch, {{"-N", "1"}, {"-t", "50"}, {"-c", "1"}}); + this->waitForAndProcessNextEvent(); + + // Get a "STANDARD JOB FAILED" and "PILOT JOB EXPIRED" event (default handler) + wrench::WorkflowTask *task2 = this->getWorkflow()->addTask("task2", 100.0, 1, 1, 1.0, 0); + wrench::StandardJob *job3 = job_manager->createStandardJob(task2, {}); + job_manager->submitJob(job3, job2->getComputeService()); + this->waitForAndProcessNextEvent(); + this->waitForAndProcessNextEvent(); + + // Get a "FILE COPY COMPLETION" event (default handler) + data_movement_manager->initiateAsynchronousFileCopy(this->test->small_file, + this->test->storage_service1, this->test->storage_service2, nullptr); + this->waitForAndProcessNextEvent(); + + // Get a "FILE COPY FAILURE" event (default handler) + data_movement_manager->initiateAsynchronousFileCopy(this->test->big_file, + this->test->storage_service1, this->test->storage_service2, nullptr); + this->waitForAndProcessNextEvent(); + + // Set a timer + double timer_off_date = wrench::Simulation::getCurrentSimulatedDate() + 10; + this->setTimer(timer_off_date, "timer went off"); + this->waitForAndProcessNextEvent(); + if (std::abs( wrench::Simulation::getCurrentSimulatedDate() - timer_off_date) > 0.1) { + throw std::runtime_error("Did not get the timer event at the right date"); + } + + return 0; } }; TEST_F(WMSTest, DefaultEventHandling) { - DO_TEST_WITH_FORK(do_DefaultHandlerWMS_test); + DO_TEST_WITH_FORK(do_DefaultHandlerWMS_test); } void WMSTest::do_DefaultHandlerWMS_test() { - // Create and initialize a simulation - auto simulation = new wrench::Simulation(); - int argc = 1; - auto argv = (char **) calloc(1, sizeof(char *)); - argv[0] = strdup("multiple_wms_test"); + // Create and initialize a simulation + auto simulation = new wrench::Simulation(); + int argc = 1; + auto argv = (char **) calloc(1, sizeof(char *)); + argv[0] = strdup("multiple_wms_test"); - ASSERT_NO_THROW(simulation->init(&argc, argv)); + ASSERT_NO_THROW(simulation->init(&argc, argv)); - // Setting up the platform - ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); + // Setting up the platform + ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); - // Get a hostname - std::string hostname1 = "Host1"; - std::string hostname2 = "Host2"; + // Get a hostname + std::string hostname1 = "Host1"; + std::string hostname2 = "Host2"; - // Create a Storage Service - ASSERT_NO_THROW(storage_service1 = simulation->add( - new wrench::SimpleStorageService(hostname1, 100.0))); + // Create a Storage Service + ASSERT_NO_THROW(storage_service1 = simulation->add( + new wrench::SimpleStorageService(hostname1, 100.0))); - // Create a Storage Service - ASSERT_NO_THROW(storage_service2 = simulation->add( - new wrench::SimpleStorageService(hostname2, 100.0))); + // Create a Storage Service + ASSERT_NO_THROW(storage_service2 = simulation->add( + new wrench::SimpleStorageService(hostname2, 100.0))); - // Create a Cloud Service - std::vector cloud_hosts; - cloud_hosts.push_back(hostname1); - ASSERT_NO_THROW(cs_cloud = simulation->add( - new wrench::CloudComputeService(hostname1, cloud_hosts, 100.0, {}, {}))); + // Create a Cloud Service + std::vector cloud_hosts; + cloud_hosts.push_back(hostname1); + ASSERT_NO_THROW(cs_cloud = simulation->add( + new wrench::CloudComputeService(hostname1, cloud_hosts, 100.0, {}, {}))); - // Create a Batch Service - std::vector batch_hosts; - batch_hosts.push_back(hostname2); - ASSERT_NO_THROW(cs_batch = simulation->add( - new wrench::BatchComputeService(hostname2, batch_hosts, 100.0, {}, {}))); + // Create a Batch Service + std::vector batch_hosts; + batch_hosts.push_back(hostname2); + ASSERT_NO_THROW(cs_batch = simulation->add( + new wrench::BatchComputeService(hostname2, batch_hosts, 100.0, {}, {}))); - // Create a WMS - auto *workflow = new wrench::Workflow(); - std::shared_ptr wms = nullptr;; - ASSERT_NO_THROW(wms = simulation->add( - new TestDefaultHandlerWMS(this, {cs_cloud, cs_batch}, {storage_service1, storage_service2}, hostname1))); + // Create a WMS + auto *workflow = new wrench::Workflow(); + std::shared_ptr wms = nullptr;; + ASSERT_NO_THROW(wms = simulation->add( + new TestDefaultHandlerWMS(this, {cs_cloud, cs_batch}, {storage_service1, storage_service2}, hostname1))); - ASSERT_NO_THROW(wms->addWorkflow(workflow, 100)); + ASSERT_NO_THROW(wms->addWorkflow(workflow, 100)); - // Create a file registry - ASSERT_NO_THROW(simulation->add( - new wrench::FileRegistryService(hostname1))); + // Create a file registry + ASSERT_NO_THROW(simulation->add( + new wrench::FileRegistryService(hostname1))); - // Create two files - this->small_file = workflow->addFile("small", 10); - this->big_file = workflow->addFile("big", 1000); + // Create two files + this->small_file = workflow->addFile("small", 10); + this->big_file = workflow->addFile("big", 1000); - // Staging the input_file on the storage service - ASSERT_NO_THROW(simulation->stageFiles({std::make_pair(this->small_file->getID(), this->small_file)}, - storage_service1)); + // Staging the input_file on the storage service + ASSERT_NO_THROW(simulation->stageFiles({std::make_pair(this->small_file->getID(), this->small_file)}, + storage_service1)); - // Running a "run a single task" simulation - ASSERT_NO_THROW(simulation->launch()); + // Running a "run a single task" simulation + ASSERT_NO_THROW(simulation->launch()); - delete simulation; - free(argv[0]); - free(argv); + delete simulation; + free(argv[0]); + free(argv); } @@ -199,12 +207,12 @@ class TestCustomHandlerWMS : public wrench::WMS { public: TestCustomHandlerWMS(WMSTest *test, - const std::set> &compute_services, - const std::set> &storage_services, - std::string &hostname) : + const std::set> &compute_services, + const std::set> &storage_services, + std::string &hostname) : wrench::WMS(nullptr, nullptr, compute_services, storage_services, {}, nullptr, hostname, "test" ) { - this->test = test; + this->test = test; } private: @@ -215,161 +223,177 @@ class TestCustomHandlerWMS : public wrench::WMS { int main() override { - // Create a data movement manager - auto data_movement_manager = this->createDataMovementManager(); - - // Create a job manager - auto job_manager = this->createJobManager(); - - // Get the file registry service - auto file_registry_service = this->getAvailableFileRegistryService(); - - // Get a "STANDARD JOB COMPLETION" event (default handler) - auto cloud = *(this->getAvailableComputeServices().begin()); - auto vm_name = cloud->createVM(4, 0.0); - auto vm_cs = cloud->startVM(vm_name); - - wrench::WorkflowTask *task1 = this->getWorkflow()->addTask("task1", 10.0, 1, 1, 1.0, 0); - wrench::StandardJob *job1 = job_manager->createStandardJob(task1, {}); - job_manager->submitJob(job1, vm_cs); - this->waitForAndProcessNextEvent(); - if (this->counter != 1) { - throw std::runtime_error("Did not get expected 'STANDARD JOB COMPLETION' event"); - } - - - // Get a "PILOT JOB STARTED" event (default handler) - wrench::PilotJob *job2 = job_manager->createPilotJob(); - job_manager->submitJob(job2, this->test->cs_batch, {{"-N","1"},{"-c","1"},{"-t","2"}}); - this->waitForAndProcessNextEvent(); - if (this->counter != 2) { - throw std::runtime_error("Did not get expected 'PILOT JOB START' event"); - } - - // Get a "STANDARD JOB FAILED" and "PILOT JOB EXPIRED" event (default handler) - wrench::WorkflowTask *task2 = this->getWorkflow()->addTask("task2", 200.0, 1, 1, 1.0, 0); - wrench::StandardJob *job3 = job_manager->createStandardJob(task2, {}); - job_manager->submitJob(job3, job2->getComputeService()); - this->waitForAndProcessNextEvent(); - if (this->counter != 3) { - throw std::runtime_error("Did not get expected 'STANDARD JOB FAILED' event"); - } - this->waitForAndProcessNextEvent(); - if (this->counter != 4) { - throw std::runtime_error("Did not get expected 'PILOT JOB EXPIRED' event"); - } - - // Get a "FILE COPY COMPLETION" event (default handler) - data_movement_manager->initiateAsynchronousFileCopy(this->test->small_file, - this->test->storage_service1, this->test->storage_service2, nullptr); - this->waitForAndProcessNextEvent(); - if (this->counter != 5) { - throw std::runtime_error("Did not get expected 'FILE COPY COMPLETED' event"); - } - - // Get a "FILE COPY FAILURE" event (default handler) - data_movement_manager->initiateAsynchronousFileCopy(this->test->big_file, - this->test->storage_service1, this->test->storage_service2, nullptr); - this->waitForAndProcessNextEvent(); - if (this->counter != 6) { - throw std::runtime_error("Did not get expected 'FILE COPY FAILURE' event"); - } - - return 0; + // Create a data movement manager + auto data_movement_manager = this->createDataMovementManager(); + + // Create a job manager + auto job_manager = this->createJobManager(); + + // Get the file registry service + auto file_registry_service = this->getAvailableFileRegistryService(); + + // Get a "STANDARD JOB COMPLETION" event (default handler) + auto cloud = *(this->getAvailableComputeServices().begin()); + auto vm_name = cloud->createVM(4, 0.0); + auto vm_cs = cloud->startVM(vm_name); + + wrench::WorkflowTask *task1 = this->getWorkflow()->addTask("task1", 10.0, 1, 1, 1.0, 0); + wrench::StandardJob *job1 = job_manager->createStandardJob(task1, {}); + job_manager->submitJob(job1, vm_cs); + this->waitForAndProcessNextEvent(); + if (this->counter != 1) { + throw std::runtime_error("Did not get expected StandardJobCompletionEvent"); + } + + + // Get a "PILOT JOB STARTED" event (default handler) + wrench::PilotJob *job2 = job_manager->createPilotJob(); + job_manager->submitJob(job2, this->test->cs_batch, {{"-N","1"},{"-c","1"},{"-t","2"}}); + this->waitForAndProcessNextEvent(); + if (this->counter != 2) { + throw std::runtime_error("Did not get expected PilotJobStartEvent"); + } + + // Get a "STANDARD JOB FAILED" and "PILOT JOB EXPIRED" event (default handler) + wrench::WorkflowTask *task2 = this->getWorkflow()->addTask("task2", 200.0, 1, 1, 1.0, 0); + wrench::StandardJob *job3 = job_manager->createStandardJob(task2, {}); + job_manager->submitJob(job3, job2->getComputeService()); + this->waitForAndProcessNextEvent(); + if (this->counter != 3) { + throw std::runtime_error("Did not get expected StandardJobFailedEvent"); + } + this->waitForAndProcessNextEvent(); + if (this->counter != 4) { + throw std::runtime_error("Did not get expected PilotJobExpiredEvent"); + } + + // Get a "FILE COPY COMPLETION" event (default handler) + data_movement_manager->initiateAsynchronousFileCopy(this->test->small_file, + this->test->storage_service1, this->test->storage_service2, nullptr); + this->waitForAndProcessNextEvent(); + if (this->counter != 5) { + throw std::runtime_error("Did not get expected FileCoompletedEvent"); + } + + // Get a "FILE COPY FAILURE" event (default handler) + data_movement_manager->initiateAsynchronousFileCopy(this->test->big_file, + this->test->storage_service1, this->test->storage_service2, nullptr); + this->waitForAndProcessNextEvent(); + if (this->counter != 6) { + throw std::runtime_error("Did not get expected FileCopyFailureEvent"); + } + + // Set a timer + double timer_off_date = wrench::Simulation::getCurrentSimulatedDate() + 10; + this->setTimer(timer_off_date, "timer went off"); + this->waitForAndProcessNextEvent(); + if (std::abs( wrench::Simulation::getCurrentSimulatedDate() - timer_off_date) > 0.1) { + throw std::runtime_error("Did not get the timer event at the right date"); + } + if (this->counter != 7) { + std::cerr << "this->counter = " << this->counter << "\n"; + throw std::runtime_error("Did not get expected TimerEvent"); + } + + return 0; } void processEventStandardJobCompletion(std::shared_ptr event) override { - this->counter = 1; + this->counter = 1; } void processEventPilotJobStart(std::shared_ptr event) override { - this->counter = 2; + this->counter = 2; } void processEventStandardJobFailure(std::shared_ptr event) override { - this->counter = 3; + this->counter = 3; } void processEventPilotJobExpiration(std::shared_ptr event) override { - this->counter = 4; + this->counter = 4; } void processEventFileCopyCompletion(std::shared_ptr event) override { - this->counter = 5; + this->counter = 5; } void processEventFileCopyFailure(std::shared_ptr event) override { - this->counter = 6; + this->counter = 6; + } + + void processEventTimer(std::shared_ptr event) override { + this->counter = 7; } }; TEST_F(WMSTest, CustomEventHandling) { - DO_TEST_WITH_FORK(do_CustomHandlerWMS_test); + DO_TEST_WITH_FORK(do_CustomHandlerWMS_test); } void WMSTest::do_CustomHandlerWMS_test() { - // Create and initialize a simulation - auto simulation = new wrench::Simulation(); - int argc = 1; - auto argv = (char **) calloc(1, sizeof(char *)); - argv[0] = strdup("multiple_wms_test"); + // Create and initialize a simulation + auto simulation = new wrench::Simulation(); + int argc = 1; + auto argv = (char **) calloc(1, sizeof(char *)); + argv[0] = strdup("multiple_wms_test"); - ASSERT_NO_THROW(simulation->init(&argc, argv)); + ASSERT_NO_THROW(simulation->init(&argc, argv)); - // Setting up the platform - ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); + // Setting up the platform + ASSERT_NO_THROW(simulation->instantiatePlatform(platform_file_path)); - // Get a hostname - std::string hostname1 = "Host1"; - std::string hostname2 = "Host2"; + // Get a hostname + std::string hostname1 = "Host1"; + std::string hostname2 = "Host2"; - // Create a Storage Service - ASSERT_NO_THROW(storage_service1 = simulation->add( - new wrench::SimpleStorageService(hostname1, 100.0))); + // Create a Storage Service + ASSERT_NO_THROW(storage_service1 = simulation->add( + new wrench::SimpleStorageService(hostname1, 100.0))); - // Create a Storage Service - ASSERT_NO_THROW(storage_service2 = simulation->add( - new wrench::SimpleStorageService(hostname2, 100.0))); + // Create a Storage Service + ASSERT_NO_THROW(storage_service2 = simulation->add( + new wrench::SimpleStorageService(hostname2, 100.0))); - // Create a Cloud Service - std::vector cloud_hosts; - cloud_hosts.push_back(hostname1); - ASSERT_NO_THROW(cs_cloud = simulation->add( - new wrench::CloudComputeService(hostname1, cloud_hosts, 100.0, {}, {}))); + // Create a Cloud Service + std::vector cloud_hosts; + cloud_hosts.push_back(hostname1); + ASSERT_NO_THROW(cs_cloud = simulation->add( + new wrench::CloudComputeService(hostname1, cloud_hosts, 100.0, {}, {}))); - // Create a Batch Service - std::vector batch_hosts; - batch_hosts.push_back(hostname1); - ASSERT_NO_THROW(cs_batch = simulation->add( - new wrench::BatchComputeService(hostname2, batch_hosts, 100.0, {}, {}))); + // Create a Batch Service + std::vector batch_hosts; + batch_hosts.push_back(hostname1); + ASSERT_NO_THROW(cs_batch = simulation->add( + new wrench::BatchComputeService(hostname2, batch_hosts, 100.0, {}, {}))); - // Create a WMS - auto *workflow = new wrench::Workflow(); - std::shared_ptr wms = nullptr;; - ASSERT_NO_THROW(wms = simulation->add( - new TestCustomHandlerWMS(this, {cs_cloud, cs_batch}, {storage_service1, storage_service2}, hostname1))); + // Create a WMS + auto *workflow = new wrench::Workflow(); + std::shared_ptr wms = nullptr;; + ASSERT_NO_THROW(wms = simulation->add( + new TestCustomHandlerWMS(this, {cs_cloud, cs_batch}, {storage_service1, storage_service2}, hostname1))); - ASSERT_NO_THROW(wms->addWorkflow(workflow, 100)); + ASSERT_NO_THROW(wms->addWorkflow(workflow, 100)); - // Create a file registry - ASSERT_NO_THROW(simulation->add( - new wrench::FileRegistryService(hostname1))); + // Create a file registry + ASSERT_NO_THROW(simulation->add( + new wrench::FileRegistryService(hostname1))); - // Create two files - this->small_file = workflow->addFile("small", 10); - this->big_file = workflow->addFile("big", 1000); + // Create two files + this->small_file = workflow->addFile("small", 10); + this->big_file = workflow->addFile("big", 1000); - // Staging the input_file on the storage service - ASSERT_NO_THROW(simulation->stageFiles({std::make_pair(this->small_file->getID(), this->small_file)}, - storage_service1)); + // Staging the input_file on the storage service + ASSERT_NO_THROW(simulation->stageFiles({std::make_pair(this->small_file->getID(), this->small_file)}, + storage_service1)); - // Running a "run a single task" simulation - ASSERT_NO_THROW(simulation->launch()); + // Running a "run a single task" simulation + ASSERT_NO_THROW(simulation->launch()); - delete simulation; - free(argv[0]); - free(argv); + delete simulation; + free(argv[0]); + free(argv); } From 1276a6c2ef3258bec4d33b0fcf01697fc5b5a52b Mon Sep 17 00:00:00 2001 From: henricasanova Date: Thu, 6 Jun 2019 10:36:39 +0200 Subject: [PATCH 40/52] (#112) Made WorkflowFile::getInputOf() public --- include/wrench/workflow/WorkflowFile.h | 2 +- src/wrench/wms/WMS.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/wrench/workflow/WorkflowFile.h b/include/wrench/workflow/WorkflowFile.h index 3945ce60e6..093d882e89 100644 --- a/include/wrench/workflow/WorkflowFile.h +++ b/include/wrench/workflow/WorkflowFile.h @@ -32,6 +32,7 @@ namespace wrench { std::string getID(); WorkflowTask *getOutputOf(); + std::map getInputOf(); bool isOutput(); @@ -50,7 +51,6 @@ namespace wrench { void setInputOf(WorkflowTask *task); - std::map getInputOf(); Workflow *workflow; // Containing workflow WorkflowFile(const std::string, double); diff --git a/src/wrench/wms/WMS.cpp b/src/wrench/wms/WMS.cpp index 5b196807dd..623dc5305a 100644 --- a/src/wrench/wms/WMS.cpp +++ b/src/wrench/wms/WMS.cpp @@ -397,10 +397,8 @@ namespace wrench { * @param message: a string message that will be in the generated TimerEvent */ void WMS::setTimer(double date, std::string message) { - Alarm::createAndStartAlarm(this->simulation, date, this->hostname, this->getWorkflow()->callback_mailbox, new AlarmWMSTimerMessage(message, 0), "wms_timer"); - } }; From 708cafb9f986d22a1228ffec7b301e23359c9b62 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Thu, 6 Jun 2019 10:56:52 +0200 Subject: [PATCH 41/52] (#112) Added WorkflowTask::getChildren() and WorkflowTask::getParents() methods --- include/wrench/workflow/WorkflowTask.h | 6 ++++-- src/wrench/workflow/WorkflowTask.cpp | 26 ++++++++++++++++++++++---- test/workflow/WorkflowTaskTest.cpp | 5 ++++- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/include/wrench/workflow/WorkflowTask.h b/include/wrench/workflow/WorkflowTask.h index 3f5ecddc87..cb7b741daf 100644 --- a/include/wrench/workflow/WorkflowTask.h +++ b/include/wrench/workflow/WorkflowTask.h @@ -38,9 +38,11 @@ namespace wrench { double getMemoryRequirement() const; - int getNumberOfChildren() const; + unsigned long getNumberOfChildren() const; + std::vector getChildren() const; - int getNumberOfParents() const; + unsigned long getNumberOfParents() const; + std::vector getParents() const; void addInputFile(WorkflowFile *file); diff --git a/src/wrench/workflow/WorkflowTask.cpp b/src/wrench/workflow/WorkflowTask.cpp index 6b83133cc1..ee7220a0fa 100644 --- a/src/wrench/workflow/WorkflowTask.cpp +++ b/src/wrench/workflow/WorkflowTask.cpp @@ -152,27 +152,45 @@ namespace wrench { * * @return a number of children */ - int WorkflowTask::getNumberOfChildren() const { - int count = 0; + unsigned long WorkflowTask::getNumberOfChildren() const { + unsigned long count = 0; for (lemon::ListDigraph::OutArcIt a(*DAG, DAG_node); a != lemon::INVALID; ++a) { ++count; } return count; } + /** + * @brief Get the children of a task + * + * @return a list of workflow tasks + */ + std::vector WorkflowTask::getChildren() const { + return this->getWorkflow()->getTaskChildren(this); + } + /** * @brief Get the number of parents of a task * * @return a number of parents */ - int WorkflowTask::getNumberOfParents() const { - int count = 0; + unsigned long WorkflowTask::getNumberOfParents() const { + unsigned long count = 0; for (lemon::ListDigraph::InArcIt a(*DAG, DAG_node); a != lemon::INVALID; ++a) { ++count; } return count; } + /** + * @brief Get the parents of a task + * + * @return a list of workflow tasks + */ + std::vector WorkflowTask::getParents() const { + return this->getWorkflow()->getTaskParents(this); + } + /** * @brief Get the state of the task * diff --git a/test/workflow/WorkflowTaskTest.cpp b/test/workflow/WorkflowTaskTest.cpp index b272372a9b..7246e822be 100644 --- a/test/workflow/WorkflowTaskTest.cpp +++ b/test/workflow/WorkflowTaskTest.cpp @@ -99,10 +99,13 @@ TEST_F(WorkflowTaskTest, TaskStructure) { ASSERT_EQ(t2->getJob(), nullptr); ASSERT_EQ(t1->getNumberOfParents(), 0); + ASSERT_EQ(t1->getParents().size(), 0); ASSERT_EQ(t2->getNumberOfParents(), 1); - + ASSERT_EQ(t2->getParents().size(), 1); ASSERT_EQ(t1->getNumberOfChildren(), 1); + ASSERT_EQ(t1->getChildren().size(), 1); ASSERT_EQ(t2->getNumberOfChildren(), 0); + ASSERT_EQ(t2->getChildren().size(), 0); ASSERT_EQ(t1->getClusterID(), ""); } From da29b97be7da85075c5121e6aad639ed408119ed Mon Sep 17 00:00:00 2001 From: henricasanova Date: Thu, 6 Jun 2019 11:35:49 +0200 Subject: [PATCH 42/52] (#112) Added Workflow::getEntryTasks() and Workflow::getExitTasks() methods --- include/wrench/workflow/Workflow.h | 3 +++ src/wrench/workflow/Workflow.cpp | 41 +++++++++++++++++++++++++++--- test/workflow/WorkflowTest.cpp | 17 +++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/include/wrench/workflow/Workflow.h b/include/wrench/workflow/Workflow.h index 4a798ce6c5..f242d56afa 100644 --- a/include/wrench/workflow/Workflow.h +++ b/include/wrench/workflow/Workflow.h @@ -73,6 +73,9 @@ namespace wrench { std::map getInputFiles(); + std::map getEntryTasks() const; + std::map getExitTasks() const; + bool isDone(); /***********************/ diff --git a/src/wrench/workflow/Workflow.cpp b/src/wrench/workflow/Workflow.cpp index 51c12742ee..0310cf758b 100644 --- a/src/wrench/workflow/Workflow.cpp +++ b/src/wrench/workflow/Workflow.cpp @@ -65,7 +65,7 @@ namespace wrench { task->workflow = this; task->DAG = this->DAG.get(); task->DAG_node = DAG->addNode(); - task->toplevel = 0; // upon creation, a task is an entry task + task->toplevel = 0; // upon creation, a task is an exit task // Add it to the DAG node's metadata (*DAG_node_map)[task->DAG_node] = task; @@ -496,6 +496,7 @@ namespace wrench { return input_files; } + /** * @brief Get the total number of flops for a list of tasks * @@ -653,7 +654,7 @@ namespace wrench { try { workflowJobs = j.at("workflow"); } catch (std::out_of_range &e) { - throw std::invalid_argument("Workflow::loadFromJson(): Could not find a workflow entry"); + throw std::invalid_argument("Workflow::loadFromJson(): Could not find a workflow exit"); } wrench::WorkflowTask *task; @@ -805,6 +806,40 @@ namespace wrench { return to_return; } + /** + * @brief Get the exit tasks of the workflow, i.e., those tasks + * that don't have parents + * @return A map of tasks indexed by their IDs + */ + std::map Workflow::getEntryTasks() const { + // TODO: This could be done more efficiently at the Lemon level + std::map entry_tasks; + for (auto const &t : this->tasks) { + auto task = t.second.get(); + if (task->getNumberOfParents() == 0) { + entry_tasks[task->getID()] = task; + } + } + return entry_tasks; + } + + /** + * @brief Get the exit tasks of the workflow, i.e., those tasks + * that don't have children + * @return A map of tasks indexed by their IDs + */ + std::map Workflow::getExitTasks() const { + // TODO: This could be done more efficiently at the lemon level + std::map exit_tasks; + for (auto const &t : this->tasks) { + auto task = t.second.get(); + if (task->getNumberOfChildren() == 0) { + exit_tasks[task->getID()] = task; + } + } + return exit_tasks; + } + /** * @brief Returns the number of levels in the workflow * @return the number of levels @@ -812,7 +847,7 @@ namespace wrench { unsigned long Workflow::getNumLevels() { unsigned long max_top_level = 0; for (auto const &t : this->tasks) { - wrench::WorkflowTask *task = t.second.get(); + auto task = t.second.get(); if (task->getNumberOfChildren() == 0) { if (1 + task->getTopLevel() > max_top_level) { max_top_level = 1 + task->getTopLevel(); diff --git a/test/workflow/WorkflowTest.cpp b/test/workflow/WorkflowTest.cpp index d20bd076bb..6ea6e72195 100644 --- a/test/workflow/WorkflowTest.cpp +++ b/test/workflow/WorkflowTest.cpp @@ -89,6 +89,23 @@ TEST_F(WorkflowTest, WorkflowStructure) { ASSERT_NE(std::find(top_level_equal_to_1_or_2.begin(), top_level_equal_to_1_or_2.end(), t4), top_level_equal_to_1_or_2.end()); + + // Get Entry tasks and check they all are in the top level, as expected + auto entry_tasks = workflow->getEntryTasks(); + auto top_level = workflow->getTasksInTopLevelRange(0, 0); + for (auto const &t : top_level) { + ASSERT_TRUE(entry_tasks.find(t->getID()) != entry_tasks.end()); + } + // Being paranoid, check that they don't have parents + for (auto const &t : entry_tasks) { + ASSERT_EQ(t.second->getNumberOfParents(), 0); + } + + // Get Exit tasks + auto exit_tasks = workflow->getExitTasks(); + ASSERT_EQ(exit_tasks.size(), 1); + ASSERT_TRUE(exit_tasks.begin()->second == t4); + // remove tasks workflow->removeTask(t4); ASSERT_EQ(0, workflow->getTaskChildren(t3).size()); From dfd7f9bb4987bf96949d1891ef450d9e00d1ef43 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Thu, 6 Jun 2019 15:11:56 +0200 Subject: [PATCH 43/52] Implemented a speed-state-change detector --- .../helpers/HostStateChangeDetector.h | 5 ++ .../helpers/HostStateChangeDetectorMessage.h | 12 +++++ .../bare_metal/BareMetalComputeService.cpp | 7 ++- .../compute/cloud/CloudComputeService.cpp | 5 ++ .../StandardJobExecutor.cpp | 2 +- .../HostStateChangeDetector.cpp | 47 ++++++++++++++++++- .../HostStateChangeDetectorMessage.cpp | 14 +++++- 7 files changed, 88 insertions(+), 4 deletions(-) diff --git a/include/wrench/services/helpers/HostStateChangeDetector.h b/include/wrench/services/helpers/HostStateChangeDetector.h index b41ae68782..c912ab2b18 100644 --- a/include/wrench/services/helpers/HostStateChangeDetector.h +++ b/include/wrench/services/helpers/HostStateChangeDetector.h @@ -36,6 +36,7 @@ namespace wrench { std::vector hosts_to_monitor, bool notify_when_turned_on, bool notify_when_turned_off, + bool notify_when_speed_change, std::shared_ptr creator, std::string mailbox_to_notify, std::map property_list = {} @@ -48,14 +49,17 @@ namespace wrench { void cleanup(bool has_terminated_cleanly, int return_value) override; void hostStateChangeCallback(std::string const &hostname); + void hostSpeedChangeCallback(std::string const &hostname); std::vector hosts_to_monitor; bool notify_when_turned_on; bool notify_when_turned_off; + bool notify_when_speed_change; std::string mailbox_to_notify; int main() override; std::vector> hosts_that_have_recently_changed_state; + std::vector> hosts_that_have_recently_changed_speed; std::shared_ptr creator; @@ -63,6 +67,7 @@ namespace wrench { std::vector hosts_that_have_recently_turned_off; unsigned int on_state_change_call_back_id; + unsigned int on_speed_change_call_back_id; }; diff --git a/include/wrench/services/helpers/HostStateChangeDetectorMessage.h b/include/wrench/services/helpers/HostStateChangeDetectorMessage.h index 3c7a4b1ab0..eb81bd73e7 100644 --- a/include/wrench/services/helpers/HostStateChangeDetectorMessage.h +++ b/include/wrench/services/helpers/HostStateChangeDetectorMessage.h @@ -50,6 +50,18 @@ namespace wrench { std::string hostname; }; + /** + * @brief A message sent by the HostStateChangeDetector to notify some listener that a host has changed speed + */ + class HostHasChangedSpeedMessage : public HostStateChangeDetectorMessage { + public: + explicit HostHasChangedSpeedMessage(std::string hostname, double speed); + /** @brief The name of the host that has tuned off */ + std::string hostname; + /** @brief The host's (new) speed */ + double speed; + }; + /***********************/ /** \endcond */ /***********************/ diff --git a/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp b/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp index 20af92b395..f63d4b4c56 100644 --- a/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp +++ b/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp @@ -532,7 +532,7 @@ namespace wrench { hosts_to_monitor.push_back(h.first); } this->host_state_change_monitor = std::shared_ptr( - new HostStateChangeDetector(this->hostname, hosts_to_monitor, true, true, + new HostStateChangeDetector(this->hostname, hosts_to_monitor, true, true, false, this->getSharedPtr(), this->mailbox_name, {{HostStateChangeDetectorProperty::MONITORING_PERIOD, "1.0"}})); this->host_state_change_monitor->simulation = this->simulation; @@ -592,6 +592,11 @@ namespace wrench { continue; } + // If the host has compute speed zero, then don't look at it + if (Simulation::getHostFlopRate(r.first) <= 0.0) { + continue; + } + if ((required_num_cores == 0) and (std::get<0>(r.second) < task->getMinNumCores())) { continue; } diff --git a/src/wrench/services/compute/cloud/CloudComputeService.cpp b/src/wrench/services/compute/cloud/CloudComputeService.cpp index 4d3a2ec371..650a17bff5 100644 --- a/src/wrench/services/compute/cloud/CloudComputeService.cpp +++ b/src/wrench/services/compute/cloud/CloudComputeService.cpp @@ -894,6 +894,11 @@ namespace wrench { continue; } + // Check that host has a non-zero compute speed + if (Simulation::getHostFlopRate(host) <= 0) { + continue; + } + // Check for RAM auto total_ram = Simulation::getHostMemoryCapacity(host); auto available_ram = total_ram - this->used_ram_per_execution_host[host]; diff --git a/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp b/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp index 30cba597d4..b466b488e3 100644 --- a/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp +++ b/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp @@ -290,7 +290,7 @@ namespace wrench { hosts_to_monitor.push_back(h.first); } this->host_state_monitor = std::shared_ptr( - new HostStateChangeDetector(this->hostname, hosts_to_monitor, true, false, + new HostStateChangeDetector(this->hostname, hosts_to_monitor, true, false, false, this->getSharedPtr(), this->mailbox_name)); this->host_state_monitor->simulation = this->simulation; this->host_state_monitor->start(this->host_state_monitor, true, false); // Daemonized, no auto-restart diff --git a/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp b/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp index 4741c8c439..4080e7b8d3 100644 --- a/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp +++ b/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp @@ -22,6 +22,7 @@ XBT_LOG_NEW_DEFAULT_CATEGORY(host_state_change_detector, "Log category for HostS void wrench::HostStateChangeDetector::cleanup(bool has_returned_from_main, int return_value) { // Unregister the callback! simgrid::s4u::Host::on_state_change.disconnect(this->on_state_change_call_back_id); + simgrid::s4u::Host::on_speed_change.disconnect(this->on_speed_change_call_back_id); } @@ -31,6 +32,7 @@ void wrench::HostStateChangeDetector::cleanup(bool has_returned_from_main, int r * @param hosts_to_monitor: the list of hosts to monitor * @param notify_when_turned_on: whether to send a notifications when hosts turn on * @param notify_when_turned_off: whether to send a notifications when hosts turn off + * @param notify_when_speed_change: whether to send a notification when hosts change speed * @param creator: the service that created this service (when its creator dies, so does this service) * @param mailbox_to_notify: the mailbox to notify * @param property_list: a property list @@ -38,7 +40,7 @@ void wrench::HostStateChangeDetector::cleanup(bool has_returned_from_main, int r */ wrench::HostStateChangeDetector::HostStateChangeDetector(std::string host_on_which_to_run, std::vector hosts_to_monitor, - bool notify_when_turned_on, bool notify_when_turned_off, + bool notify_when_turned_on, bool notify_when_turned_off, bool notify_when_speed_change, std::shared_ptr creator, std::string mailbox_to_notify, std::map property_list) : @@ -46,6 +48,7 @@ wrench::HostStateChangeDetector::HostStateChangeDetector(std::string host_on_whi this->hosts_to_monitor = hosts_to_monitor; this->notify_when_turned_on = notify_when_turned_on; this->notify_when_turned_off = notify_when_turned_off; + this->notify_when_speed_change = notify_when_speed_change; this->mailbox_to_notify = mailbox_to_notify; this->creator = creator; @@ -57,6 +60,13 @@ wrench::HostStateChangeDetector::HostStateChangeDetector(std::string host_on_whi [this](simgrid::s4u::Host const &h) { this->hostStateChangeCallback(h.get_name()); }); + + // Connect my member method to the on_speed_change signal from SimGrid regarding Hosts + this->on_speed_change_call_back_id = simgrid::s4u::Host::on_speed_change.connect( + [this](simgrid::s4u::Host const &h) { + this->hostSpeedChangeCallback(h.get_name()); + }); + } void wrench::HostStateChangeDetector::hostStateChangeCallback(std::string const &hostname) { @@ -67,6 +77,15 @@ void wrench::HostStateChangeDetector::hostStateChangeCallback(std::string const } } +void wrench::HostStateChangeDetector::hostSpeedChangeCallback(std::string const &hostname) { + if (std::find(this->hosts_to_monitor.begin(), this->hosts_to_monitor.end(), hostname) != + this->hosts_to_monitor.end()) { + auto host = simgrid::s4u::Host::by_name(hostname); + double speed = host->get_speed(); + this->hosts_that_have_recently_changed_speed.push_back(std::make_pair(hostname, speed)); + } +} + int wrench::HostStateChangeDetector::main() { @@ -79,6 +98,7 @@ int wrench::HostStateChangeDetector::main() { // Sleeping for my monitoring period Simulation::sleep(this->getPropertyValueAsDouble(HostStateChangeDetectorProperty::MONITORING_PERIOD)); + // State Changes while (not this->hosts_that_have_recently_changed_state.empty()) { auto host_info = this->hosts_that_have_recently_changed_state.at(0); std::string hostname = std::get<0>(host_info); @@ -104,8 +124,33 @@ int wrench::HostStateChangeDetector::main() { WRENCH_INFO("Network error '%s' while notifying mailbox of a host state change ... ignoring", e->toString().c_str()); } + } + + // Speed Changes + while (not this->hosts_that_have_recently_changed_speed.empty()) { + auto host_info = this->hosts_that_have_recently_changed_speed.at(0); + std::string hostname = std::get<0>(host_info); + double new_speed = std::get<1>(host_info); + this->hosts_that_have_recently_changed_speed.erase(this->hosts_that_have_recently_changed_speed.begin()); + + HostStateChangeDetectorMessage *msg; + + if (this->notify_when_speed_change) { + msg = new HostHasChangedSpeedMessage(hostname, new_speed); + continue; + } + WRENCH_INFO("Notifying mailbox '%s' that host '%s' has changed speed", this->mailbox_to_notify.c_str(), + hostname.c_str()); + try { + S4U_Mailbox::dputMessage(this->mailbox_to_notify, msg); + } catch (std::shared_ptr &e) { + WRENCH_INFO("Network error '%s' while notifying mailbox of a speed state change ... ignoring", + e->toString().c_str()); + } } + + } return 0; } diff --git a/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetectorMessage.cpp b/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetectorMessage.cpp index d203a965e9..94fd56f3c6 100644 --- a/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetectorMessage.cpp +++ b/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetectorMessage.cpp @@ -34,11 +34,23 @@ namespace wrench { /** * @brief Constructor * - * @param hostname: the name of the host that has turned of + * @param hostname: the name of the host that has turned off */ HostHasTurnedOffMessage::HostHasTurnedOffMessage(std::string hostname) : HostStateChangeDetectorMessage("HostHasTurnedOffMessage") { this->hostname = hostname; } + /** + * @brief Constructor + * + * @param hostname: the name of the host that has changed speed + * @param speed: the host's new speed + */ + HostHasChangedSpeedMessage::HostHasChangedSpeedMessage(std::string hostname, double speed) : + HostStateChangeDetectorMessage("HostHasChangedSpeedMessage") { + this->hostname = hostname; + this->speed = speed; + } + } From b6a5058d088f321f52d3e8c6cf357f32c99bb913 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Thu, 6 Jun 2019 15:27:57 +0200 Subject: [PATCH 44/52] bug-- --- .../host_state_change_detector/HostStateChangeDetector.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp b/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp index 4080e7b8d3..47203b6420 100644 --- a/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp +++ b/src/wrench/services/helpers/host_state_change_detector/HostStateChangeDetector.cpp @@ -137,6 +137,7 @@ int wrench::HostStateChangeDetector::main() { if (this->notify_when_speed_change) { msg = new HostHasChangedSpeedMessage(hostname, new_speed); + } else { continue; } From 35e474b4e4e708a19ea2662deb1f9aa26f0a6596 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Thu, 6 Jun 2019 15:46:18 +0200 Subject: [PATCH 45/52] Made Compute Service aware to host speed changes whenever applicable, and unwilling to dispatch work to hosts with speed 0 --- CMakeLists.txt | 3 +++ .../compute/bare_metal/BareMetalComputeService.cpp | 5 ++++- .../compute/standard_job_executor/StandardJobExecutor.cpp | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e4a1f4087a..8742e3b1da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,9 @@ add_definitions("-Wall -Wno-unused-variable -Wno-unused-private-field") if (ENABLE_BATSCHED) add_definitions(-DENABLE_BATSCHED) endif () +if (ENABLE_MESSAGE_MANAGER) + add_definitions(-DMESSAGE_MANAGER) +endif () set(CMAKE_CXX_STANDARD 11) diff --git a/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp b/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp index f63d4b4c56..03e9663d3e 100644 --- a/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp +++ b/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp @@ -532,7 +532,7 @@ namespace wrench { hosts_to_monitor.push_back(h.first); } this->host_state_change_monitor = std::shared_ptr( - new HostStateChangeDetector(this->hostname, hosts_to_monitor, true, true, false, + new HostStateChangeDetector(this->hostname, hosts_to_monitor, true, true, true, this->getSharedPtr(), this->mailbox_name, {{HostStateChangeDetectorProperty::MONITORING_PERIOD, "1.0"}})); this->host_state_change_monitor->simulation = this->simulation; @@ -794,6 +794,9 @@ namespace wrench { if (auto msg = std::dynamic_pointer_cast(message)) { // Do nothing, just wake up return true; + } else if (auto msg = std::dynamic_pointer_cast(message)) { + // Do nothing, just wake up + return true; } else if (auto msg = std::dynamic_pointer_cast(message)) { // If all hosts being off should not cause the service to terminate, then nevermind if (this->getPropertyValueAsString( diff --git a/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp b/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp index b466b488e3..8ba30079a9 100644 --- a/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp +++ b/src/wrench/services/compute/standard_job_executor/StandardJobExecutor.cpp @@ -474,6 +474,11 @@ namespace wrench { continue; } + // Does the host have non-zero compute speed? + if (Simulation::getHostFlopRate(hostname) <= 0.0) { + continue; + } + // Does the host have enough cores? unsigned long num_available_cores = this->core_availabilities[hostname]; if (num_available_cores < minimum_num_cores) { @@ -623,6 +628,9 @@ namespace wrench { if (auto msg = std::dynamic_pointer_cast(message)) { // Do nothing, just wake up return true; + } else if (auto msg = std::dynamic_pointer_cast(message)) { + // Do nothing, just wake up + return true; } else if (auto msg = std::dynamic_pointer_cast(message)) { processWorkunitExecutorCompletion(msg->workunit_executor, msg->workunit); return true; From 21eafb87f961c28bb3f2de9a43d0f13a676be01b Mon Sep 17 00:00:00 2001 From: henricasanova Date: Fri, 7 Jun 2019 12:39:21 +0200 Subject: [PATCH 46/52] warnings-- --- test/energy_consumption/EnergyConsumptionTest.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/energy_consumption/EnergyConsumptionTest.cpp b/test/energy_consumption/EnergyConsumptionTest.cpp index 7ab51c2364..9480cd714d 100644 --- a/test/energy_consumption/EnergyConsumptionTest.cpp +++ b/test/energy_consumption/EnergyConsumptionTest.cpp @@ -184,7 +184,7 @@ class EnergyApiAccessExceptionsTestWMS : public wrench::WMS { throw std::runtime_error( "Should not have been able to read the energy for dummy hosts" ); - } catch (std::exception e) { + } catch (std::exception &e) { WRENCH_INFO("Expected exception as we were trying to measure the energy for a dummy host that is not available"); } @@ -193,7 +193,7 @@ class EnergyApiAccessExceptionsTestWMS : public wrench::WMS { throw std::runtime_error( "Should not have been able to read the energy for dummy hosts" ); - } catch (std::exception e) { + } catch (std::exception &e) { WRENCH_INFO("Expected exception as we were trying to measure the energy for a dummy host that is not available"); } @@ -202,7 +202,7 @@ class EnergyApiAccessExceptionsTestWMS : public wrench::WMS { throw std::runtime_error( "Should not have been able to read the energy for dummy hosts" ); - } catch (std::exception e) { + } catch (std::exception &e) { WRENCH_INFO("Expected exception as we were trying to measure the energy for a dummy host that is not available"); } @@ -211,7 +211,7 @@ class EnergyApiAccessExceptionsTestWMS : public wrench::WMS { throw std::runtime_error( "Should not have been able to read the energy for dummy hosts" ); - } catch (std::exception e) { + } catch (std::exception &e) { WRENCH_INFO("Expected exception as we were trying to access energy plugin for a dummy host that is not available"); } @@ -220,7 +220,7 @@ class EnergyApiAccessExceptionsTestWMS : public wrench::WMS { throw std::runtime_error( "Should not have been able to read the energy for dummy hosts" ); - } catch (std::exception e) { + } catch (std::exception &e) { WRENCH_INFO("Expected exception as we were trying to access energy plugin for a dummy host that is not available"); } @@ -229,7 +229,7 @@ class EnergyApiAccessExceptionsTestWMS : public wrench::WMS { throw std::runtime_error( "Should not have been able to read the energy for dummy hosts" ); - } catch (std::exception e) { + } catch (std::exception &e) { WRENCH_INFO("Unexpected exception as we were trying to access energy plugin for a correct host that is available"); } } From 53da740899cde80e08841c22456d74cca0b4b40a Mon Sep 17 00:00:00 2001 From: henricasanova Date: Fri, 7 Jun 2019 15:07:55 +0200 Subject: [PATCH 47/52] (#111) In the process of updating wrench-init --- tools/wrench/wrench-init | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/wrench/wrench-init b/tools/wrench/wrench-init index 5eece897b4..a23da7ce0a 100755 --- a/tools/wrench/wrench-init +++ b/tools/wrench/wrench-init @@ -90,7 +90,7 @@ def _write_cmakelists(project_dir): 'set(CMAKE_CXX_STANDARD 11)', '', '# include directories for dependencies and WRENCH libraries', - 'include_directories(src/ /usr/local/include /usr/local/include/wrench)', + 'include_directories(src/ /usr/local/include /opt/local/include /usr/local/include/wrench)', '', '# source files', 'set(SOURCE_FILES', @@ -291,8 +291,8 @@ def _write_simple_wms(project_dir): 'public:', ' SimpleWMS(std::unique_ptr standard_job_scheduler,', ' std::unique_ptr pilot_job_scheduler,', - ' const std::set &compute_services,', - ' const std::set &storage_services,', + ' const std::set> &compute_services,', + ' const std::set> &storage_services,', ' const std::string &hostname);', '', 'private:', @@ -321,8 +321,8 @@ def _write_simple_wms(project_dir): ' */', 'SimpleWMS::SimpleWMS(std::unique_ptr standard_job_scheduler,', ' std::unique_ptr pilot_job_scheduler,', - ' const std::set &compute_services,', - ' const std::set &storage_services,', + ' const std::set> &compute_services,', + ' const std::set> &storage_services,', ' const std::string &hostname) : wrench::WMS(', ' std::move(standard_job_scheduler),', ' std::move(pilot_job_scheduler),', @@ -355,7 +355,7 @@ def _write_simple_wms(project_dir): ' std::vector ready_tasks = this->getWorkflow()->getReadyTasks();', '', ' // Get the available compute services', - ' std::set compute_services = this->getAvailableComputeServices();', + ' auto compute_services = this->getAvailableComputeServices();', '', ' if (compute_services.empty()) {', ' WRENCH_INFO("Aborting - No compute services available!");', @@ -363,7 +363,7 @@ def _write_simple_wms(project_dir): ' }', '', ' // Run ready tasks with defined scheduler implementation', - ' this->getStandardJobScheduler()->scheduleTasks(this->getAvailableComputeServices(), ready_tasks);', + ' this->getStandardJobScheduler()->scheduleTasks(this->getAvailableComputeServices(), ready_tasks);', '', ' // Wait for a workflow execution event, and process it', ' try {', From 55b4f1f60994f97aacc30013a5b448694b6d4b1a Mon Sep 17 00:00:00 2001 From: henricasanova Date: Fri, 7 Jun 2019 15:43:22 +0200 Subject: [PATCH 48/52] (#111) Updated wrench-init so that it generates code compatible with current WRENCH APIs --- .../simple-example/SimpleSimulatorBatch.cpp | 295 ++++++++-------- .../simple-example/SimpleSimulatorCloud.cpp | 316 +++++++++--------- ...lComputeServiceResourceInformationTest.cpp | 2 +- .../VirtualizedClusterServiceTest.cpp | 6 +- tools/wrench/wrench-init | 54 ++- 5 files changed, 334 insertions(+), 339 deletions(-) diff --git a/examples/simple-example/SimpleSimulatorBatch.cpp b/examples/simple-example/SimpleSimulatorBatch.cpp index 25529801b9..2be74f0577 100644 --- a/examples/simple-example/SimpleSimulatorBatch.cpp +++ b/examples/simple-example/SimpleSimulatorBatch.cpp @@ -15,153 +15,152 @@ int main(int argc, char **argv) { - /* - * Declaration of the top-level WRENCH simulation object - */ - wrench::Simulation simulation; - - /* - * Initialization of the simulation, which may entail extracting WRENCH-specific and - * Simgrid-specific command-line arguments that can modify general simulation behavior. - * Two special command-line arguments are --help-wrench and --help-simgrid, which print - * details about available command-line arguments. - */ - simulation.init(&argc, argv); - - /* - * Parsing of the command-line arguments for this WRENCH simulation - */ - if (argc != 3) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - exit(1); - } - - /* The first argument is the platform description file, written in XML following the SimGrid-defined DTD */ - char *platform_file = argv[1]; - /* The second argument is the workflow description file, written in XML using the DAX DTD */ - char *workflow_file = argv[2]; - - - /* Reading and parsing the workflow description file to create a wrench::Workflow object */ - std::cerr << "Loading workflow..." << std::endl; - wrench::Workflow workflow; - workflow.loadFromDAXorJSON(workflow_file, "1000Gf"); - std::cerr << "The workflow has " << workflow.getNumberOfTasks() << " tasks " << std::endl; - std::cerr.flush(); - - /* Reading and parsing the platform description file to instantiate a simulated platform */ - std::cerr << "Instantiating SimGrid platform..." << std::endl; - simulation.instantiatePlatform(platform_file); - - /* Get a vector of all the hosts in the simulated platform */ - std::vector hostname_list = simulation.getHostnameList(); - - /* Instantiate a storage service, to be stated on some host in the simulated platform, - * and adding it to the simulation. A wrench::StorageService is an abstraction of a service on - * which files can be written and read. This particular storage service has a capacity - * of 10,000,000,000,000 bytes, and is a SimpleStorageService instance. The SimpleStorageService - * is a barebone storage service implementation provided by WRENCH. - * Throughout the simulation execution, input/output files of workflow tasks will be located - * in this storage service. - */ - std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1]; - std::cerr << "Instantiating a SimpleStorageService on " << storage_host << "..." << std::endl; - auto storage_service = simulation.add( - new wrench::SimpleStorageService(storage_host, 10000000000000.0)); - - std::string wms_host = hostname_list[0]; - - /* Instantiate a batch service, to be started on some host in the simulation platform. - * A batch service is an abstraction of a compute service that corresponds to - * batch-scheduled platforms in which jobs are submitted to a queue and dispatched - * to compute nodes according to various scheduling algorithms. - * In this example, this particular batch service has no scratch storage space (size = 0). - * The last argument to the constructor - * shows how to configure particular simulated behaviors of the compute service via a property - * list. In this example, one specifies that the message that will be send to the service to - * terminate it will be 2048 bytes. See the documentation to find out all available - * configurable properties for each kind of service. - */ - std::shared_ptr batch_service; - - /* Add the batch service to the simulation, catching a possible exception */ - try { - batch_service = simulation.add(new wrench::BatchComputeService( - wms_host, hostname_list, 0, {}, - {{wrench::BatchComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 2048}})); - - } catch (std::invalid_argument &e) { - std::cerr << "Error: " << e.what() << std::endl; - std::exit(1); - } - - /* Create a list of compute services that will be used by the WMS */ - std::set> compute_services; - compute_services.insert(batch_service); - - /* Create a list of storage services that will be used by the WMS */ - std::set> storage_services; - storage_services.insert(storage_service); - - /* Instantiate a WMS, to be stated on some host (wms_host), which is responsible - * for executing the workflow, and uses a scheduler (BatchStandardJobScheduler). That scheduler - * is instantiated with the batch service, the list of hosts available for running - * tasks, and also provided a pointer to the simulation object. - * - * The WMS implementation is in SimpleWMS.[cpp|h]. - */ - std::cerr << "Instantiating a WMS on " << wms_host << "..." << std::endl; - auto wms = simulation.add( - new wrench::SimpleWMS(std::unique_ptr( - new wrench::BatchStandardJobScheduler(storage_service)), - nullptr, compute_services, storage_services, wms_host)); - - wms->addWorkflow(&workflow); - - /* Instantiate a file registry service to be started on some host. This service is - * essentially a replica catalog that stores pairs so that - * any service, in particular a WMS, can discover where workflow files are stored. - */ - std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0]; - std::cerr << "Instantiating a FileRegistryService on " << file_registry_service_host << "..." << std::endl; - auto file_registry_service = - simulation.add(new wrench::FileRegistryService(file_registry_service_host)); - - /* It is necessary to store, or "stage", input files for the first task(s) of the workflow on some storage - * service, so that workflow execution can be initiated. The getInputFiles() method of the Workflow class - * returns the set of all workflow files that are not generated by workflow tasks, and thus are only input files. - * These files are then staged on the storage service. - */ - std::cerr << "Staging input files..." << std::endl; - std::map input_files = workflow.getInputFiles(); - for (auto f : input_files) { - std::cerr << "---> " << f.second->getID() << "\n"; - } - try { - simulation.stageFiles(input_files, storage_service); - } catch (std::runtime_error &e) { - std::cerr << "Exception: " << e.what() << std::endl; - return 0; - } - - /* Launch the simulation. This call only returns when the simulation is complete. */ - std::cerr << "Launching the Simulation..." << std::endl; - try { - simulation.launch(); - } catch (std::runtime_error &e) { - std::cerr << "Exception: " << e.what() << std::endl; + /* + * Declaration of the top-level WRENCH simulation object + */ + wrench::Simulation simulation; + + /* + * Initialization of the simulation, which may entail extracting WRENCH-specific and + * Simgrid-specific command-line arguments that can modify general simulation behavior. + * Two special command-line arguments are --help-wrench and --help-simgrid, which print + * details about available command-line arguments. + */ + simulation.init(&argc, argv); + + /* + * Parsing of the command-line arguments for this WRENCH simulation + */ + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + exit(1); + } + + /* The first argument is the platform description file, written in XML following the SimGrid-defined DTD */ + char *platform_file = argv[1]; + /* The second argument is the workflow description file, written in XML using the DAX DTD */ + char *workflow_file = argv[2]; + + + /* Reading and parsing the workflow description file to create a wrench::Workflow object */ + std::cerr << "Loading workflow..." << std::endl; + wrench::Workflow workflow; + workflow.loadFromDAXorJSON(workflow_file, "1000Gf"); + std::cerr << "The workflow has " << workflow.getNumberOfTasks() << " tasks " << std::endl; + std::cerr.flush(); + + /* Reading and parsing the platform description file to instantiate a simulated platform */ + std::cerr << "Instantiating SimGrid platform..." << std::endl; + simulation.instantiatePlatform(platform_file); + + /* Get a vector of all the hosts in the simulated platform */ + std::vector hostname_list = simulation.getHostnameList(); + + /* Create a list of storage services that will be used by the WMS */ + std::set> storage_services; + + /* Instantiate a storage service, to be stated on some host in the simulated platform, + * and adding it to the simulation. A wrench::StorageService is an abstraction of a service on + * which files can be written and read. This particular storage service has a capacity + * of 10,000,000,000,000 bytes, and is a SimpleStorageService instance. The SimpleStorageService + * is a barebone storage service implementation provided by WRENCH. + * Throughout the simulation execution, input/output files of workflow tasks will be located + * in this storage service. + */ + std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1]; + std::cerr << "Instantiating a SimpleStorageService on " << storage_host << "..." << std::endl; + auto storage_service = simulation.add( + new wrench::SimpleStorageService(storage_host, 10000000000000.0)); + storage_services.insert(storage_service); + + std::string wms_host = hostname_list[0]; + + /* Create a list of compute services that will be used by the WMS */ + std::set> compute_services; + + /* Instantiate a batch service, to be started on some host in the simulation platform. + * A batch service is an abstraction of a compute service that corresponds to + * batch-scheduled platforms in which jobs are submitted to a queue and dispatched + * to compute nodes according to various scheduling algorithms. + * In this example, this particular batch service has no scratch storage space (size = 0). + * The last argument to the constructor + * shows how to configure particular simulated behaviors of the compute service via a property + * list. In this example, one specifies that the message that will be send to the service to + * terminate it will be 2048 bytes. See the documentation to find out all available + * configurable properties for each kind of service. + */ + + /* Add the batch service to the simulation, catching a possible exception */ + try { + auto batch_service = simulation.add(new wrench::BatchComputeService( + wms_host, hostname_list, 0, {}, + {{wrench::BatchComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 2048}})); + compute_services.insert(batch_service); + } catch (std::invalid_argument &e) { + std::cerr << "Error: " << e.what() << std::endl; + std::exit(1);} + + + + /* Instantiate a WMS, to be stated on some host (wms_host), which is responsible + * for executing the workflow, and uses a scheduler (BatchStandardJobScheduler). That scheduler + * is instantiated with the batch service, the list of hosts available for running + * tasks, and also provided a pointer to the simulation object. + * + * The WMS implementation is in SimpleWMS.[cpp|h]. + */ + std::cerr << "Instantiating a WMS on " << wms_host << "..." << std::endl; + auto wms = simulation.add( + new wrench::SimpleWMS(std::unique_ptr( + new wrench::BatchStandardJobScheduler(storage_service)), + nullptr, compute_services, storage_services, wms_host)); + + wms->addWorkflow(&workflow); + + /* Instantiate a file registry service to be started on some host. This service is + * essentially a replica catalog that stores pairs so that + * any service, in particular a WMS, can discover where workflow files are stored. + */ + std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0]; + std::cerr << "Instantiating a FileRegistryService on " << file_registry_service_host << "..." << std::endl; + auto file_registry_service = + simulation.add(new wrench::FileRegistryService(file_registry_service_host)); + + /* It is necessary to store, or "stage", input files for the first task(s) of the workflow on some storage + * service, so that workflow execution can be initiated. The getInputFiles() method of the Workflow class + * returns the set of all workflow files that are not generated by workflow tasks, and thus are only input files. + * These files are then staged on the storage service. + */ + std::cerr << "Staging input files..." << std::endl; + std::map input_files = workflow.getInputFiles(); + for (auto f : input_files) { + std::cerr << "---> " << f.second->getID() << "\n"; + } + try { + simulation.stageFiles(input_files, storage_service); + } catch (std::runtime_error &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return 0; + } + + /* Launch the simulation. This call only returns when the simulation is complete. */ + std::cerr << "Launching the Simulation..." << std::endl; + try { + simulation.launch(); + } catch (std::runtime_error &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return 0; + } + std::cerr << "Simulation done!" << std::endl; + + /* Simulation results can be examined via simulation.output, which provides access to traces + * of events. In the code below, we retrieve the trace of all task completion events, print how + * many such events there are, and print some information for the first such event. + */ + std::vector *> trace; + trace = simulation.getOutput().getTrace(); + std::cerr << "Number of entries in TaskCompletion trace: " << trace.size() << std::endl; + std::cerr << "Task in first trace entry: " << trace[0]->getContent()->getTask()->getID() << std::endl; + return 0; - } - std::cerr << "Simulation done!" << std::endl; - - /* Simulation results can be examined via simulation.output, which provides access to traces - * of events. In the code below, we retrieve the trace of all task completion events, print how - * many such events there are, and print some information for the first such event. - */ - std::vector *> trace; - trace = simulation.getOutput().getTrace(); - std::cerr << "Number of entries in TaskCompletion trace: " << trace.size() << std::endl; - std::cerr << "Task in first trace entry: " << trace[0]->getContent()->getTask()->getID() << std::endl; - - return 0; } diff --git a/examples/simple-example/SimpleSimulatorCloud.cpp b/examples/simple-example/SimpleSimulatorCloud.cpp index d1291d88b9..87ff37726f 100644 --- a/examples/simple-example/SimpleSimulatorCloud.cpp +++ b/examples/simple-example/SimpleSimulatorCloud.cpp @@ -23,163 +23,163 @@ */ int main(int argc, char **argv) { - /* - * Declaration of the top-level WRENCH simulation object - */ - wrench::Simulation simulation; - - /* - * Initialization of the simulation, which may entail extracting WRENCH-specific and - * Simgrid-specific command-line arguments that can modify general simulation behavior. - * Two special command-line arguments are --help-wrench and --help-simgrid, which print - * details about available command-line arguments. - */ - simulation.init(&argc, argv); - - /* - * Parsing of the command-line arguments for this WRENCH simulation - */ - if (argc != 3) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - exit(1); - } - - /* The first argument is the platform description file, written in XML following the SimGrid-defined DTD */ - char *platform_file = argv[1]; - /* The second argument is the workflow description file, written in XML using the DAX DTD */ - char *workflow_file = argv[2]; - - /* Reading and parsing the workflow description file to create a wrench::Workflow object */ - std::cerr << "Loading workflow..." << std::endl; - wrench::Workflow workflow; - workflow.loadFromDAXorJSON(workflow_file, "1000Gf"); - std::cerr << "The workflow has " << workflow.getNumberOfTasks() << " tasks " << std::endl; - std::cerr.flush(); - - /* Reading and parsing the platform description file to instantiate a simulated platform */ - std::cerr << "Instantiating SimGrid platform..." << std::endl; - simulation.instantiatePlatform(platform_file); - - /* Get a vector of all the hosts in the simulated platform */ - std::vector hostname_list = simulation.getHostnameList(); - - /* Instantiate a storage service, to be stated on some host in the simulated platform, - * and adding it to the simulation. A wrench::StorageService is an abstraction of a service on - * which files can be written and read. This particular storage service has a capacity - * of 10,000,000,000,000 bytes, and is a SimpleStorageService instance. The SimpleStorageService - * is a barebone storage service implementation provided by WRENCH. - * Throughout the simulation execution, input/output files of workflow tasks will be located - * in this storage service. - */ - std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1]; - std::cerr << "Instantiating a SimpleStorageService on " << storage_host << "..." << std::endl; - auto storage_service = simulation.add( - new wrench::SimpleStorageService(storage_host, 10000000000000.0)); - - /* Construct a list of hosts (in the example only one host) on which the - * cloud service will be able to run tasks - */ - std::string executor_host = hostname_list[(hostname_list.size() > 1) ? 1 : 0]; - std::vector execution_hosts = {executor_host}; - - /* Instantiate a cloud service, to be started on some host in the simulation platform. - * A cloud service is an abstraction of a compute service that corresponds to a - * Cloud platform that provides access to virtualized compute resources. - * In this example, this particular cloud service has no scratch storage space (size = 0). - * The last argument to the constructor - * shows how to configure particular simulated behaviors of the compute service via a property - * list. In this example, one specified that the message that will be send to the service to - * terminate it will by 1024 bytes. See the documentation to find out all available - * configurable properties for each kind of service. - */ - std::string wms_host = hostname_list[0]; - std::shared_ptr cloud_service; - - /* Add the cloud service to the simulation, catching a possible exception */ - try { - cloud_service = simulation.add(new wrench::CloudComputeService( - wms_host, execution_hosts, 0, {}, - {{wrench::CloudComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 1024}})); - - } catch (std::invalid_argument &e) { - std::cerr << "Error: " << e.what() << std::endl; - std::exit(1); - } - - /* Create a list of compute services that will be used by the WMS */ - std::set> compute_services; - compute_services.insert(cloud_service); - - /* Create a list of storage services that will be used by the WMS */ - std::set> storage_services; - storage_services.insert(storage_service); - - /* Instantiate a WMS, to be stated on some host (wms_host), which is responsible - * for executing the workflow, and uses a scheduler (CloudStandardJobScheduler). That scheduler - * is instantiated with the cloud service, the list of hosts available for running - * tasks, and also provided a pointer to the simulation object. - * - * The WMS implementation is in SimpleWMS.[cpp|h]. - */ - auto wms = simulation.add( - new wrench::SimpleWMS(std::unique_ptr( - new wrench::CloudStandardJobScheduler(storage_service)), - nullptr, compute_services, storage_services, wms_host)); - - wms->addWorkflow(&workflow); - - /* Instantiate a file registry service to be started on some host. This service is - * essentially a replica catalog that stores pairs so that - * any service, in particular a WMS, can discover where workflow files are stored. - */ - std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0]; - std::cerr << "Instantiating a FileRegistryService on " << file_registry_service_host << "..." << std::endl; - wrench::FileRegistryService *file_registry_service = - new wrench::FileRegistryService(file_registry_service_host); - simulation.add(file_registry_service); - - // TRYING NETWORK PROXIMITY SERVICE WITH VIVALDI.... - std::vector hostname_list_copy(hostname_list); - std::string hostname_copy(hostname_list[0]); - wrench::NetworkProximityService *NPS = new wrench::NetworkProximityService(hostname_copy, hostname_list_copy, { - {wrench::NetworkProximityServiceProperty::NETWORK_PROXIMITY_SERVICE_TYPE, "alltoall"}, - {wrench::NetworkProximityServiceProperty::NETWORK_DAEMON_COMMUNICATION_COVERAGE, "1.0"}, - {wrench::NetworkProximityServiceProperty::NETWORK_PROXIMITY_MEASUREMENT_PERIOD, "600"}, - {wrench::NetworkProximityServiceProperty::NETWORK_PROXIMITY_MEASUREMENT_PERIOD_MAX_NOISE, "10"}}); - - simulation.add(NPS); - /* It is necessary to store, or "stage", input files for the first task(s) of the workflow on some storage - * service, so that workflow execution can be initiated. The getInputFiles() method of the Workflow class - * returns the set of all workflow files that are not generated by workflow tasks, and thus are only input files. - * These files are then staged on the storage service. - */ - std::cerr << "Staging input files..." << std::endl; - std::map input_files = workflow.getInputFiles(); - try { - simulation.stageFiles(input_files, storage_service); - } catch (std::runtime_error &e) { - std::cerr << "Exception: " << e.what() << std::endl; - return 0; - } - - /* Launch the simulation. This call only returns when the simulation is complete. */ - std::cerr << "Launching the Simulation..." << std::endl; - try { - simulation.launch(); - } catch (std::runtime_error &e) { - std::cerr << "Exception: " << e.what() << std::endl; + /* + * Declaration of the top-level WRENCH simulation object + */ + wrench::Simulation simulation; + + /* + * Initialization of the simulation, which may entail extracting WRENCH-specific and + * Simgrid-specific command-line arguments that can modify general simulation behavior. + * Two special command-line arguments are --help-wrench and --help-simgrid, which print + * details about available command-line arguments. + */ + simulation.init(&argc, argv); + + /* + * Parsing of the command-line arguments for this WRENCH simulation + */ + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + exit(1); + } + + /* The first argument is the platform description file, written in XML following the SimGrid-defined DTD */ + char *platform_file = argv[1]; + /* The second argument is the workflow description file, written in XML using the DAX DTD */ + char *workflow_file = argv[2]; + + /* Reading and parsing the workflow description file to create a wrench::Workflow object */ + std::cerr << "Loading workflow..." << std::endl; + wrench::Workflow workflow; + workflow.loadFromDAXorJSON(workflow_file, "1000Gf"); + std::cerr << "The workflow has " << workflow.getNumberOfTasks() << " tasks " << std::endl; + std::cerr.flush(); + + /* Reading and parsing the platform description file to instantiate a simulated platform */ + std::cerr << "Instantiating SimGrid platform..." << std::endl; + simulation.instantiatePlatform(platform_file); + + /* Get a vector of all the hosts in the simulated platform */ + std::vector hostname_list = simulation.getHostnameList(); + + /* Create a list of storage services that will be used by the WMS */ + std::set> storage_services; + + /* Instantiate a storage service, to be stated on some host in the simulated platform, + * and adding it to the simulation. A wrench::StorageService is an abstraction of a service on + * which files can be written and read. This particular storage service has a capacity + * of 10,000,000,000,000 bytes, and is a SimpleStorageService instance. The SimpleStorageService + * is a barebone storage service implementation provided by WRENCH. + * Throughout the simulation execution, input/output files of workflow tasks will be located + * in this storage service. + */ + std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1]; + std::cerr << "Instantiating a SimpleStorageService on " << storage_host << "..." << std::endl; + auto storage_service = simulation.add( + new wrench::SimpleStorageService(storage_host, 10000000000000.0)); + storage_services.insert(storage_service); + + /* Construct a list of hosts (in the example only one host) on which the + * cloud service will be able to run tasks + */ + std::string executor_host = hostname_list[(hostname_list.size() > 1) ? 1 : 0]; + std::vector execution_hosts = {executor_host}; + + /* Create a list of compute services that will be used by the WMS */ + std::set> compute_services; + + /* Instantiate a cloud service, to be started on some host in the simulation platform. + * A cloud service is an abstraction of a compute service that corresponds to a + * Cloud platform that provides access to virtualized compute resources. + * In this example, this particular cloud service has no scratch storage space (size = 0). + * The last argument to the constructor + * shows how to configure particular simulated behaviors of the compute service via a property + * list. In this example, one specified that the message that will be send to the service to + * terminate it will by 1024 bytes. See the documentation to find out all available + * configurable properties for each kind of service. + */ + std::string wms_host = hostname_list[0]; + + /* Add the cloud service to the simulation, catching a possible exception */ + try { + auto cloud_service = simulation.add(new wrench::CloudComputeService( + wms_host, execution_hosts, 0, {}, + {{wrench::CloudComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 1024}})); + compute_services.insert(cloud_service); + } catch (std::invalid_argument &e) { + std::cerr << "Error: " << e.what() << std::endl; + std::exit(1); + } + + + + /* Instantiate a WMS, to be stated on some host (wms_host), which is responsible + * for executing the workflow, and uses a scheduler (CloudStandardJobScheduler). That scheduler + * is instantiated with the cloud service, the list of hosts available for running + * tasks, and also provided a pointer to the simulation object. + * + * The WMS implementation is in SimpleWMS.[cpp|h]. + */ + auto wms = simulation.add( + new wrench::SimpleWMS(std::unique_ptr( + new wrench::CloudStandardJobScheduler(storage_service)), + nullptr, compute_services, storage_services, wms_host)); + + wms->addWorkflow(&workflow); + + /* Instantiate a file registry service to be started on some host. This service is + * essentially a replica catalog that stores pairs so that + * any service, in particular a WMS, can discover where workflow files are stored. + */ + std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0]; + std::cerr << "Instantiating a FileRegistryService on " << file_registry_service_host << "..." << std::endl; + wrench::FileRegistryService *file_registry_service = + new wrench::FileRegistryService(file_registry_service_host); + simulation.add(file_registry_service); + + // TRYING NETWORK PROXIMITY SERVICE WITH VIVALDI.... + std::vector hostname_list_copy(hostname_list); + std::string hostname_copy(hostname_list[0]); + wrench::NetworkProximityService *NPS = new wrench::NetworkProximityService(hostname_copy, hostname_list_copy, { + {wrench::NetworkProximityServiceProperty::NETWORK_PROXIMITY_SERVICE_TYPE, "alltoall"}, + {wrench::NetworkProximityServiceProperty::NETWORK_DAEMON_COMMUNICATION_COVERAGE, "1.0"}, + {wrench::NetworkProximityServiceProperty::NETWORK_PROXIMITY_MEASUREMENT_PERIOD, "600"}, + {wrench::NetworkProximityServiceProperty::NETWORK_PROXIMITY_MEASUREMENT_PERIOD_MAX_NOISE, "10"}}); + + simulation.add(NPS); + /* It is necessary to store, or "stage", input files for the first task(s) of the workflow on some storage + * service, so that workflow execution can be initiated. The getInputFiles() method of the Workflow class + * returns the set of all workflow files that are not generated by workflow tasks, and thus are only input files. + * These files are then staged on the storage service. + */ + std::cerr << "Staging input files..." << std::endl; + std::map input_files = workflow.getInputFiles(); + try { + simulation.stageFiles(input_files, storage_service); + } catch (std::runtime_error &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return 0; + } + + /* Launch the simulation. This call only returns when the simulation is complete. */ + std::cerr << "Launching the Simulation..." << std::endl; + try { + simulation.launch(); + } catch (std::runtime_error &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return 0; + } + std::cerr << "Simulation done!" << std::endl; + + /* Simulation results can be examined via simulation.output, which provides access to traces + * of events. In the code below, we retrieve the trace of all task completion events, print how + * many such events there are, and print some information for the first such event. + */ + std::vector *> trace; + trace = simulation.getOutput().getTrace(); + std::cerr << "Number of entries in TaskCompletion trace: " << trace.size() << std::endl; + std::cerr << "Task in first trace entry: " << trace[0]->getContent()->getTask()->getID() << std::endl; + return 0; - } - std::cerr << "Simulation done!" << std::endl; - - /* Simulation results can be examined via simulation.output, which provides access to traces - * of events. In the code below, we retrieve the trace of all task completion events, print how - * many such events there are, and print some information for the first such event. - */ - std::vector *> trace; - trace = simulation.getOutput().getTrace(); - std::cerr << "Number of entries in TaskCompletion trace: " << trace.size() << std::endl; - std::cerr << "Task in first trace entry: " << trace[0]->getContent()->getTask()->getID() << std::endl; - - return 0; } diff --git a/test/compute_services/BareMetalComputeService/BareMetalComputeServiceResourceInformationTest.cpp b/test/compute_services/BareMetalComputeService/BareMetalComputeServiceResourceInformationTest.cpp index a3a51a5b0f..32f6fa986b 100644 --- a/test/compute_services/BareMetalComputeService/BareMetalComputeServiceResourceInformationTest.cpp +++ b/test/compute_services/BareMetalComputeService/BareMetalComputeServiceResourceInformationTest.cpp @@ -188,7 +188,7 @@ class ResourceInformationTestWMS : public wrench::WMS { if ((idle_core_counts.size() != 2) or (idle_core_counts[0] != 1) or (idle_core_counts[1] != 2)) { - throw std::runtime_error("getNumIdleCores() should return {1,2} or {2,1} for compute service #1"); + throw std::runtime_error("getPerHostNumIdleCores() should return {1,2} or {2,1} for compute service #1"); } if (this->test->compute_service1->getTotalNumIdleCores() != 3) { diff --git a/test/compute_services/VirtualizedClusterServiceTest.cpp b/test/compute_services/VirtualizedClusterServiceTest.cpp index 817536c6db..1d2c7e72f4 100644 --- a/test/compute_services/VirtualizedClusterServiceTest.cpp +++ b/test/compute_services/VirtualizedClusterServiceTest.cpp @@ -559,7 +559,7 @@ class CloudNumCoresTestWMS : public wrench::WMS { unsigned long sum_num_idle_cores = this->test->compute_service->getTotalNumIdleCores(); if (sum_num_cores != 6 || sum_num_idle_cores != 6) { - throw std::runtime_error("getHostNumCores() and getNumIdleCores() should be 6 (they report " + + throw std::runtime_error("getTotalNumCores() and getTotalNumIdleCores() should be 6 (they report " + std::to_string(sum_num_cores) + " and " + std::to_string(sum_num_idle_cores)+ ")"); } @@ -570,7 +570,7 @@ class CloudNumCoresTestWMS : public wrench::WMS { sum_num_idle_cores = cs->getTotalNumIdleCores(); if (sum_num_idle_cores != 4) { - throw std::runtime_error("getNumIdleCores() should be 4 (it is reported as " + std::to_string(sum_num_idle_cores) + ")"); + throw std::runtime_error("getTotalNumIdleCores() should be 4 (it is reported as " + std::to_string(sum_num_idle_cores) + ")"); } // create and start a VM with two cores @@ -578,7 +578,7 @@ class CloudNumCoresTestWMS : public wrench::WMS { sum_num_idle_cores = cs->getTotalNumIdleCores(); if (sum_num_idle_cores != 2) { - throw std::runtime_error("getHostNumCores() and getNumIdleCores() should be 2 (it is reported as " + std::to_string(sum_num_idle_cores) + ")"); + throw std::runtime_error("getTotalNumCores() and getTotalNumIdleCores() should be 2 (it is reported as " + std::to_string(sum_num_idle_cores) + ")"); } } catch (wrench::WorkflowExecutionException &e) { diff --git a/tools/wrench/wrench-init b/tools/wrench/wrench-init index a23da7ce0a..0c8c3f4c16 100755 --- a/tools/wrench/wrench-init +++ b/tools/wrench/wrench-init @@ -177,61 +177,57 @@ def _write_main(project_dir, compute_service): '', ' // Instantiate a storage service', ' std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1];', - ' wrench::StorageService *storage_service = simulation.add(new wrench::SimpleStorageService(storage_host, 10000000000000.0));', + ' auto storage_service = simulation.add(new wrench::SimpleStorageService(storage_host, 10000000000000.0));', '', ' // Construct a list of hosts (in this example only one host)', ' std::string executor_host = hostname_list[(hostname_list.size() > 1) ? 1 : 0];', ' std::vector execution_hosts = {executor_host};', '', ' // Create a list of storage services that will be used by the WMS', - ' std::set storage_services;', + ' std::set> storage_services;', ' storage_services.insert(storage_service);', '', ' // Create a list of compute services that will be used by the WMS', - ' std::set compute_services;', + ' std::set> compute_services;', '', ' std::string wms_host = hostname_list[0];' ]) if compute_service == 'cloud': file_contents.extend([ - ' // Instantiate a cloud service', - ' wrench::CloudService *cloud_service = new wrench::CloudService(', + ' // Instantiate a cloud service and add it to the simulation', + ' try {', + ' auto cloud_service = new wrench::CloudComputeService(', ' wms_host, execution_hosts, 0, {},', - ' {{wrench::CloudServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, "1024"}});', + ' {{wrench::CloudComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 1024}});', '', - ' // Add the cloud service to the simulation', - ' try {', - ' simulation.add(cloud_service);', + ' compute_services.insert(simulation.add(cloud_service));', ' } catch (std::invalid_argument &e) {', ' std::cerr << "Error: " << e.what() << std::endl;', ' std::exit(1);', ' }', - ' compute_services.insert(cloud_service);', '' ]) elif compute_service == 'batch': file_contents.extend([ - ' // Instantiate a batch service', - ' wrench::ComputeService *batch_service = new wrench::BatchService(', + ' // Instantiate a batch service and add it to the simulation', + ' try {', + ' auto batch_service = new wrench::BatchService(', ' wms_host, hostname_list, 0, {},', - ' {{wrench::BatchServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, "2048"}});', + ' {{wrench::BatchServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 2048}});', '', - ' // Add the batch service to the simulation', - ' try {', - ' simulation.add(batch_service);', + ' compute_services.insert(simulation.add(batch_service));', ' } catch (std::invalid_argument &e) {', ' std::cerr << "Error: " << e.what() << std::endl;', ' std::exit(1);', ' }', - ' compute_services.insert(batch_service);', '' ]) file_contents.extend([ ' // Instantiate a WMS', - ' wrench::WMS *wms = simulation.add(', + ' auto wms = simulation.add(', ' new SimpleWMS(std::unique_ptr(', ' new SimpleStandardJobScheduler(storage_service)),', ' nullptr, compute_services, storage_services, wms_host));', @@ -239,12 +235,12 @@ def _write_main(project_dir, compute_service): '', ' // Instantiate a file registry service', ' std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0];', - ' wrench::FileRegistryService * file_registry_service =', + ' auto file_registry_service =', ' new wrench::FileRegistryService(file_registry_service_host);', ' simulation.add(file_registry_service);', '', ' // It is necessary to store, or "stage", input files', - ' std::map input_files = workflow.getInputFiles();', + ' auto input_files = workflow.getInputFiles();', ' try {', ' simulation.stageFiles(input_files, storage_service);', ' } catch (std::runtime_error &e) {', @@ -405,14 +401,14 @@ def _write_simple_scheduler(project_dir, compute_service): '', 'class SimpleStandardJobScheduler : public wrench::StandardJobScheduler {', 'public:', - ' SimpleStandardJobScheduler(wrench::StorageService *default_storage_service) :', + ' SimpleStandardJobScheduler(std::shared_ptr default_storage_service) :', ' default_storage_service(default_storage_service) {}', '', - ' void scheduleTasks(const std::set &compute_services,', + ' void scheduleTasks(const std::set> &compute_services,', ' const std::vector &tasks);', '', 'private:', - ' wrench::StorageService *default_storage_service;', + ' std::shared_ptr default_storage_service;', ]) if compute_service == 'cloud': @@ -442,7 +438,7 @@ def _write_simple_scheduler(project_dir, compute_service): ' *', ' * @throw std::runtime_error', ' */', - 'void SimpleStandardJobScheduler::scheduleTasks(const std::set &compute_services,', + 'void SimpleStandardJobScheduler::scheduleTasks(const std::set> &compute_services,', ' const std::vector &tasks) {', '', ' // Check that the right compute_services is passed', @@ -450,13 +446,13 @@ def _write_simple_scheduler(project_dir, compute_service): ' throw std::runtime_error("This example Simple Scheduler requires a single compute service");', ' }', '', - ' wrench::ComputeService *compute_service = *compute_services.begin();' + ' auto compute_service = *compute_services.begin();' ]) if compute_service == 'cloud': file_contents.extend([ - ' wrench::CloudService *cloud_service;', - ' if (not(cloud_service = dynamic_cast(compute_service))) {', + ' std::shared_ptr cloud_service;', + ' if (not(cloud_service = std::dynamic_pointer_cast(compute_service))) {', ' throw std::runtime_error("This example Cloud Scheduler can only handle a cloud service");', ' }', '', @@ -471,7 +467,7 @@ def _write_simple_scheduler(project_dir, compute_service): '', ' // Check that it can run it right now in terms of idle cores', ' try {', - ' std::map num_idle_cores = compute_service->getNumIdleCores();', + ' std::map num_idle_cores = compute_service->getPerHostNumIdleCores();', ' for (auto const &ic : num_idle_cores) {', ' sum_num_idle_cores += ic.second;', ' }', @@ -479,7 +475,7 @@ def _write_simple_scheduler(project_dir, compute_service): ' throw std::runtime_error("Unable to get the number of idle cores.");', ' }', '', - ' std::map file_locations;', + ' std::map> file_locations;', ' for (auto f : task->getInputFiles()) {', ' file_locations.insert(std::make_pair(f, default_storage_service));', ' }', From c514c2b64e4a6c33837c0d5e658d5f09604653af Mon Sep 17 00:00:00 2001 From: henricasanova Date: Fri, 7 Jun 2019 15:48:20 +0200 Subject: [PATCH 49/52] Example tweaks --- examples/simple-example/SimpleSimulatorBatch.cpp | 2 +- examples/simple-example/SimpleSimulatorCloud.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple-example/SimpleSimulatorBatch.cpp b/examples/simple-example/SimpleSimulatorBatch.cpp index 2be74f0577..2d47a5839e 100644 --- a/examples/simple-example/SimpleSimulatorBatch.cpp +++ b/examples/simple-example/SimpleSimulatorBatch.cpp @@ -132,7 +132,7 @@ int main(int argc, char **argv) { * These files are then staged on the storage service. */ std::cerr << "Staging input files..." << std::endl; - std::map input_files = workflow.getInputFiles(); + auto input_files = workflow.getInputFiles(); for (auto f : input_files) { std::cerr << "---> " << f.second->getID() << "\n"; } diff --git a/examples/simple-example/SimpleSimulatorCloud.cpp b/examples/simple-example/SimpleSimulatorCloud.cpp index 87ff37726f..5ab08bcd63 100644 --- a/examples/simple-example/SimpleSimulatorCloud.cpp +++ b/examples/simple-example/SimpleSimulatorCloud.cpp @@ -154,7 +154,7 @@ int main(int argc, char **argv) { * These files are then staged on the storage service. */ std::cerr << "Staging input files..." << std::endl; - std::map input_files = workflow.getInputFiles(); + auto input_files = workflow.getInputFiles(); try { simulation.stageFiles(input_files, storage_service); } catch (std::runtime_error &e) { From 976a7c2c5b0a2223356ac5f1695e85b3604ad2fc Mon Sep 17 00:00:00 2001 From: Rafael Ferreira da Silva Date: Tue, 18 Jun 2019 10:49:08 +0200 Subject: [PATCH 50/52] fixing travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f256eb3cd1..3d1b7ab3f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,9 +39,9 @@ before_install: fi script: + # building wrench - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker exec -w /home/wrench/wrench/build -it wrench cmake -DENABLE_BATSCHED=${BATSCHED} -DCMAKE_VERBOSE_MAKEFILE=ON -DCOVERAGE=1 ..; - #docker exec -w /home/wrench/wrench/build -it wrench make all unit_tests doc-gh; docker exec -w /home/wrench/wrench/build -it wrench make all unit_tests; docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests; fi From 2dead0b3767cfd14f95bbf2237976a0c89b7bef4 Mon Sep 17 00:00:00 2001 From: Rafael Ferreira da Silva Date: Tue, 18 Jun 2019 11:29:20 +0200 Subject: [PATCH 51/52] adding wait for unit tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3d1b7ab3f0..c2cee3e896 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ script: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker exec -w /home/wrench/wrench/build -it wrench cmake -DENABLE_BATSCHED=${BATSCHED} -DCMAKE_VERBOSE_MAKEFILE=ON -DCOVERAGE=1 ..; docker exec -w /home/wrench/wrench/build -it wrench make all unit_tests; - docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests; + travis_wait docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests; fi after_success: From e61ad3a8806b43a4248d8e95b05a0326998a0ac8 Mon Sep 17 00:00:00 2001 From: Rafael Ferreira da Silva Date: Tue, 18 Jun 2019 23:20:05 +0200 Subject: [PATCH 52/52] adding travis_wait for tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c2cee3e896..c7bfeb65e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ script: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker exec -w /home/wrench/wrench/build -it wrench cmake -DENABLE_BATSCHED=${BATSCHED} -DCMAKE_VERBOSE_MAKEFILE=ON -DCOVERAGE=1 ..; docker exec -w /home/wrench/wrench/build -it wrench make all unit_tests; - travis_wait docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests; + travis_wait sleep infinity & docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests; fi after_success: