Merge pull request #48 from ICRAR/master

merge

daliuge-engine/dlg/drop.py

@@ -1132,19 +1132,31 @@ def setCompleted(self):
         '''
         Override this method in order to get the size of the drop set once it is completed.
         '''
+        # TODO: This implementation is almost a verbatim copy of the base class'
+        # so we should look into merging them
         status = self.status
         if status == DROPStates.CANCELLED:
             return
-        if status not in [DROPStates.INITIALIZED, DROPStates.WRITING]:
+        elif status == DROPStates.SKIPPED:
+            self._fire('dropCompleted', status=status)
+            return
+        elif status not in [DROPStates.INITIALIZED, DROPStates.WRITING]:
             raise Exception("%r not in INITIALIZED or WRITING state (%s), cannot setComplete()" % (self, self.status))
 
         self._closeWriters()
 
         logger.debug("Moving %r to COMPLETED", self)
         self.status = DROPStates.COMPLETED
 
-        # here we set the size
-        self._size = os.stat(self.path).st_size
+        # here we set the size. It could happen that nothing is written into
+        # this file, in which case we create an empty file so applications
+        # downstream don't fail to read
+        try:
+            self._size = os.stat(self.path).st_size
+        except FileNotFoundError:
+            with open(self.path, 'wb'):
+                pass
+            self._size = 0
         # Signal our subscribers that the show is over
         self._fire('dropCompleted', status=DROPStates.COMPLETED)
 

daliuge-engine/dlg/manager/web/dim.html

@@ -11,6 +11,7 @@
 	<script src="/static/js/bootbox.min.js"></script>
 	<link href="/static/css/bootstrap.min.css" rel="stylesheet" type="text/css"/>
 	<link  href="/static/css/dm.css" rel="stylesheet" type="text/css"/>
+	<link  href="/static/css/progressBar.css" rel="stylesheet" type="text/css"/>
 	<style>
 		@import "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.4.0/css/font-awesome.css";
 	</style>

daliuge-engine/dlg/manager/web/dm.html

@@ -11,6 +11,7 @@
 	<script src="/static/js/bootbox.min.js"></script>
 	<link href="/static/css/bootstrap.min.css" rel="stylesheet" type="text/css"/>
 	<link  href="/static/css/dm.css" rel="stylesheet" type="text/css"/>
+	<link  href="/static/css/progressBar.css" rel="stylesheet" type="text/css"/>
 	<style>
 		@import "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.4.0/css/font-awesome.css";
 	</style>

daliuge-engine/dlg/manager/web/session.html

@@ -13,6 +13,7 @@
 	<script src="/static/js/dm.js"></script>
 	<link href="/static/css/bootstrap.min.css" rel="stylesheet" type="text/css"/>
 	<link  href="/static/css/session.css" rel="stylesheet" type="text/css"/>
+	<link  href="/static/css/progressBar.css" rel="stylesheet" type="text/css"/>
 	<style>
 		@import "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.4.0/css/font-awesome.css";
 	</style>
@@ -162,7 +163,7 @@
 	d3.select('#pg-progress-bar').remove();
 
 	const heightValue = 300;
-const widthValue = 600;
+	const widthValue = 600;
 	// Set up zoom support
 	var svg = d3.select("svg");
 

daliuge-engine/dlg/manager/web/static/css/dm.css

@@ -80,4 +80,29 @@ color:rgba(0,35,73,1);
 .inactiveLink {
     pointer-events: none;
     cursor: default;
- }
+ }
+
+ tr td:nth-child(n+2),tr th:nth-child(n+2) { 
+  text-align: right;
+}
+
+tr td.actions button{
+  padding: 2px 4px;
+  font-size:16px;  
+}
+
+tr td.actions{
+  padding:9px 12px;
+}
+
+.status svg {
+    width: 100%;
+    height: 20px;
+    position: relative;
+  }
+
+.status {
+  width: 150px;
+  height:30px;
+  overflow: hidden;
+}

daliuge-engine/dlg/manager/web/static/css/progressBar.css

@@ -0,0 +1,48 @@
+
+  .node rect, .node polygon {
+    stroke-width: 2.0px;
+    stroke: #bbb;
+    height:100%;
+  }
+
+  /* DROP states */
+  .node.initialized :first-child, rect.initialized {
+    fill: #ffe;
+  }
+
+  .node.writing :first-child, rect.writing {
+    fill: #ecde7b;
+  }
+
+  .node.completed :first-child, rect.completed {
+    fill: #7bdc7b;
+  }
+
+  .node.expired :first-child, rect.expired {
+    fill: #700000;
+  }
+
+  .node.cancelled :first-child, rect.cancelled {
+    fill: #cccccc;
+  }
+
+  .node.deleted :first-child, rect.deleted {
+    color: #700000;
+  }
+
+  /* AppDROP states */
+  .node.not_run :first-child, rect.not_run {
+    fill: #ffe;
+  }
+
+  .node.running :first-child, rect.running {
+    fill: #ecde7b;
+  }
+
+  .node.finished :first-child, rect.finished {
+    fill: #7bdc7b;
+  }
+
+  .node.error :first-child, rect.error {
+    fill: #e44f33;
+  }

daliuge-engine/dlg/manager/web/static/css/session.css

@@ -52,61 +52,6 @@ svg {
   overflow: hidden;
 }
 
-.node rect, .node polygon {
-  stroke-width: 2.0px;
-  stroke: #bbb;
-}
-
-/* DROP states */
-.node.initialized :first-child, rect.initialized {
-  fill: #ffe;
-}
-
-.node.writing :first-child, rect.writing {
-  fill: #ecde7b;
-}
-
-.node.completed :first-child, rect.completed {
-  fill: #7bdc7b;
-}
-
-.node.expired :first-child, rect.expired {
-  fill: #700000;
-}
-
-.node.cancelled :first-child, rect.cancelled {
-  fill: #cccccc;
-}
-
-.node.deleted :first-child, rect.deleted {
-  color: #700000;
-}
-
-.node.skipped :first-child, rect.skipped {
-  fill: #53c4f6;
-}
-
-/* AppDROP states */
-.node.not_run :first-child, rect.not_run {
-  fill: #ffe;
-}
-
-.node.running :first-child, rect.running {
-  fill: #ecde7b;
-}
-
-.node.finished :first-child, rect.finished {
-  fill: #7bdc7b;
-}
-
-.node.error :first-child, rect.error {
-  fill: #e44f33;
-}
-
-.node.container polygon {
-  stroke-dasharray: 5;
-}
-
 .node g div {
   width: 180px;
   height: 60px;

daliuge-engine/dlg/manager/web/static/js/dm.js

@@ -165,7 +165,7 @@ function loadSessions(serverUrl, tbodyEl, refreshBtn, selectedNode, delay) {
 
 		var sessions = response;
 		sessions.sort(function comp(a,b) {
-			return (a.sessionId < b.sessionId) ? -1 : (a.sessionId > b.sessionId);
+			return (a.sessionId > b.sessionId) ? -1 : (a.sessionId < b.sessionId);
 		});
 		// console.log(sessions[0]);
 		var rows = tbodyEl.selectAll('tr').data(sessions);
@@ -179,19 +179,20 @@ function loadSessions(serverUrl, tbodyEl, refreshBtn, selectedNode, delay) {
 
 		var statusCells = rows.selectAll('td.status').data(function values(s) { return [uniqueSessionStatus(s.status)]; });
 		statusCells.enter().append('td').classed('status', true).text(function(s) { return sessionStatusToString(s); })
-		statusCells.text(function(s) {return sessionStatusToString(s)});
-		statusCells.append('svg');
+		statusCells.text(function(s) {return sessionStatusToString(s)})
 		statusCells.exit().remove()
 
-		//WIP statusbars in dim
+		var sizeCells = rows.selectAll('td.size').data(function values(s) { return [s.size]; });
+		sizeCells.enter().append('td').classed('size', true).text(String)
+		sizeCells.text(String)
+		sizeCells.exit().remove()
+
+		//progressbars in dim
 
 		const width = $('#sessionsTable').find('.status').innerWidth();
 		var graph_update_handler = function(oids, dropSpecs) {};
 
-		console.log(width)
 		var status_update_handler = function(statuses){
-			//doesnt enter
-			console.log("status_counts");
 			var states = ['completed', 'finished',
 						'running', 'writing',
 						'error', 'expired', 'deleted',
@@ -220,7 +221,7 @@ function loadSessions(serverUrl, tbodyEl, refreshBtn, selectedNode, delay) {
 			});
 			var rects = d3.select('#sessionsTable .status svg').selectAll('rect').data(status_counts);
 			rects.enter().append('rect')
-				.style('height', 20).style('width', 0).style('x', 0).style('y', 20)
+				.style('height', 20).style('width', 0).style('x', 0).style('y', 0)
 				.transition().delay(0).duration(500)
 				.style('x', function(d) { return d[0] + 20; })
 				.style('width', function(d) { return d[1]; })
@@ -230,16 +231,6 @@ function loadSessions(serverUrl, tbodyEl, refreshBtn, selectedNode, delay) {
 				.attr('class', function(d) { return states[status_counts.indexOf(d)]; });
 			rects.exit().remove();
 		};
-
-
-		startStatusQuery(serverUrl, sessions[0].sessionId, selectedNode, graph_update_handler,
-			status_update_handler, 1000);
-
-
-		var sizeCells = rows.selectAll('td.size').data(function values(s) { return [s.size]; });
-		sizeCells.enter().append('td').classed('size', true).text(String)
-		sizeCells.text(String)
-		sizeCells.exit().remove()
 
 		statusCells = rows.selectAll('td.details').data(function values(s) { return [s.sessionId]; });
 		statusCells.enter().append('td').classed('details', true)
@@ -263,21 +254,35 @@ function loadSessions(serverUrl, tbodyEl, refreshBtn, selectedNode, delay) {
 				cancel_session(serverUrl, session.sessionId, cancelSessionBtn); 
 			});
 		})
-
+		
 		//update status colours and hide cancel button if finished or cancelled
 		$(".status").each(function(){
 			var currentStatus = $(this).html()
+
 			if(currentStatus==="Cancelled"){
 				$(this).css("color","grey");
 				$(this).parent().find(".actions").find("button").hide();
+				$(this).parent().removeClass("progressRunning")
 			}else if(currentStatus==="Deploying"){
 				$(this).css("color","blue");
+				$(this).parent().removeClass("progressRunning")
 			}
 			else if (currentStatus==="Running") {
-				$(this).css("color","orange");
-			}else{
+				$(this).text("");
+				$(this).parent().find(".actions").find("button").show();
+				$(this).append("<svg>")
+				if(!$(this).parent().hasClass('progressRunning')){
+					startStatusQuery(serverUrl, $(this).parent().find(".id").text(), selectedNode, graph_update_handler,
+						status_update_handler, 1000);
+					$(this).parent().addClass("progressRunning")
+				}
+			}else if (currentStatus==="Finished"){
 				$(this).css("color","#00af28");
 				$(this).parent().find(".actions").find("button").hide();
+				$(this).parent().removeClass("progressRunning")
+			}else{
+				$(this).css("color","purple");
+				$(this).parent().removeClass("progressRunning")
 			}
 		})
 
@@ -428,9 +433,6 @@ function startStatusQuery(serverUrl, sessionId, selectedNode, graph_update_handl
 			if (status == 3 || status == 4 || status == 5) {
 				startGraphStatusUpdates(serverUrl, sessionId, selectedNode, delay,
 				                        status_update_handler);
-
-
-									//WIP this?
 			}
 			else if( status == 0 || status == 1 || status == 2 || status == -1 ){
 				// schedule a new JSON request

daliuge-engine/test/test_drop.py

@@ -36,7 +36,7 @@
     DirectoryContainer, ContainerDROP, InputFiredAppDROP, RDBMSDrop
 from dlg.droputils import DROPWaiterCtx
 from dlg.exceptions import InvalidDropException
-from dlg.apps.simple import NullBarrierApp, SimpleBranch
+from dlg.apps.simple import NullBarrierApp, SimpleBranch, SleepAndCopyApp
 
 
 try:
@@ -132,6 +132,17 @@ def _test_write_withDropType(self, dropType):
         self.assertEqual(a.checksum, test_crc)
         self.assertEqual(cChecksum, test_crc)
 
+    def test_no_write_to_file_drop(self):
+        """Check that FileDrops can be *not* written"""
+        a = FileDROP('a', 'a')
+        b = SleepAndCopyApp('b', 'b')
+        c = InMemoryDROP('c', 'c')
+        a.addConsumer(b)
+        b.addOutput(c)
+        with DROPWaiterCtx(self, c):
+            a.setCompleted()
+        self.assertEqual(droputils.allDropContents(c), b'')
+
     def test_simple_chain(self):
         '''
         Simple test that creates a pipeline-like chain of commands.
@@ -782,12 +793,12 @@ def test_rdbms_drop(self):
         finally:
             os.unlink(dbfile)
 
-class BranchAppDropTests(unittest.TestCase):
+class BranchAppDropTestsBase(object):
     """Tests for the BranchAppDrop class"""
 
     def _simple_branch_with_outputs(self, result, uids):
         a = SimpleBranch(uids[0], uids[0], result=result)
-        b, c = (InMemoryDROP(x, x) for x in uids[1:])
+        b, c = (self.DataDropType(x, x) for x in uids[1:])
         a.addOutput(b)
         a.addOutput(c)
         return a, b, c
@@ -821,7 +832,7 @@ def _test_single_branch_graph(self, result, levels):
 
         for level, uids in zip(range(levels), all_uids):
             if level % 2:
-                x, y = (InMemoryDROP(uid, uid) for uid in uids)
+                x, y = (self.DataDropType(uid, uid) for uid in uids)
                 last_true.addOutput(x)
                 last_false.addOutput(y)
             else:
@@ -903,4 +914,11 @@ def test_multi_branch_more_levels(self):
         """Like test_skipped_propagates, but events propagate more levels"""
         for levels in (3, 4, 5, 6, 7):
             self._test_multi_branch_graph(True, levels)
-            self._test_multi_branch_graph(False, levels)
+            self._test_multi_branch_graph(False, levels)
+
+
+class BranchAppDropTestsWithMemoryDrop(BranchAppDropTestsBase, unittest.TestCase):
+    DataDropType = InMemoryDROP
+
+class BranchAppDropTestsWithFileDrop(BranchAppDropTestsBase, unittest.TestCase):
+    DataDropType = FileDROP