Merge a3e8b47 into 208dd2d

daliuge-common/dlg/common/streams.py

@@ -136,7 +136,7 @@ def read(self, n=-1):
             raise ValueError("n must be positive")
 
         if self.buflen >= n:
-            self.buflen -= n;
+            self.buflen -= n
             data = b''.join(self.buf)
             self.buf = [data[n:]]
             return data[:n]

daliuge-common/dlg/restutils.py

@@ -97,9 +97,10 @@ class RestClient(object):
     The base class for our REST clients
     """
 
-    def __init__(self, host, port, timeout=None):
+    def __init__(self, host, port, url_prefix='', timeout=None):
         self.host = host
         self.port = port
+        self.url_prefix = url_prefix
         self.timeout = timeout
         self._conn = None
         self._resp = None
@@ -160,7 +161,9 @@ def _DELETE(self, url):
     def _request(self, url, method, content=None, headers={}):
 
         # Do the HTTP stuff...
-        logger.debug("Sending %s request to %s:%d%s", method, self.host, self.port, url)
+        url = self.url_prefix + url
+        logger.debug("Sending %s request to %s:%d%s", method,
+                     self.host, self.port, url)
 
         if not common.portIsOpen(self.host, self.port, self.timeout):
             raise RestClientException("Cannot connect to %s:%d after %.2f [s]" % (self.host, self.port, self.timeout))
@@ -177,7 +180,8 @@ def _request(self, url, method, content=None, headers={}):
         if self._resp.status != http.HTTPStatus.OK:
 
             msg = 'Error on remote %s@%s:%s%s (status %d): ' % \
-                  (method, self.host, self.port, url, self._resp.status)
+                  (method, self.host, self.port,
+                   url, self._resp.status)
 
             try:
                 error = json.loads(self._resp.read().decode('utf-8'))
@@ -201,4 +205,4 @@ def _request(self, url, method, content=None, headers={}):
 
         if not self._resp.length:
             return None, None
-        return codecs.getreader('utf-8')(self._resp), self._resp
+        return codecs.getreader('utf-8')(self._resp), self._resp

daliuge-engine/test/manager/test_daemon.py

@@ -63,7 +63,7 @@ def create_daemon(self, *args, **kwargs):
         # in the daemon's hand
         #self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT))
         try:
-            restutils.RestClient('localhost', 9000, 10)._GET('/anything')
+            restutils.RestClient('localhost', 9000, timeout=10)._GET('/anything')
         except restutils.RestClientException:
             # We don't care about the result
             pass

daliuge-engine/test/manager/test_dim.py

@@ -229,10 +229,10 @@ def test_fullRound(self):
         args = ['--port', str(restPort), '-N',hostname, '-qqq']
         dimProcess = tool.start_process('dim', args)
 
-        with testutils.terminating(dimProcess, 10):
+        with testutils.terminating(dimProcess, timeout=10):
 
             # Wait until the REST server becomes alive
-            self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")
+            self.assertTrue(utils.portIsOpen('localhost', restPort, timeout=10), "REST server didn't come up in time")
 
             # The DIM is still empty
             sessions = testutils.get(self, '/sessions', restPort)

daliuge-engine/test/manager/test_mm.py

@@ -224,7 +224,7 @@ def test_fullRound(self):
         with testutils.terminating(mmProcess, 10):
 
             # Wait until the REST server becomes alive
-            self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time")
+            self.assertTrue(utils.portIsOpen('localhost', restPort, timeout=10), "REST server didn't come up in time")
 
             # The DIM is still empty
             sessions = testutils.get(self, '/sessions', restPort)

daliuge-engine/test/manager/test_rest.py

@@ -64,7 +64,7 @@ def tearDown(self):
 
     def test_index(self):
         # Just check that the HTML pages load properly
-        with RestClient(hostname, constants.NODE_DEFAULT_REST_PORT, 10) as c:
+        with RestClient(hostname, constants.NODE_DEFAULT_REST_PORT, timeout=10) as c:
             c._GET('/')
             c._GET('/session')
 

daliuge-engine/test/manager/test_scalability.py

@@ -88,18 +88,19 @@ def test_submit_hugegraph(self):
         n_drops = drops_per_branch * branches * 2 + 1
         graph, completed_uids = create_graph(branches=branches, drops_per_branch=drops_per_branch)
         self.assertEqual(n_drops, len(graph))
-        self._run_graph(graph, completed_uids, 5)
+        self._run_graph(graph, completed_uids, timeout=5)
 
-    def _run_graph(self, graph, completed_uids, timeout):
+    def _run_graph(self, graph, completed_uids, timeout=5):
 
         sessionId = 'lala'
         restPort  = 8888
         args = ['--port', str(restPort), '-N', hostname, '-qq']
 
+        logger.debug("Starting NM on port %d" % restPort)
         c = client.NodeManagerClient(port=restPort)
         dimProcess = tool.start_process('dim', args)
 
-        with testutils.terminating(dimProcess, timeout):
+        with testutils.terminating(dimProcess, timeout=timeout):
             c.create_session(sessionId)
             logger.info("Appending graph")
             c.append_graph(sessionId, graph)

daliuge-translator/dlg/dropmake/web/graph_init.js

@@ -2,19 +2,25 @@
 require([
     "/static/main.js",
 ]);
-require([
-    'https://cdnjs.cloudflare.com/ajax/libs/echarts/5.1.1/echarts.min.js',
-], function (echarts) {
-    var chart = echarts.init(document.getElementById('main'), {renderer:'canvas'});
 
-    // window.onresize = function () {
-    //     chart.resize();
-    // };
+//initial initialisation of graphs
+$(document).ready(function(){
+    var type = "default"
+    var initBtn = false
+    graphInit(type)
+});
 
-    chart.on('click', function (params) {
-        console.log(params, params.data);
-    });
+//event listener for graph buttons
+$(".graphChanger").click(function(){
+    var type = $(this).val()
+    $(this).addClass("active")
+    var initBtn = true
+    graphInit(type)
+})
+
+function graphInit(type){
     $.ajax({
+        //get data
         url: "/pgt_jsonbody?pgt_name="+pgtName,
         dataType: "json",
         type: 'get',
@@ -26,39 +32,82 @@ require([
             }
         },
         success: function(data){
+
             // echarts only displays the name, which is also the key for the edges, thus
             // we need to make sure that the labels are both meaningful and unique.
+            //all nodes and edges
             var graphData = {'nodeDataArray':[], 'linkDataArray':[]};
+
+
+            //partitions 
             var graphDataParts = {'nodeDataArray':[], 'linkDataArray':[]};
             var newElement = {};
             let keyIndex = new Map();
-            var nodeCatgColors = {'Data':'blue', 'Component': 'red'}
+            //shapes and colors for different node types 
+            var nodeCatgColors = {'Data':'#c59a1a', 'Component': '#002349'}
+            var nodeCatgShape = {'Data':'path://M 300 100 L 1000 100 L 800 200 L 100 200 z', 'Component':'rect'}
+            var nodeCount = 0
+            data.nodeDataArray.forEach(element => {
+                nodeCount++
+            })
+            console.log(nodeCount)
+            //pick initial graph depending on node amount
+            if(type==="default"){
+                if(nodeCount<100){
+                    type="graph"
+                }else{
+                    type="sankey"
+                }
+            }
+            //hide other graph option when it doesnt make sense
+            if(nodeCount<50){
+                $(".graphChanger").hide();
+            }else if(nodeCount>150){
+                $(".graphChanger").hide();
+            }
+            console.log(type)
             data.nodeDataArray.forEach(element => {
                 newElement = {};
                 if (!element.hasOwnProperty("isGroup")){
                     // helper map to fix the links later
                     keyIndex.set(element.key, element.text + '-' + element.key.toString());
+                    //data options
                     newElement.name = element.text + '-' + element.key.toString();
-                    newElement.label = {
-                        'rotate': 45,
-                        'fontSize': 10,
-                        'offset': [-20,-20],
-                        'fontWeight' : 700,
-                        'textBorderColor' : 'white',
-                        'textBorderWidth' : 2,
-                        'textBorderType' : 'solid'
 
-                    };
+                    if(type==="sankey"){
+                        newElement.label = {
+                            'rotate': 45,
+                            'fontSize': 10,
+                            'offset': [-20,-20],
+                            'fontWeight' : 700,
+                            'color':'white',
+                            'textBorderColor':'black',
+                            'textBorderWidth' : 2.5,
+                            'textBorderType' : 'solid'
+                        };
+                    }else{
+                        newElement.label = {
+                            'fontSize': 10,
+                            'fontWeight' : 500,
+                            'color':'white',
+                            "position":"inside",
+                            'textBorderColor':'black',
+                            'textBorderWidth' : 2.5,
+                        };
+                    }
+
                     newElement.itemStyle = {};
                     newElement.itemStyle.color = nodeCatgColors[element.category];
+                    newElement.symbol = nodeCatgShape[element.category];
+                    newElement.symbolSize = [60, 30]
                     graphData.nodeDataArray.push(newElement);
                 }
                 else {
                     newElement.name = element.key.toString();
                     graphDataParts.nodeDataArray.push(newElement);                  
                 }
             });
-
+            
             data.linkDataArray.forEach(element => {
                 newElement = {};
                 newElement.source = keyIndex.get(element.from);
@@ -67,40 +116,59 @@ require([
                 graphData.linkDataArray.push(newElement);
             });
 
-            // don't show labels if there are too many nodes. (SETTING?)
+
+            //remove previous graph and active button, if any
+            $("#main").empty();
+            $(".graphChanger").removeClass("active")
+            //add new div depending on type
+            $("#main").append("<div id='"+type+"'></div>")
+            $("#"+type+"Button").addClass("active")
+
+            //re-initialise new graph
+            var chart = echarts.init(document.getElementById(type),null, {renderer:'canvas'});
+            graphSetup(type, chart, graphData, graphDataParts)
+
+        }
+    });
+}
+
+function graphSetup(type, chart, graphData,graphDataParts){
+
+            // don't show labels if there are too many nodes. 
             var show_labels = (graphData.nodeDataArray.length > 350) ? false:true;
 
-            console.log(data.nodeDataArray);
-            console.log(data.linkDataArray);
-            console.log(graphData.nodeDataArray);
-            console.log(graphData.linkDataArray);
-            console.log(keyIndex)
             chart.setOption({
+                layout: "dagre",
                 tooltip: {
                     trigger: 'item',
                     triggerOn: 'mousemove'
                 },
                 animation: true,
                 series: [
                     {
-                        type: 'sankey',
+                        type: type,
+                        symbolSize: 20,
                         roam: true,
+                        zoom:1.15,
                         label: {
-                            show: show_labels
+                            show:show_labels
                         },
+
                         emphasis:{
                             focus: 'adjacency' 
                         },
                         nodeAlign: 'right',
+
                         data: graphData.nodeDataArray,
                         links: graphData.linkDataArray,
                         lineStyle: {
-                            color: 'source',
+                            color: 'grey',
                             curveness: 0.5
                         }
                     }
                 ]
             });
-        }
+    chart.on('click', function (params) {
+        console.log(params, params.data);
     });
-});
+}

daliuge-translator/dlg/dropmake/web/lg_web.py

@@ -306,7 +306,11 @@ def gen_pg():
         return "Must specify DALiUGE manager host"
     try:
         mport = int(request.query.get("dlg_mgr_port"))
-        mgr_client = CompositeManagerClient(host=mhost, port=mport, timeout=30)
+        try:
+            mprefix = request.query.get("dlg_mgr_prefix")
+        except:
+            mprefix = ''
+        mgr_client = CompositeManagerClient(host=mhost, port=mport, url_prefix=mprefix, timeout=30)
         # 1. get a list of nodes
         node_list = mgr_client.nodes()
         # 2. mapping PGTP to resources (node list)
@@ -326,7 +330,7 @@ def gen_pg():
             # mgr_client.deploy_session(ssid, completed_uids=[])
             # print "session deployed"
             # 3. redirect to the master drop manager
-            redirect("http://{0}:{1}/session?sessionId={2}".format(mhost, mport, ssid))
+            redirect("http://{0}:{1}{2}/session?sessionId={3}".format(mhost, mport, mprefix, ssid))
         else:
             response.content_type = 'application/json'
             response.set_header("Content-Disposition", "attachment; filename=%s" % (pgt_id))

daliuge-translator/dlg/dropmake/web/main.js

@@ -31,18 +31,22 @@ $( document ).ready(function() {
 function saveSettings(){
   var newPort = $("#managerPortInput").val();
   var newHost = $("#managerHostInput").val().replace(/\s+/g, '');
+  var newPrefix = $("#managerPrefixInput").val().replace(/\s+/g, '');
   console.log("Host set to:'"+newHost+"'");
   console.log("Port set to:'"+newPort+"'");
+  console.log("Prefix set to:'"+newPrefix+"'");
 
   window.localStorage.setItem("manager_port", newPort);
   window.localStorage.setItem("manager_host", newHost);
+  window.localStorage.setItem("manager_prefix", newPrefix);
   $('#settingsModal').modal('hide')    
 }
 
 function fillOutSettings(){
   //get setting values from local storage
   var manager_host = window.localStorage.getItem("manager_host");
   var manager_port = window.localStorage.getItem("manager_port");
+  var manager_prefix = window.localStorage.getItem("manager_prefix");
 
   //fill settings with saved or default values
   if (!manager_host){
@@ -56,6 +60,11 @@ function fillOutSettings(){
   }else{
     $("#managerPortInput").val(manager_port);
   };
+  if (!manager_prefix){
+    $("#managerPrefixInput").val("");
+  }else{
+    $("#managerPrefixInput").val(manager_prefix);
+  };
 }
 
   function makeJSON() {