From a059c8c94e35bc73f17fe7b02092b0c9852f3925 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Wed, 3 Jul 2019 23:31:36 +0800
Subject: [PATCH 001/194] Rename change team request query to "current-team"
 from "team"

---
 src/WebUI/dotnet/WebPortal/Startup.cs         |  4 +--
 .../Views/Shared/_LoginPartial.cshtml         | 32 +++++++++++--------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Startup.cs b/src/WebUI/dotnet/WebPortal/Startup.cs
index 4a5ff5aa5..518014a0f 100755
--- a/src/WebUI/dotnet/WebPortal/Startup.cs
+++ b/src/WebUI/dotnet/WebPortal/Startup.cs
@@ -489,9 +489,9 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env,
 
             app.Use(async (context, next) =>
             {
-                if (context.Request.Query.ContainsKey("team") && context.Session.GetString("Teams") != null)
+                if (context.Request.Query.ContainsKey("current-team") && context.Session.GetString("Teams") != null)
                 {
-                    var team = context.Request.Query["Team"];
+                    var team = context.Request.Query["current-team"];
                     var teams = JsonConvert.DeserializeObject<string[]>(context.Session.GetString("Teams"));
                     if (Array.Exists(teams, t => t.Equals(team)))
                     {
diff --git a/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml b/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml
index 4ef74eb47..c8b2d082b 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml
@@ -8,18 +8,22 @@
 {
     <ul class="nav navbar-nav navbar-right">
 
-        @if (ViewData["isAuthorized"] != null && (bool)ViewData["isAuthorized"])
+        @if (Context.Session.GetString("Team") != null)
         {
-        <li class="dropdown" id="teams-dropdown">
-            <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">@Context.Session.GetString("Team") <span class="caret"></span></a>
-            <ul class="dropdown-menu">
-                @{ var teams = JsonConvert.DeserializeObject<string[]>(Context.Session.GetString("Teams")); }
-                @foreach (var team in teams)
-                {
-                    <li><a href="#">@team</a></li>
-                }
-            </ul>
-        </li>
+            try
+            {
+                <li class="dropdown" id="teams-dropdown">
+                    <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">@Context.Session.GetString("Team") <span class="caret"></span></a>
+                    <ul class="dropdown-menu">
+                        @{ var teams = JsonConvert.DeserializeObject<string[]>(Context.Session.GetString("Teams")); }
+                        @foreach (var team in teams)
+                        {
+                            <li><a href="#">@team</a></li>
+                        }
+                    </ul>
+                </li>
+            }
+            catch { /* ignored */ }
         }
 
         <li><a asp-area="" asp-controller="Home" asp-action="AccountSettings">Hello, @ViewData["Username"]</a></li>
@@ -30,12 +34,12 @@
             var querystring = location.search.replace(/^\?/, '');
             var queries = querystring.split('&');
             queries = queries.filter(function (query) {
-                return !/^\s*$/.test(query) && !/^team=/i.test(query);
+                return !/^\s*$/.test(query) && !/^current-team=/i.test(query);
             });
             if (queries.length > 0) {
-                querystring = '?' + queries.join('&') + '&team=';
+                querystring = '?' + queries.join('&') + '&current-team=';
             } else {
-                querystring = '?team='
+                querystring = '?current-team=';
             }
 
             $('#teams-dropdown .dropdown-menu a').each(function () {

From 7bc9d1c59b7cd4656f7edf3ef316f7902d0f7331 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 4 Jul 2019 00:00:58 +0800
Subject: [PATCH 002/194] Set authorized clusters in session when using
 password auth

---
 .../WebPortal/Controllers/dlwsController.cs   | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index 160768f45..91d2a71de 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -144,7 +144,7 @@ public async Task<string> GetLog(string jobId)
         private async Task<Tuple<bool, string>> processRestfulAPICommon()
         {
             var passwdLogin = false;
-            if (HttpContext.Request.Query.ContainsKey("Email") && HttpContext.Request.Query.ContainsKey("Key"))
+            if (HttpContext.Request.Query.ContainsKey("Email") && HttpContext.Request.Query.ContainsKey("Key") && HttpContext.Request.Query.ContainsKey("Team"))
             {
 
                 var databases = Startup.Database;
@@ -152,7 +152,8 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
                 var lst = new List<string>();
                 string email = HttpContext.Request.Query["Email"];
                 string password = HttpContext.Request.Query["Key"];
-                bool bFindUser = false; 
+                bool bFindUser = false;
+                var authorizedClusters = new HashSet<string>();
 
                 foreach (var pair in databases)
                 {
@@ -164,6 +165,7 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
 
                     await priorEntrys.ForEachAsync(userEntry =>
                     {
+                        authorizedClusters.Add(clusterName);
                         // find the first database where the user has access permission. 
                         if (!passwdLogin)
                         {
@@ -184,6 +186,11 @@ await priorEntrys.ForEachAsync(userEntry =>
                     }
                     );
                 }
+                if (passwdLogin)
+                {
+                    HttpContext.Session.SetString("AuthorizedClusters", JsonConvert.SerializeObject(authorizedClusters));
+                    HttpContext.Session.SetString("Team", HttpContext.Request.Query["Team"]);
+                }
                 if ( !bFindUser )
                 {
                     return new Tuple<bool, string>(passwdLogin, "Unrecognized Username & Password for RestfulAPI call");
@@ -478,16 +485,16 @@ public async Task<string> PostAsync(TemplateParams templateParams)
         [HttpPost("postJob")]
         public async Task<ActionResult> postJob(TemplateParams templateParams)
         {
-            if (!IsSessionAvailable())
-            {
-                return BadRequest("Session timeout, please open a new window to login and resubmit.");
-            }
-
             var tuple = await processRestfulAPICommon();
             var passwdLogin = tuple.Item1;
             if (!String.IsNullOrEmpty(tuple.Item2))
                 return Content(tuple.Item2);
 
+            if (!IsSessionAvailable() && !passwdLogin)
+            {
+                return BadRequest("Session timeout, please open a new window to login and resubmit.");
+            }
+
 
             if (!User.Identity.IsAuthenticated && !passwdLogin)
             {

From ea4882b3a92204b9a3b9502bc76b48155fbdad10 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 26 Jun 2019 06:58:15 +0000
Subject: [PATCH 003/194] clean useless code and format

---
 src/ClusterManager/job_manager.py | 103 +++++++++---------------------
 1 file changed, 31 insertions(+), 72 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 616922800..ce18bef29 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -74,6 +74,7 @@ def CheckMountPoints(mplist, mp):
             ret = False
     return ret
 
+
 def SubmitRegularJob(job):
     ret = {}
     dataHandler = DataHandler()
@@ -81,43 +82,28 @@ def SubmitRegularJob(job):
     try:
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
 
-        jobParams["pvc_job"] = "jobs-" + jobParams["jobId"]
-        jobParams["pvc_work"] = "work-" + jobParams["jobId"]
-        jobParams["pvc_data"] = "storage-" + jobParams["jobId"]
-
-
         if "jobPath" not in jobParams or len(jobParams["jobPath"].strip()) == 0:
-            dataHandler.SetJobError(jobParams["jobId"],"ERROR: job-path does not exist")
+            dataHandler.SetJobError(jobParams["jobId"], "ERROR: job-path does not exist")
             return False
 
         if "workPath" not in jobParams or len(jobParams["workPath"].strip()) == 0:
-            dataHandler.SetJobError(jobParams["jobId"],"ERROR: work-path does not exist")
+            dataHandler.SetJobError(jobParams["jobId"], "ERROR: work-path does not exist")
             return False
 
-        #if "dataPath" not in jobParams or len(jobParams["dataPath"].strip()) == 0:
-        #    dataHandler.SetJobError(jobParams["jobId"],"ERROR: data-path does not exist")
-        #    return False
-
-
-        jobPath,workPath,dataPath = GetStoragePath(jobParams["jobPath"],jobParams["workPath"],jobParams["dataPath"])
-
+        jobPath, workPath, dataPath = GetStoragePath(jobParams["jobPath"], jobParams["workPath"], jobParams["dataPath"])
 
-        localJobPath = os.path.join(config["storage-mount-path"],jobPath)
+        localJobPath = os.path.join(config["storage-mount-path"], jobPath)
 
         if not os.path.exists(localJobPath):
-            if "userId" in jobParams:
-                mkdirsAsUser(localJobPath,jobParams["userId"])
-                mkdirsAsUser(os.path.join(localJobPath,"models"),jobParams["userId"])
-            else:
-                mkdirsAsUser(localJobPath,"0")
-                mkdirsAsUser(os.path.join(localJobPath,"models"),"0")
+            mkdirsAsUser(localJobPath, jobParams["userId"])
+            mkdirsAsUser(os.path.join(localJobPath, "models"), jobParams["userId"])
 
         jobParams["LaunchCMD"] = ""
         if "cmd" not in jobParams:
             jobParams["cmd"] = ""
 
         if isinstance(jobParams["cmd"], basestring) and not jobParams["cmd"] == "":
-            launchScriptPath = os.path.join(localJobPath,"launch-%s.sh" % jobParams["jobId"])
+            launchScriptPath = os.path.join(localJobPath, "launch-%s.sh" % jobParams["jobId"])
             with open(launchScriptPath, 'w') as f:
                 f.write("#!/bin/bash -x\n")
                 f.write("mkdir /opt; \n")
@@ -129,14 +115,13 @@ def SubmitRegularJob(job):
                 os.system("chown -R %s %s" % (jobParams["userId"], launchScriptPath))
             jobParams["LaunchCMD"] = "[\"bash\", \"/job/launch-%s.sh\"]" % jobParams["jobId"]
 
-
         jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
 
         jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
 
         ENV = Environment(loader=FileSystemLoader("/"))
 
-        jobTempDir = os.path.join(config["root-path"],"Jobs_Templete")
+        jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
         jobTemp = os.path.join(jobTempDir, "RegularJob.yaml.template")
 
         jobParams["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
@@ -144,41 +129,35 @@ def SubmitRegularJob(job):
         jobParams["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
         jobParams["nvidiaDriverPath"] = nvidiaDriverPath
 
-
         jobParams["rest-api"] = config["rest-api"]
 
         if "mountpoints" not in jobParams:
             jobParams["mountpoints"] = []
         for onemount in jobParams["mountpoints"]:
-            onemount["name"] = onemount["containerPath"].replace("/","").lower()
-
-        # mp = {"name":"nvidia-driver","containerPath":"/usr/local/nvidia","hostPath":nvidiaDriverPath, "enabled":True}
-        # if CheckMountPoints(jobParams["mountpoints"],mp):
-        #    jobParams["mountpoints"].append(mp)
+            onemount["name"] = onemount["containerPath"].replace("/", "").lower()
 
-        mp = {"name":"job","containerPath":"/job","hostPath":jobParams["hostjobPath"], "enabled":True}
-        if CheckMountPoints(jobParams["mountpoints"],mp):
+        mp = {"name": "job", "containerPath": "/job", "hostPath": jobParams["hostjobPath"], "enabled": True}
+        if CheckMountPoints(jobParams["mountpoints"], mp):
             jobParams["mountpoints"].append(mp)
 
-        mp = {"name":"work","containerPath":"/work","hostPath":jobParams["hostworkPath"], "enabled":True}
-        if CheckMountPoints(jobParams["mountpoints"],mp):
+        mp = {"name": "work", "containerPath": "/work", "hostPath": jobParams["hostworkPath"], "enabled": True}
+        if CheckMountPoints(jobParams["mountpoints"], mp):
             jobParams["mountpoints"].append(mp)
 
-        mp = {"name":"data","containerPath":"/data","hostPath":jobParams["hostdataPath"], "enabled":True}
-        if CheckMountPoints(jobParams["mountpoints"],mp):
+        mp = {"name": "data", "containerPath": "/data", "hostPath": jobParams["hostdataPath"], "enabled": True}
+        if CheckMountPoints(jobParams["mountpoints"], mp):
             jobParams["mountpoints"].append(mp)
 
         userAlias = getAlias(jobParams["userName"])
         jobParams["user_email"] = jobParams["userName"]
         jobParams["homeFolderHostpath"] = os.path.join(config["storage-mount-path"], GetWorkPath(userAlias))
 
-        if CheckMountPoints(jobParams["mountpoints"],mp):
+        if CheckMountPoints(jobParams["mountpoints"], mp):
             jobParams["mountpoints"].append(mp)
 
         for idx in range(len(jobParams["mountpoints"])):
             if "name" not in jobParams["mountpoints"][idx]:
-                jobParams["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-","")
-
+                jobParams["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
 
         jobParams["pod_ip_range"] = config["pod_ip_range"]
         if "usefreeflow" in config:
@@ -198,7 +177,7 @@ def SubmitRegularJob(job):
             while (i <= end):
                 pod = {}
                 pod["podName"] = jobParams["jobId"]+"-pod-"+str(c)
-                pod["envs"] = [{"name":jobParams["hyperparametername"],"value":i}]
+                pod["envs"] = [{"name": jobParams["hyperparametername"], "value":i}]
                 i += step
                 c += 1
                 pods.append(pod)
@@ -212,44 +191,26 @@ def SubmitRegularJob(job):
             jobParams["env"] = []
         jobParams["commonenv"] = copy.copy(jobParams["env"])
 
-
         for pod in pods:
             jobParams["podName"] = pod["podName"]
             jobParams["env"] = jobParams["commonenv"] + pod["envs"]
 
             if "kube_custom_scheduler" in config and config["kube_custom_scheduler"]:
                 container = {}
-                container["requests"] = {"alpha.gpu/numgpu" : int(jobParams["resourcegpu"])}
+                container["requests"] = {"alpha.gpu/numgpu": int(jobParams["resourcegpu"])}
                 podInfo = {}
                 podInfo["podname"] = jobParams["podName"]
                 if "useGPUTopology" in jobParams and jobParams["useGPUTopology"]:
-                    # add topology constraints explicitly - for testing
-                    # if (jobParams["resourcegpu"] >= 2):
-                    #     # both cards in same inner group
-                    #     container["requests"]["alpha/grpresource/gpugrp1/0/gpugrp0/0/gpu/0/cards"] = 1
-                    #     container["requests"]["alpha/grpresource/gpugrp1/0/gpugrp0/0/gpu/1/cards"] = 1
-                    # if (jobParams["resourcegpu"] >= 3):
-                    #     container["requests"]["alpha/grpresource/gpugrp1/0/gpugrp0/1/gpu/2/cards"] = 1
-                    # if (jobParams["resourcegpu"] >= 4):
-                    #     container["requests"]["alpha/grpresource/gpugrp1/0/gpugrp0/1/gpu/3/cards"] = 1
-                    # if (jobParams["resourcegpu"] >= 5):
-                    #     container["requests"]["alpha/grpresource/gpugrp1/1/gpugrp0/2/gpu/4/cards"] = 1
-                    # if (jobParams["resourcegpu"] >= 6):
-                    #     container["requests"]["alpha/grpresource/gpugrp1/1/gpugrp0/2/gpu/5/cards"] = 1
-                    # if (jobParams["resourcegpu"] >= 7):
-                    #     container["requests"]["alpha/grpresource/gpugrp1/1/gpugrp0/3/gpu/6/cards"] = 1
-                    # if (jobParams["resourcegpu"] >= 8):
-                    #     container["requests"]["alpha/grpresource/gpugrp1/1/gpugrp0/3/gpu/7/cards"] = 1
-                    podInfo["requests"] = {"alpha.gpu/gpu-generate-topology" : 1}
+                    podInfo["requests"] = {"alpha.gpu/gpu-generate-topology": 1}
                 else:
                     # for cases when desired topology is explictly given or not desired
-                    podInfo["requests"] = {"alpha.gpu/gpu-generate-topology" : 0}
-                podInfo["runningcontainer"] = {jobParams["podName"] : container}
+                    podInfo["requests"] = {"alpha.gpu/gpu-generate-topology": 0}
+                podInfo["runningcontainer"] = {jobParams["podName"]: container}
 
                 if "annotations" not in jobParams:
                     jobParams["annotations"] = {}
                 jobParams["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
-                jobParams["resourcegpu"] = 0 # gpu requests specified through annotation
+                jobParams["resourcegpu"] = 0  # gpu requests specified through annotation
 
                 if "gpuType" in jobParams:
                     if "nodeSelector" not in jobParams:
@@ -279,20 +240,18 @@ def SubmitRegularJob(job):
             f.write(jobDescription)
 
         output = k8sUtils.kubectl_create(jobDescriptionPath)
-        logging.info("Submitted job %s to k8s, returned with status %s" %(job["jobId"], output))
+        logging.info("Submitted job %s to k8s, returned with status %s" % (job["jobId"], output))
 
         ret["output"] = output
 
         ret["jobId"] = jobParams["jobId"]
 
-
         if "userName" not in jobParams:
             jobParams["userName"] = ""
 
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobStatus","scheduling")
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobDescriptionPath",jobParams["jobDescriptionPath"])
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobDescription",base64.b64encode(jobDescription))
-
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", jobParams["jobDescriptionPath"])
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(jobDescription))
 
         jobMeta = {}
         jobMeta["jobDescriptionPath"] = jobParams["jobDescriptionPath"]
@@ -302,14 +261,14 @@ def SubmitRegularJob(job):
         jobMeta["LaunchCMD"] = jobParams["LaunchCMD"]
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobMeta",jobMetaStr)
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobMeta", jobMetaStr)
     except Exception as e:
         print(e)
         ret["error"] = str(e)
         retries = dataHandler.AddandGetJobRetries(jobParams["jobId"])
         if retries >= 5:
-            dataHandler.UpdateJobTextField(jobParams["jobId"],"jobStatus","error")
-            dataHandler.UpdateJobTextField(jobParams["jobId"],"errorMsg","Cannot submit job!" + str(e))
+            dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "error")
+            dataHandler.UpdateJobTextField(jobParams["jobId"], "errorMsg", "Cannot submit job!" + str(e))
     dataHandler.Close()
     return ret
 

From 7583fcb82fac12a6bed79f2fe32d66c877266b70 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 26 Jun 2019 09:50:23 +0000
Subject: [PATCH 004/194] add Job JobSchema and test

---
 src/ClusterManager/job.py      |  75 ++++++++++++++++++++++++
 src/ClusterManager/test_job.py | 104 +++++++++++++++++++++++++++++++++
 2 files changed, 179 insertions(+)
 create mode 100644 src/ClusterManager/job.py
 create mode 100644 src/ClusterManager/test_job.py

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
new file mode 100644
index 000000000..bedc2a86c
--- /dev/null
+++ b/src/ClusterManager/job.py
@@ -0,0 +1,75 @@
+import sys
+import os
+from datetime import date
+from marshmallow import Schema, fields, pprint, post_load, validate
+
+import logging
+import logging.config
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+
+# TODO remove it latter
+def create_log(logdir='.'):
+    if not os.path.exists(logdir):
+        os.system("mkdir -p " + logdir)
+    with open('logging.yaml') as f:
+        logging_config = yaml.load(f)
+        f.close()
+        logging_config["handlers"]["file"]["filename"] = logdir + "/jobmanager.log"
+        logging.config.dictConfig(logging_config)
+
+
+class Job:
+    def __init__(self, job_id, mountpoints=None):
+        self.job_id = job_id
+        self.mountpoints = mountpoints
+
+    def add_mountpoints(self, mountpoint):
+        '''
+        1. Silently skip if the name/hostPath/containerPath duplicates with an existing one.
+        2. Name would be normalized.
+
+        Mountpoint example:
+            {
+            "enabled":true,
+            "containerPath":"/home/username",
+            "hostPath":"/dlwsdata/work/username",
+            "name":"homefolder"
+            }
+        '''
+        if mountpoint is None:
+            return
+        if self.mountpoints is None:
+            self.mountpoints = []
+
+        # add each items in the list one by one
+        if isinstance(mountpoint, list):
+            for m in mountpoint:
+                self.add_mountpoints(m)
+                return
+
+        # only allow alphanumeric in "name"
+        if "name" not in mountpoint or mountpoint["name"] == "":
+            mountpoint["name"] = mountpoint["containerPath"]
+        mountpoint["name"] = ''.join(c for c in mountpoint["name"] if c.isalnum())
+
+        # skip dulicate entry
+        for item in self.mountpoints:
+            if item["name"] == mountpoint["name"] or item["containerPath"] == mountpoint["containerPath"] or item["hostPath"] == mountpoint["hostPath"]:
+                logging.warn("Duplciate mountpoint: %s" % mountpoint)
+                return
+
+        self.mountpoints.append(mountpoint)
+
+
+class JobSchema(Schema):
+    job_id = fields.String(required=True,
+                           # Correctly mappging the name
+                           dump_to="jobId", load_from="jobId",
+                           # We use the id in k8s object, it should be a `DNS-1035 label`
+                           validate=validate.Regexp(r'[a-z]([-a-z0-9]*[a-z0-9])?', 0))
+    mountpoints = fields.Dict(required=False)
+
+    @post_load
+    def make_user(self, data, **kwargs):
+        return Job(**data)
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
new file mode 100644
index 000000000..427d51341
--- /dev/null
+++ b/src/ClusterManager/test_job.py
@@ -0,0 +1,104 @@
+import unittest
+from job import Job, JobSchema
+
+
+class TestJobSchema(unittest.TestCase):
+    def test_loads(self):
+        job_json = '{"jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c"}'
+
+        job, errors = JobSchema().loads(job_json)
+        self.assertFalse(errors)
+        self.assertEqual(job.job_id, "ce7dca49-28df-450a-a03b-51b9c2ecc69c")
+
+        job, _ = JobSchema().load({"jobId": "first_job"})
+        self.assertEqual(job.job_id, "first_job")
+
+    def test_job_id_schema(self):
+        # regex used for validation is '[a-z]([-a-z0-9]*[a-z0-9])?'
+        job, errors = JobSchema().load({"jobId": "8first_job"})
+        self.assertEqual(errors, {'jobId': ['String does not match expected pattern.']})
+
+        job, errors = JobSchema().load({"jobId": "8First_job"})
+        self.assertEqual(errors, {'jobId': ['String does not match expected pattern.']})
+
+        job, errors = JobSchema().load({"jobId": "first-job"})
+        self.assertFalse(errors)
+
+    def test_dump(self):
+        job = Job(job_id="test_job")
+
+        result, errors = JobSchema().dump(job)
+
+        self.assertFalse(errors)
+        self.assertEqual(result["jobId"], "test_job")
+
+
+class TestJob(unittest.TestCase):
+
+    def test_add_mountpoints_with_none(self):
+        job, errors = JobSchema().load({"jobId": "first_job"})
+        self.assertFalse(errors)
+
+        job.add_mountpoints(None)
+
+    def test_add_mountpoints_without_name(self):
+        job, errors = JobSchema().load({"jobId": "first_job"})
+        self.assertFalse(errors)
+
+        # add one mountpoint without "name"
+        mountpoint1 = {
+            "enabled": True,
+            "containerPath": "/home/username",
+            "hostPath": "/dlwsdata/work/username",
+        }
+        job.add_mountpoints(mountpoint1)
+        self.assertEqual(1, len(job.mountpoints))
+
+    def test_add_mountpoints(self):
+        job, errors = JobSchema().load({"jobId": "first_job"})
+        self.assertFalse(errors)
+
+        # add one mountpoint
+        mountpoint1 = {
+            "enabled": True,
+            "containerPath": "/home/username",
+            "hostPath": "/dlwsdata/work/username",
+            "name": "homefolder"
+        }
+        job.add_mountpoints(mountpoint1)
+        self.assertEqual(1, len(job.mountpoints))
+
+        # would silently skip
+        job.add_mountpoints(mountpoint1)
+        self.assertEqual(1, len(job.mountpoints))
+
+        # name would be normalized, only allow alphanumeric, so it would be a duplicate
+        mountpoint1a = {
+            "enabled": True,
+            "containerPath": "/home/path",
+            "hostPath": "/dlwsdata/work/path",
+            "name": "homefolder-"
+        }
+        job.add_mountpoints(mountpoint1a)
+        self.assertEqual(1, len(job.mountpoints))
+
+        # add another mountpoint
+        mountpoint2 = {
+            "enabled": True,
+            "containerPath": "/home/path1",
+            "hostPath": "/dlwsdata/work/path1",
+            "name": "homepath1"
+        }
+        job.add_mountpoints(mountpoint2)
+        self.assertEqual(2, len(job.mountpoints))
+
+        # add a list
+        mountpoints = [{
+            "enabled": True,
+            "containerPath": "/home/path2",
+            "hostPath": "/dlwsdata/work/path2",
+            "name": "homepath2"
+        }]
+        job.add_mountpoints(mountpoints)
+        self.assertEqual(3, len(job.mountpoints))
+

From b5a2f3c85414802cf0786fd23793376afbcc72aa Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 02:48:39 +0000
Subject: [PATCH 005/194] refine validation message

---
 src/ClusterManager/job.py      | 2 +-
 src/ClusterManager/test_job.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index bedc2a86c..22e08088f 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -67,7 +67,7 @@ class JobSchema(Schema):
                            # Correctly mappging the name
                            dump_to="jobId", load_from="jobId",
                            # We use the id in k8s object, it should be a `DNS-1035 label`
-                           validate=validate.Regexp(r'[a-z]([-a-z0-9]*[a-z0-9])?', 0))
+                           validate=validate.Regexp(r'[a-z]([-a-z0-9]*[a-z0-9])?', error="'{input}' does not match expected pattern {regex}."))
     mountpoints = fields.Dict(required=False)
 
     @post_load
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 427d51341..1cd7cd3ee 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -16,10 +16,10 @@ def test_loads(self):
     def test_job_id_schema(self):
         # regex used for validation is '[a-z]([-a-z0-9]*[a-z0-9])?'
         job, errors = JobSchema().load({"jobId": "8first_job"})
-        self.assertEqual(errors, {'jobId': ['String does not match expected pattern.']})
+        self.assertTrue("jobId" in errors)
 
-        job, errors = JobSchema().load({"jobId": "8First_job"})
-        self.assertEqual(errors, {'jobId': ['String does not match expected pattern.']})
+        job, errors = JobSchema().load({"jobId": "First_job"})
+        self.assertTrue("jobId" in errors)
 
         job, errors = JobSchema().load({"jobId": "first-job"})
         self.assertFalse(errors)

From e032ed8c2c84ac2700b62bdc569cbf8f0d9907d2 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 03:26:23 +0000
Subject: [PATCH 006/194] fix job_id schema

---
 src/ClusterManager/job.py      |  7 +++++--
 src/ClusterManager/test_job.py | 13 ++++++-------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 22e08088f..0e5cf5070 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -66,8 +66,11 @@ class JobSchema(Schema):
     job_id = fields.String(required=True,
                            # Correctly mappging the name
                            dump_to="jobId", load_from="jobId",
-                           # We use the id in k8s object, it should be a `DNS-1035 label`
-                           validate=validate.Regexp(r'[a-z]([-a-z0-9]*[a-z0-9])?', error="'{input}' does not match expected pattern {regex}."))
+                           # We use the id as "name" in k8s object.
+                           # By convention, the "names" of Kubernetes resources should be
+                           #  up to maximum length of 253 characters and consist of lower case
+                           # alphanumeric characters, -, and ., but certain resources have more specific restrictions.
+                           validate=validate.Regexp(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$', 0, error="'{input}' does not match expected pattern {regex}."))
     mountpoints = fields.Dict(required=False)
 
     @post_load
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 1cd7cd3ee..1b767ec15 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -10,12 +10,11 @@ def test_loads(self):
         self.assertFalse(errors)
         self.assertEqual(job.job_id, "ce7dca49-28df-450a-a03b-51b9c2ecc69c")
 
-        job, _ = JobSchema().load({"jobId": "first_job"})
-        self.assertEqual(job.job_id, "first_job")
+        job, _ = JobSchema().load({"jobId": "first-job"})
+        self.assertEqual(job.job_id, "first-job")
 
     def test_job_id_schema(self):
-        # regex used for validation is '[a-z]([-a-z0-9]*[a-z0-9])?'
-        job, errors = JobSchema().load({"jobId": "8first_job"})
+        job, errors = JobSchema().load({"jobId": "first job"})
         self.assertTrue("jobId" in errors)
 
         job, errors = JobSchema().load({"jobId": "First_job"})
@@ -36,13 +35,13 @@ def test_dump(self):
 class TestJob(unittest.TestCase):
 
     def test_add_mountpoints_with_none(self):
-        job, errors = JobSchema().load({"jobId": "first_job"})
+        job, errors = JobSchema().load({"jobId": "first-job"})
         self.assertFalse(errors)
 
         job.add_mountpoints(None)
 
     def test_add_mountpoints_without_name(self):
-        job, errors = JobSchema().load({"jobId": "first_job"})
+        job, errors = JobSchema().load({"jobId": "first-job"})
         self.assertFalse(errors)
 
         # add one mountpoint without "name"
@@ -55,7 +54,7 @@ def test_add_mountpoints_without_name(self):
         self.assertEqual(1, len(job.mountpoints))
 
     def test_add_mountpoints(self):
-        job, errors = JobSchema().load({"jobId": "first_job"})
+        job, errors = JobSchema().load({"jobId": "first-job"})
         self.assertFalse(errors)
 
         # add one mountpoint

From 07491ea0699730f044010f16d2b370558e70cae8 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 03:29:33 +0000
Subject: [PATCH 007/194] use Job object in job_manager

---
 src/ClusterManager/job_manager.py | 35 +++++++++++++------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index ce18bef29..0647fd1c8 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -35,6 +35,7 @@
 
 import logging
 import logging.config
+from job import Job, JobSchema
 
 
 nvidiaDriverPath = config["nvidiaDriverPath"]
@@ -80,6 +81,10 @@ def SubmitRegularJob(job):
     dataHandler = DataHandler()
 
     try:
+        job_object, errors = JobSchema().load(job)
+        # TODO assert job_object is a Job
+        assert(isinstance(job_object, Job))
+
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
 
         if "jobPath" not in jobParams or len(jobParams["jobPath"].strip()) == 0:
@@ -131,34 +136,24 @@ def SubmitRegularJob(job):
 
         jobParams["rest-api"] = config["rest-api"]
 
-        if "mountpoints" not in jobParams:
-            jobParams["mountpoints"] = []
-        for onemount in jobParams["mountpoints"]:
-            onemount["name"] = onemount["containerPath"].replace("/", "").lower()
+        if "mountpoints" in jobParams:
+            job_object.add_mountpoints(jobParams["mountpoints"])
 
         mp = {"name": "job", "containerPath": "/job", "hostPath": jobParams["hostjobPath"], "enabled": True}
-        if CheckMountPoints(jobParams["mountpoints"], mp):
-            jobParams["mountpoints"].append(mp)
+        job_object.add_mountpoints(mp)
 
         mp = {"name": "work", "containerPath": "/work", "hostPath": jobParams["hostworkPath"], "enabled": True}
-        if CheckMountPoints(jobParams["mountpoints"], mp):
-            jobParams["mountpoints"].append(mp)
+        job_object.add_mountpoints(mp)
 
         mp = {"name": "data", "containerPath": "/data", "hostPath": jobParams["hostdataPath"], "enabled": True}
-        if CheckMountPoints(jobParams["mountpoints"], mp):
-            jobParams["mountpoints"].append(mp)
+        job_object.add_mountpoints(mp)
+
+        jobParams["mountpoints"] = job_object.mountpoints
 
         userAlias = getAlias(jobParams["userName"])
         jobParams["user_email"] = jobParams["userName"]
         jobParams["homeFolderHostpath"] = os.path.join(config["storage-mount-path"], GetWorkPath(userAlias))
 
-        if CheckMountPoints(jobParams["mountpoints"], mp):
-            jobParams["mountpoints"].append(mp)
-
-        for idx in range(len(jobParams["mountpoints"])):
-            if "name" not in jobParams["mountpoints"][idx]:
-                jobParams["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
-
         jobParams["pod_ip_range"] = config["pod_ip_range"]
         if "usefreeflow" in config:
             jobParams["usefreeflow"] = config["usefreeflow"]
@@ -265,10 +260,10 @@ def SubmitRegularJob(job):
     except Exception as e:
         print(e)
         ret["error"] = str(e)
-        retries = dataHandler.AddandGetJobRetries(jobParams["jobId"])
+        retries = dataHandler.AddandGetJobRetries(job["jobId"])
         if retries >= 5:
-            dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "error")
-            dataHandler.UpdateJobTextField(jobParams["jobId"], "errorMsg", "Cannot submit job!" + str(e))
+            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
+            dataHandler.UpdateJobTextField(job["jobId"], "errorMsg", "Cannot submit job!" + str(e))
     dataHandler.Close()
     return ret
 

From d415dac91b9f973b49aa51d6a76cc5f0a9fa2720 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 04:40:01 +0000
Subject: [PATCH 008/194] Job: add field "email"

---
 src/ClusterManager/job.py      | 10 +++++--
 src/ClusterManager/test_job.py | 50 +++++++++++++++++++++-------------
 2 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 0e5cf5070..fc328cf2a 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -20,8 +20,9 @@ def create_log(logdir='.'):
 
 
 class Job:
-    def __init__(self, job_id, mountpoints=None):
+    def __init__(self, job_id, email, mountpoints=None):
         self.job_id = job_id
+        self.email = email
         self.mountpoints = mountpoints
 
     def add_mountpoints(self, mountpoint):
@@ -69,8 +70,11 @@ class JobSchema(Schema):
                            # We use the id as "name" in k8s object.
                            # By convention, the "names" of Kubernetes resources should be
                            #  up to maximum length of 253 characters and consist of lower case
-                           # alphanumeric characters, -, and ., but certain resources have more specific restrictions.
-                           validate=validate.Regexp(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$', 0, error="'{input}' does not match expected pattern {regex}."))
+                           # alphanumeric characters, -, and .,
+                           # but certain resources have more specific restrictions.
+                           validate=validate.Regexp(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$',
+                                                    error="'{input}' does not match expected pattern {regex}."))
+    email = fields.Email(required=True, dump_to="userName", load_from="userName")
     mountpoints = fields.Dict(required=False)
 
     @post_load
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 1b767ec15..af636f70f 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -1,48 +1,62 @@
 import unittest
+import json
 from job import Job, JobSchema
 
+VALID_JOB_ATTRIBUTES = {
+    "jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c",
+    "userName": "user@foo.com"
+}
+
 
 class TestJobSchema(unittest.TestCase):
+
     def test_loads(self):
-        job_json = '{"jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c"}'
+        job_json = json.dumps(VALID_JOB_ATTRIBUTES)
 
         job, errors = JobSchema().loads(job_json)
         self.assertFalse(errors)
-        self.assertEqual(job.job_id, "ce7dca49-28df-450a-a03b-51b9c2ecc69c")
-
-        job, _ = JobSchema().load({"jobId": "first-job"})
-        self.assertEqual(job.job_id, "first-job")
+        self.assertEqual(job.job_id, VALID_JOB_ATTRIBUTES["jobId"])
+        self.assertEqual(job.email, VALID_JOB_ATTRIBUTES["userName"])
 
     def test_job_id_schema(self):
-        job, errors = JobSchema().load({"jobId": "first job"})
-        self.assertTrue("jobId" in errors)
+        job, errors = JobSchema().load(VALID_JOB_ATTRIBUTES)
+        self.assertFalse(errors)
 
-        job, errors = JobSchema().load({"jobId": "First_job"})
+        # uppercase
+        attrs = VALID_JOB_ATTRIBUTES.copy()
+        attrs.update({"jobId": "First-job"})
+        job, errors = JobSchema().load(attrs)
         self.assertTrue("jobId" in errors)
 
-        job, errors = JobSchema().load({"jobId": "first-job"})
-        self.assertFalse(errors)
+        # space
+        attrs = VALID_JOB_ATTRIBUTES.copy()
+        attrs.update({"jobId": "first job"})
+        job, errors = JobSchema().load(attrs)
+        self.assertTrue("jobId" in errors)
 
     def test_dump(self):
-        job = Job(job_id="test_job")
+        job = Job(job_id="test-job", email="user@foo.com")
 
         result, errors = JobSchema().dump(job)
 
         self.assertFalse(errors)
-        self.assertEqual(result["jobId"], "test_job")
+        self.assertEqual(result["jobId"], "test-job")
+        self.assertEqual(result["userName"], "user@foo.com")
 
 
 class TestJob(unittest.TestCase):
 
-    def test_add_mountpoints_with_none(self):
-        job, errors = JobSchema().load({"jobId": "first-job"})
+    def create_a_job(self):
+        job, errors = JobSchema().load(VALID_JOB_ATTRIBUTES)
         self.assertFalse(errors)
+        return job
 
+    def test_add_mountpoints_with_none(self):
+        job = self.create_a_job()
         job.add_mountpoints(None)
 
     def test_add_mountpoints_without_name(self):
-        job, errors = JobSchema().load({"jobId": "first-job"})
-        self.assertFalse(errors)
+        job = self.create_a_job()
 
         # add one mountpoint without "name"
         mountpoint1 = {
@@ -54,8 +68,7 @@ def test_add_mountpoints_without_name(self):
         self.assertEqual(1, len(job.mountpoints))
 
     def test_add_mountpoints(self):
-        job, errors = JobSchema().load({"jobId": "first-job"})
-        self.assertFalse(errors)
+        job = self.create_a_job()
 
         # add one mountpoint
         mountpoint1 = {
@@ -100,4 +113,3 @@ def test_add_mountpoints(self):
         }]
         job.add_mountpoints(mountpoints)
         self.assertEqual(3, len(job.mountpoints))
-

From a1b54a735ecf3550178923aaff307673abe85b5a Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 06:33:20 +0000
Subject: [PATCH 009/194] Job: add attr "cluster", a dict of cluster config

---
 src/ClusterManager/job.py      | 16 ++++++++++++++--
 src/ClusterManager/test_job.py | 19 +++++++++++++++++--
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index fc328cf2a..5d704d5b2 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -6,7 +6,6 @@
 import logging
 import logging.config
 
-sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
 
 # TODO remove it latter
 def create_log(logdir='.'):
@@ -20,7 +19,13 @@ def create_log(logdir='.'):
 
 
 class Job:
-    def __init__(self, job_id, email, mountpoints=None):
+    def __init__(self, cluster, job_id, email, mountpoints=None):
+        """
+        job_id: an unique string for the job.
+        email: user's email.
+        cluster: cluster config.
+        """
+        self.cluster = cluster
         self.job_id = job_id
         self.email = email
         self.mountpoints = mountpoints
@@ -62,8 +67,15 @@ def add_mountpoints(self, mountpoint):
 
         self.mountpoints.append(mountpoint)
 
+    def get_alias(self):
+        return self.email.split("@")[0].strip()
+
+    def get_homefolder_hostpath(self):
+        return os.path.join(self.cluster["storage-mount-path"], "work/", self.get_alias())
+
 
 class JobSchema(Schema):
+    cluster = fields.Dict(required=True)
     job_id = fields.String(required=True,
                            # Correctly mappging the name
                            dump_to="jobId", load_from="jobId",
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index af636f70f..3825aa8fc 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -1,10 +1,17 @@
 import unittest
 import json
+import sys
+import os
 from job import Job, JobSchema
 
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+from config import config
+
+
 VALID_JOB_ATTRIBUTES = {
+    "cluster": config,
     "jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c",
-    "userName": "user@foo.com"
+    "userName": "user@foo.com",
 }
 
 
@@ -35,7 +42,11 @@ def test_job_id_schema(self):
         self.assertTrue("jobId" in errors)
 
     def test_dump(self):
-        job = Job(job_id="test-job", email="user@foo.com")
+        job = Job(
+            cluster=config,
+            job_id="test-job",
+            email="user@foo.com"
+        )
 
         result, errors = JobSchema().dump(job)
 
@@ -113,3 +124,7 @@ def test_add_mountpoints(self):
         }]
         job.add_mountpoints(mountpoints)
         self.assertEqual(3, len(job.mountpoints))
+
+    def test_get_homefolder_hostpath(self):
+        job = self.create_a_job()
+        self.assertEqual("/dlwsdata/work/user", job.get_homefolder_hostpath())

From 86147e32fb046525139a490671fed68ef4958d1f Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 07:28:12 +0000
Subject: [PATCH 010/194] refine mountpoints

---
 src/ClusterManager/job.py      | 36 +++++++++++++++++++++++++++++++++-
 src/ClusterManager/test_job.py | 22 +++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 5d704d5b2..0b2d71018 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -19,16 +19,29 @@ def create_log(logdir='.'):
 
 
 class Job:
-    def __init__(self, cluster, job_id, email, mountpoints=None):
+    def __init__(self,
+                 cluster,
+                 job_id,
+                 email,
+                 mountpoints=None,
+                 job_path="",
+                 work_path="",
+                 data_path="",
+                 ):
         """
         job_id: an unique string for the job.
         email: user's email.
         cluster: cluster config.
+        job_path: relative path, on shared storage, for example "user_alias/jobs/date/job_id".
+        work_path: relative path, on shared storage, for example "user_alias".
         """
         self.cluster = cluster
         self.job_id = job_id
         self.email = email
         self.mountpoints = mountpoints
+        self.job_path = job_path
+        self.work_path = work_path
+        self.data_path = data_path
 
     def add_mountpoints(self, mountpoint):
         '''
@@ -73,6 +86,24 @@ def get_alias(self):
     def get_homefolder_hostpath(self):
         return os.path.join(self.cluster["storage-mount-path"], "work/", self.get_alias())
 
+    def get_job_path_hostpath(self):
+        return os.path.join(self.cluster["storage-mount-path"], "work/", self.job_path)
+
+    def job_path_mountpoint(self):
+        assert(len(self.job_path) > 0)
+        job_host_path = os.path.join(self.cluster["storage-mount-path"], "work", self.job_path)
+        return {"name": "job", "containerPath": "/job", "hostPath": job_host_path, "enabled": True}
+
+    def work_path_mountpoint(self):
+        assert(len(self.work_path) > 0)
+        work_host_path = os.path.join(self.cluster["storage-mount-path"], "work", self.work_path)
+        return {"name": "work", "containerPath": "/work", "hostPath": work_host_path, "enabled": True}
+
+    def data_path_mountpoint(self):
+        assert(self.data_path is not None)
+        data_host_path = os.path.join(self.cluster["storage-mount-path"], "storage", self.data_path)
+        return {"name": "data", "containerPath": "/data", "hostPath": data_host_path, "enabled": True}
+
 
 class JobSchema(Schema):
     cluster = fields.Dict(required=True)
@@ -88,6 +119,9 @@ class JobSchema(Schema):
                                                     error="'{input}' does not match expected pattern {regex}."))
     email = fields.Email(required=True, dump_to="userName", load_from="userName")
     mountpoints = fields.Dict(required=False)
+    job_path = fields.String(required=False, dump_to="jobPath", load_from="jobPath")
+    work_path = fields.String(required=False, dump_to="workPath", load_from="workPath")
+    data_path = fields.String(required=False, dump_to="dataPath", load_from="dataPath")
 
     @post_load
     def make_user(self, data, **kwargs):
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 3825aa8fc..0cbf4b8f4 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -12,6 +12,7 @@
     "cluster": config,
     "jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c",
     "userName": "user@foo.com",
+    "jobPath": "user_alias/jobs/date/job_id"
 }
 
 
@@ -128,3 +129,24 @@ def test_add_mountpoints(self):
     def test_get_homefolder_hostpath(self):
         job = self.create_a_job()
         self.assertEqual("/dlwsdata/work/user", job.get_homefolder_hostpath())
+
+    def test_get_job_path_hostpath(self):
+        job = self.create_a_job()
+        self.assertEqual("user_alias/jobs/date/job_id", job.job_path)
+        self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.get_job_path_hostpath())
+
+    def test_job_work_data_mountpoints(self):
+        job = self.create_a_job()
+
+        job.job_path = "user_alias/jobs/date/job_id"
+        job.work_path = "user_alias"
+        job.data_path = ""
+
+        self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.job_path_mountpoint()["hostPath"])
+        self.assertEqual("/dlwsdata/work/user_alias", job.work_path_mountpoint()["hostPath"])
+        self.assertEqual("/dlwsdata/storage/xx", job.data_path_mountpoint()["hostPath"])
+
+        job.add_mountpoints(job.job_path_mountpoint())
+        job.add_mountpoints(job.work_path_mountpoint())
+        job.add_mountpoints(job.data_path_mountpoint())
+        self.assertEquals(3, len(job.mountpoints()))

From 6e4814b55c2307dbf9035a2d7ae9ce369c79a6fe Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 07:56:16 +0000
Subject: [PATCH 011/194] refine job_manager

---
 src/ClusterManager/job_manager.py | 54 +++++++++++++------------------
 1 file changed, 22 insertions(+), 32 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 0647fd1c8..10c6dd839 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -7,6 +7,7 @@
 import sys
 import datetime
 import copy
+import traceback
 
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../storage"))
@@ -81,24 +82,26 @@ def SubmitRegularJob(job):
     dataHandler = DataHandler()
 
     try:
+        job["cluster"] = config
         job_object, errors = JobSchema().load(job)
         # TODO assert job_object is a Job
         assert(isinstance(job_object, Job))
 
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
-
-        if "jobPath" not in jobParams or len(jobParams["jobPath"].strip()) == 0:
-            dataHandler.SetJobError(jobParams["jobId"], "ERROR: job-path does not exist")
-            return False
-
-        if "workPath" not in jobParams or len(jobParams["workPath"].strip()) == 0:
-            dataHandler.SetJobError(jobParams["jobId"], "ERROR: work-path does not exist")
+        if any(required_field not in jobParams for required_field in ["jobPath", "workPath", "cmd"]):
+            dataHandler.SetJobError(jobParams["jobId"], "ERROR: required fileds missing in jobParams.")
             return False
 
-        jobPath, workPath, dataPath = GetStoragePath(jobParams["jobPath"], jobParams["workPath"], jobParams["dataPath"])
-
-        localJobPath = os.path.join(config["storage-mount-path"], jobPath)
+        job_object.job_path = jobParams["jobPath"]
+        job_object.work_path = jobParams["workPath"]
+        job_object.data_path = jobParams["dataPath"]
+        if "mountpoints" in jobParams:
+            job_object.add_mountpoints(jobParams["mountpoints"])
+        job_object.add_mountpoints(job_object.job_path_mountpoint())
+        job_object.add_mountpoints(job_object.work_path_mountpoint())
+        job_object.add_mountpoints(job_object.data_path_mountpoint())
 
+        localJobPath = job_object.get_job_path_hostpath()
         if not os.path.exists(localJobPath):
             mkdirsAsUser(localJobPath, jobParams["userId"])
             mkdirsAsUser(os.path.join(localJobPath, "models"), jobParams["userId"])
@@ -129,30 +132,14 @@ def SubmitRegularJob(job):
         jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
         jobTemp = os.path.join(jobTempDir, "RegularJob.yaml.template")
 
-        jobParams["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
-        jobParams["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
-        jobParams["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
         jobParams["nvidiaDriverPath"] = nvidiaDriverPath
 
+        # TODO will need to move it out of jobParams
         jobParams["rest-api"] = config["rest-api"]
 
-        if "mountpoints" in jobParams:
-            job_object.add_mountpoints(jobParams["mountpoints"])
-
-        mp = {"name": "job", "containerPath": "/job", "hostPath": jobParams["hostjobPath"], "enabled": True}
-        job_object.add_mountpoints(mp)
-
-        mp = {"name": "work", "containerPath": "/work", "hostPath": jobParams["hostworkPath"], "enabled": True}
-        job_object.add_mountpoints(mp)
-
-        mp = {"name": "data", "containerPath": "/data", "hostPath": jobParams["hostdataPath"], "enabled": True}
-        job_object.add_mountpoints(mp)
-
         jobParams["mountpoints"] = job_object.mountpoints
-
-        userAlias = getAlias(jobParams["userName"])
         jobParams["user_email"] = jobParams["userName"]
-        jobParams["homeFolderHostpath"] = os.path.join(config["storage-mount-path"], GetWorkPath(userAlias))
+        jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
 
         jobParams["pod_ip_range"] = config["pod_ip_range"]
         if "usefreeflow" in config:
@@ -171,7 +158,7 @@ def SubmitRegularJob(job):
             c = 0
             while (i <= end):
                 pod = {}
-                pod["podName"] = jobParams["jobId"]+"-pod-"+str(c)
+                pod["podName"] = jobParams["jobId"] + "-pod-" + str(c)
                 pod["envs"] = [{"name": jobParams["hyperparametername"], "value":i}]
                 i += step
                 c += 1
@@ -217,7 +204,7 @@ def SubmitRegularJob(job):
             user_info = dataHandler.GetIdentityInfo(jobParams["userName"])[0]
             jobParams["gid"] = user_info["gid"]
             jobParams["uid"] = user_info["uid"]
-            jobParams["user"] = userAlias
+            jobParams["user"] = job_object.get_alias()
 
             template = ENV.get_template(os.path.abspath(jobTemp))
             job_description = template.render(job=jobParams)
@@ -241,6 +228,7 @@ def SubmitRegularJob(job):
 
         ret["jobId"] = jobParams["jobId"]
 
+        # TODO why?
         if "userName" not in jobParams:
             jobParams["userName"] = ""
 
@@ -258,7 +246,9 @@ def SubmitRegularJob(job):
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobMeta", jobMetaStr)
     except Exception as e:
-        print(e)
+        logging.error("Submit job failed: %s" % job)
+        traceback.print_exc()
+        logging.error(e)
         ret["error"] = str(e)
         retries = dataHandler.AddandGetJobRetries(job["jobId"])
         if retries >= 5:
@@ -268,7 +258,6 @@ def SubmitRegularJob(job):
     return ret
 
 
-
 def SubmitPSDistJob(job):
     ret = {}
     dataHandler = DataHandler()
@@ -533,6 +522,7 @@ def KillJob(job, desiredState="killed"):
     return False
 
 
+# TODO remove it latter
 def getAlias(username):
     if "@" in username:
         username = username.split("@")[0].strip()

From 32cbac5711ed9d53d89cb55077856b02bfabf905 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 09:42:47 +0000
Subject: [PATCH 012/194] extract the launch-script generation to a method

---
 src/ClusterManager/job.py         | 17 +++++++++++++++++
 src/ClusterManager/job_manager.py | 26 +++-----------------------
 src/ClusterManager/test_job.py    | 16 ++++++++++++++--
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 0b2d71018..7d618d3ef 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -6,6 +6,9 @@
 import logging
 import logging.config
 
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+from osUtils import mkdirsAsUser
+
 
 # TODO remove it latter
 def create_log(logdir='.'):
@@ -104,6 +107,20 @@ def data_path_mountpoint(self):
         data_host_path = os.path.join(self.cluster["storage-mount-path"], "storage", self.data_path)
         return {"name": "data", "containerPath": "/data", "hostPath": data_host_path, "enabled": True}
 
+    def generate_launch_script(self, path_to_save, user_id, gpu_num, user_script):
+        if not os.path.exists(path_to_save):
+            mkdirsAsUser(path_to_save, user_id)
+
+        file_name = "launch-%s.sh" % self.job_id
+        launch_script_file = os.path.join(path_to_save, file_name)
+        with open(launch_script_file, 'w') as f:
+            f.write("#!/bin/bash -x\n")
+            f.write("mkdir /opt; \n")
+            f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % gpu_num)
+            f.write("bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
+        os.system("sudo chown %s %s" % (user_id, launch_script_file))
+        return file_name
+
 
 class JobSchema(Schema):
     cluster = fields.Dict(required=True)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 10c6dd839..186c3cbd0 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -88,7 +88,7 @@ def SubmitRegularJob(job):
         assert(isinstance(job_object, Job))
 
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
-        if any(required_field not in jobParams for required_field in ["jobPath", "workPath", "cmd"]):
+        if any(required_field not in jobParams for required_field in ["jobPath", "workPath", "cmd", "userId", "resourcegpu"]):
             dataHandler.SetJobError(jobParams["jobId"], "ERROR: required fileds missing in jobParams.")
             return False
 
@@ -102,29 +102,9 @@ def SubmitRegularJob(job):
         job_object.add_mountpoints(job_object.data_path_mountpoint())
 
         localJobPath = job_object.get_job_path_hostpath()
-        if not os.path.exists(localJobPath):
-            mkdirsAsUser(localJobPath, jobParams["userId"])
-            mkdirsAsUser(os.path.join(localJobPath, "models"), jobParams["userId"])
-
-        jobParams["LaunchCMD"] = ""
-        if "cmd" not in jobParams:
-            jobParams["cmd"] = ""
-
-        if isinstance(jobParams["cmd"], basestring) and not jobParams["cmd"] == "":
-            launchScriptPath = os.path.join(localJobPath, "launch-%s.sh" % jobParams["jobId"])
-            with open(launchScriptPath, 'w') as f:
-                f.write("#!/bin/bash -x\n")
-                f.write("mkdir /opt; \n")
-                f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % jobParams["resourcegpu"])
-                # TODO refine it later
-                f.write("bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % jobParams["cmd"])
-            f.close()
-            if "userId" in jobParams:
-                os.system("chown -R %s %s" % (jobParams["userId"], launchScriptPath))
-            jobParams["LaunchCMD"] = "[\"bash\", \"/job/launch-%s.sh\"]" % jobParams["jobId"]
-
+        script_file = job_object.generate_launch_script(localJobPath, jobParams["userId"], jobParams["resourcegpu"], jobParams["cmd"])
+        jobParams["LaunchCMD"] = "[\"bash\", \"/job/%s\"]" % script_file
         jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
-
         jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
 
         ENV = Environment(loader=FileSystemLoader("/"))
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 0cbf4b8f4..326448237 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -144,9 +144,21 @@ def test_job_work_data_mountpoints(self):
 
         self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.job_path_mountpoint()["hostPath"])
         self.assertEqual("/dlwsdata/work/user_alias", job.work_path_mountpoint()["hostPath"])
-        self.assertEqual("/dlwsdata/storage/xx", job.data_path_mountpoint()["hostPath"])
+        self.assertEqual("/dlwsdata/storage/", job.data_path_mountpoint()["hostPath"])
 
         job.add_mountpoints(job.job_path_mountpoint())
         job.add_mountpoints(job.work_path_mountpoint())
         job.add_mountpoints(job.data_path_mountpoint())
-        self.assertEquals(3, len(job.mountpoints()))
+        self.assertEquals(3, len(job.mountpoints))
+
+    def test_generate_launch_script(self):
+        job = self.create_a_job()
+
+        path_to_save = "/tmp"
+        user_id = "20000"
+        gpu_num = 3
+        user_script = "sleep infinity"
+        script_file = job.generate_launch_script(path_to_save, user_id, gpu_num, user_script)
+
+        # "launch-%s.sh" % self.job_id
+        self.assertEqual("launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh", script_file)

From 52c75e9e6234233296ec97e532359a30bb8c0c14 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 09:51:54 +0000
Subject: [PATCH 013/194] cleanup useless code

---
 src/ClusterManager/job_manager.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 186c3cbd0..39d4e865d 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -39,10 +39,6 @@
 from job import Job, JobSchema
 
 
-nvidiaDriverPath = config["nvidiaDriverPath"]
-
-
-
 def printlog(msg):
     print("%s - %s" % (datetime.datetime.utcnow().strftime("%x %X"),msg))
 
@@ -112,8 +108,6 @@ def SubmitRegularJob(job):
         jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
         jobTemp = os.path.join(jobTempDir, "RegularJob.yaml.template")
 
-        jobParams["nvidiaDriverPath"] = nvidiaDriverPath
-
         # TODO will need to move it out of jobParams
         jobParams["rest-api"] = config["rest-api"]
 
@@ -374,12 +368,10 @@ def SubmitPSDistJob(job):
                     distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
                     distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
                     distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
-                    distJobParam["nvidiaDriverPath"] = nvidiaDriverPath
 
                     if "mountpoints" not in distJobParam:
                         distJobParam["mountpoints"] = []
 
-                    # distJobParam["mountpoints"].append({"name":"nvidia-driver","containerPath":"/usr/local/nvidia","hostPath":nvidiaDriverPath})
                     distJobParam["mountpoints"].append({"name":"job","containerPath":"/job","hostPath":distJobParam["hostjobPath"]})
                     distJobParam["mountpoints"].append({"name":"work","containerPath":"/work","hostPath":distJobParam["hostworkPath"]})
                     distJobParam["mountpoints"].append({"name":"data","containerPath":"/data","hostPath":distJobParam["hostdataPath"]})

From 8a19a3878c30cc52d1a828311c4c48bce4a38bac Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 27 Jun 2019 10:16:07 +0000
Subject: [PATCH 014/194] refine: instance job template

---
 src/ClusterManager/job.py         |  9 +++++++++
 src/ClusterManager/job_manager.py | 17 ++++++++++-------
 src/ClusterManager/test_job.py    |  5 +++++
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 7d618d3ef..e1d00bab1 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -2,6 +2,7 @@
 import os
 from datetime import date
 from marshmallow import Schema, fields, pprint, post_load, validate
+from jinja2 import Environment, FileSystemLoader, Template
 
 import logging
 import logging.config
@@ -121,6 +122,14 @@ def generate_launch_script(self, path_to_save, user_id, gpu_num, user_script):
         os.system("sudo chown %s %s" % (user_id, launch_script_file))
         return file_name
 
+    def get_template(self):
+        """Return abosulutely path of job template file on the node."""
+        path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "RegularJob.yaml.template"))
+        ENV = Environment(loader=FileSystemLoader("/"))
+        template = ENV.get_template(path)
+        assert(isinstance(template, Template))
+        return template
+
 
 class JobSchema(Schema):
     cluster = fields.Dict(required=True)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 39d4e865d..3b1669f43 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -84,7 +84,15 @@ def SubmitRegularJob(job):
         assert(isinstance(job_object, Job))
 
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
-        if any(required_field not in jobParams for required_field in ["jobPath", "workPath", "cmd", "userId", "resourcegpu"]):
+        if any(required_field not in jobParams for required_field in
+                [
+                    "jobtrainingtype",
+                    "jobPath",
+                    "workPath",
+                    "cmd",
+                    "userId",
+                    "resourcegpu",
+                ]):
             dataHandler.SetJobError(jobParams["jobId"], "ERROR: required fileds missing in jobParams.")
             return False
 
@@ -103,11 +111,6 @@ def SubmitRegularJob(job):
         jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
         jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
 
-        ENV = Environment(loader=FileSystemLoader("/"))
-
-        jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
-        jobTemp = os.path.join(jobTempDir, "RegularJob.yaml.template")
-
         # TODO will need to move it out of jobParams
         jobParams["rest-api"] = config["rest-api"]
 
@@ -180,7 +183,7 @@ def SubmitRegularJob(job):
             jobParams["uid"] = user_info["uid"]
             jobParams["user"] = job_object.get_alias()
 
-            template = ENV.get_template(os.path.abspath(jobTemp))
+            template = job_object.get_template()
             job_description = template.render(job=jobParams)
             jobDescriptionList.append(job_description)
 
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 326448237..60665db13 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -162,3 +162,8 @@ def test_generate_launch_script(self):
 
         # "launch-%s.sh" % self.job_id
         self.assertEqual("launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh", script_file)
+
+    def test_get_template(self):
+        job = self.create_a_job()
+
+        self.assertIsNotNone(job.get_template())

From cf6ae050cdc36ae4833aab8648a7cd91ca47e580 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 01:59:12 +0000
Subject: [PATCH 015/194] user's mountpoints first, but should after 'jobPath'

---
 src/ClusterManager/job_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 3b1669f43..775028633 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -99,9 +99,10 @@ def SubmitRegularJob(job):
         job_object.job_path = jobParams["jobPath"]
         job_object.work_path = jobParams["workPath"]
         job_object.data_path = jobParams["dataPath"]
+        # TODO user's mountpoints first, but should after 'job_path'
+        job_object.add_mountpoints(job_object.job_path_mountpoint())
         if "mountpoints" in jobParams:
             job_object.add_mountpoints(jobParams["mountpoints"])
-        job_object.add_mountpoints(job_object.job_path_mountpoint())
         job_object.add_mountpoints(job_object.work_path_mountpoint())
         job_object.add_mountpoints(job_object.data_path_mountpoint())
 

From 8de64e28d5b1caaae36cc1a1998c49859471613a Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 05:48:34 +0000
Subject: [PATCH 016/194] cleanup job_manager

---
 src/ClusterManager/job_manager.py | 88 +++++++++++++++++--------------
 1 file changed, 47 insertions(+), 41 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 775028633..301b2c7d9 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -125,24 +125,28 @@ def SubmitRegularJob(job):
         else:
             jobParams["usefreeflow"] = False
 
-        print ("Render Job: %s" % jobParams)
-        jobDescriptionList = []
+
+        # inject gid, uid and user
+        # TODO it should return only one entry
+        user_info = dataHandler.GetIdentityInfo(jobParams["userName"])[0]
+        jobParams["gid"] = user_info["gid"]
+        jobParams["uid"] = user_info["uid"]
+        jobParams["user"] = job_object.get_alias()
 
         pods = []
-        if "hyperparametername" in jobParams and "hyperparameterstartvalue" in jobParams and "hyperparameterendvalue" in jobParams and "hyperparameterstep" in jobParams:
-            i = int(jobParams["hyperparameterstartvalue"])
+        if all(hyper_parameter in jobParams for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
+            env_name = jobParams["hyperparametername"]
+            start = int(jobParams["hyperparameterstartvalue"])
             end = int(jobParams["hyperparameterendvalue"])
             step = int(jobParams["hyperparameterstep"])
-            c = 0
-            while (i <= end):
-                pod = {}
-                pod["podName"] = jobParams["jobId"] + "-pod-" + str(c)
-                pod["envs"] = [{"name": jobParams["hyperparametername"], "value":i}]
-                i += step
-                c += 1
+
+            for idx, val in enumerate(range(start, end, step)):
+                pod = jobParams.copy()
+                pod["podName"] = "{0}-pod-{1}".format(jobParams["jobId"], idx)
+                pod["envs"] = [{"name": env_name, "value": val}]
                 pods.append(pod)
         else:
-                pod = {}
+                pod = jobParams.copy()
                 pod["podName"] = jobParams["jobId"]
                 pod["envs"] = []
                 pods.append(pod)
@@ -151,41 +155,43 @@ def SubmitRegularJob(job):
             jobParams["env"] = []
         jobParams["commonenv"] = copy.copy(jobParams["env"])
 
-        for pod in pods:
-            jobParams["podName"] = pod["podName"]
-            jobParams["env"] = jobParams["commonenv"] + pod["envs"]
+        print("Render Job: %s" % jobParams)
+        jobDescriptionList = []
 
+        for pod in pods:
             if "kube_custom_scheduler" in config and config["kube_custom_scheduler"]:
-                container = {}
-                container["requests"] = {"alpha.gpu/numgpu": int(jobParams["resourcegpu"])}
-                podInfo = {}
-                podInfo["podname"] = jobParams["podName"]
-                if "useGPUTopology" in jobParams and jobParams["useGPUTopology"]:
-                    podInfo["requests"] = {"alpha.gpu/gpu-generate-topology": 1}
+                if "useGPUTopology" in pod and pod["useGPUTopology"]:
+                    gpu_topology_flag = 1
                 else:
                     # for cases when desired topology is explictly given or not desired
-                    podInfo["requests"] = {"alpha.gpu/gpu-generate-topology": 0}
-                podInfo["runningcontainer"] = {jobParams["podName"]: container}
-
-                if "annotations" not in jobParams:
-                    jobParams["annotations"] = {}
-                jobParams["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
-                jobParams["resourcegpu"] = 0  # gpu requests specified through annotation
-
-                if "gpuType" in jobParams:
-                    if "nodeSelector" not in jobParams:
-                        jobParams["nodeSelector"] = {}
-                    jobParams["nodeSelector"]["gpuType"] = jobParams["gpuType"]
-
-            # inject gid, uid and user
-            # TODO it should return only one entry
-            user_info = dataHandler.GetIdentityInfo(jobParams["userName"])[0]
-            jobParams["gid"] = user_info["gid"]
-            jobParams["uid"] = user_info["uid"]
-            jobParams["user"] = job_object.get_alias()
+                    gpu_topology_flag = 0
+                pod_name = pod["podName"]
+                request_gpu = int(pod["resourcegpu"])
+
+                podInfo = {
+                    "podname": pod_name,
+                    "requests": {
+                        "alpha.gpu/gpu-generate-topology": gpu_topology_flag
+                    },
+                    "runningcontainer": {
+                        pod_name: {
+                            "requests": {"alpha.gpu/numgpu": request_gpu}
+                        },
+                    },
+                }
+
+                if "annotations" not in pod:
+                    pod["annotations"] = {}
+                pod["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
+                pod["resourcegpu"] = 0  # gpu requests specified through annotation
+
+                if "gpuType" in pod:
+                    if "nodeSelector" not in pod:
+                        pod["nodeSelector"] = {}
+                    pod["nodeSelector"]["gpuType"] = pod["gpuType"]
 
             template = job_object.get_template()
-            job_description = template.render(job=jobParams)
+            job_description = template.render(job=pod)
             jobDescriptionList.append(job_description)
 
         jobDescription = "\n---\n".join(jobDescriptionList)

From d7464b6c42a196f0adc015c34c48366e516cfd84 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 08:09:46 +0000
Subject: [PATCH 017/194] extract class PodTemplate

---
 src/ClusterManager/job_manager.py       | 54 ++++----------
 src/ClusterManager/pod_template.py      | 44 ++++++++++++
 src/ClusterManager/test_pod_template.py | 95 +++++++++++++++++++++++++
 3 files changed, 152 insertions(+), 41 deletions(-)
 create mode 100644 src/ClusterManager/pod_template.py
 create mode 100644 src/ClusterManager/test_pod_template.py

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 301b2c7d9..c034ead5a 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -37,6 +37,7 @@
 import logging
 import logging.config
 from job import Job, JobSchema
+from pod_template import PodTemplate
 
 
 def printlog(msg):
@@ -125,7 +126,6 @@ def SubmitRegularJob(job):
         else:
             jobParams["usefreeflow"] = False
 
-
         # inject gid, uid and user
         # TODO it should return only one entry
         user_info = dataHandler.GetIdentityInfo(jobParams["userName"])[0]
@@ -155,46 +155,18 @@ def SubmitRegularJob(job):
             jobParams["env"] = []
         jobParams["commonenv"] = copy.copy(jobParams["env"])
 
-        print("Render Job: %s" % jobParams)
-        jobDescriptionList = []
+        if "kube_custom_scheduler" in config and config["kube_custom_scheduler"]:
+            enable_custom_scheduler = True
+        else:
+            enable_custom_scheduler = False
+        pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
 
+        print("Render Job: %s" % jobParams)
+        job_description_list = []
         for pod in pods:
-            if "kube_custom_scheduler" in config and config["kube_custom_scheduler"]:
-                if "useGPUTopology" in pod and pod["useGPUTopology"]:
-                    gpu_topology_flag = 1
-                else:
-                    # for cases when desired topology is explictly given or not desired
-                    gpu_topology_flag = 0
-                pod_name = pod["podName"]
-                request_gpu = int(pod["resourcegpu"])
-
-                podInfo = {
-                    "podname": pod_name,
-                    "requests": {
-                        "alpha.gpu/gpu-generate-topology": gpu_topology_flag
-                    },
-                    "runningcontainer": {
-                        pod_name: {
-                            "requests": {"alpha.gpu/numgpu": request_gpu}
-                        },
-                    },
-                }
-
-                if "annotations" not in pod:
-                    pod["annotations"] = {}
-                pod["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
-                pod["resourcegpu"] = 0  # gpu requests specified through annotation
-
-                if "gpuType" in pod:
-                    if "nodeSelector" not in pod:
-                        pod["nodeSelector"] = {}
-                    pod["nodeSelector"]["gpuType"] = pod["gpuType"]
-
-            template = job_object.get_template()
-            job_description = template.render(job=pod)
-            jobDescriptionList.append(job_description)
-
-        jobDescription = "\n---\n".join(jobDescriptionList)
+            job_description = pod_template.generate_pod_yaml(pod)
+            job_description_list.append(job_description)
+        job_description = "\n---\n".join(job_description_list)
 
         jobDescriptionPath = os.path.join(config["storage-mount-path"], jobParams["jobDescriptionPath"])
         if not os.path.exists(os.path.dirname(os.path.realpath(jobDescriptionPath))):
@@ -203,7 +175,7 @@ def SubmitRegularJob(job):
             output = k8sUtils.kubectl_delete(jobDescriptionPath)
 
         with open(jobDescriptionPath, 'w') as f:
-            f.write(jobDescription)
+            f.write(job_description)
 
         output = k8sUtils.kubectl_create(jobDescriptionPath)
         logging.info("Submitted job %s to k8s, returned with status %s" % (job["jobId"], output))
@@ -218,7 +190,7 @@ def SubmitRegularJob(job):
 
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", jobParams["jobDescriptionPath"])
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(jobDescription))
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(job_description))
 
         jobMeta = {}
         jobMeta["jobDescriptionPath"] = jobParams["jobDescriptionPath"]
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
new file mode 100644
index 000000000..ac3d010bc
--- /dev/null
+++ b/src/ClusterManager/pod_template.py
@@ -0,0 +1,44 @@
+import json
+from jinja2 import Template
+
+
+class PodTemplate():
+    def __init__(self, template, enable_custom_scheduler):
+        self.template = template
+        self.enable_custom_scheduler = enable_custom_scheduler
+
+    def generate_pod_yaml(self, pod):
+        assert(isinstance(self.template, Template))
+        if self.enable_custom_scheduler:
+            if "useGPUTopology" in pod and pod["useGPUTopology"]:
+                gpu_topology_flag = 1
+            else:
+                # for cases when desired topology is explictly given or not desired
+                gpu_topology_flag = 0
+            pod_name = pod["podName"]
+            request_gpu = int(pod["resourcegpu"])
+
+            podInfo = {
+                "podname": pod_name,
+                "requests": {
+                    "alpha.gpu/gpu-generate-topology": gpu_topology_flag
+                },
+                "runningcontainer": {
+                    pod_name: {
+                        "requests": {"alpha.gpu/numgpu": request_gpu}
+                    },
+                },
+            }
+
+            if "annotations" not in pod:
+                pod["annotations"] = {}
+            pod["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
+            pod["resourcegpu"] = 0  # gpu requests specified through annotation
+
+        if "nodeSelector" not in pod:
+            pod["nodeSelector"] = {}
+        if "gpuType" in pod:
+            pod["nodeSelector"]["gpuType"] = pod["gpuType"]
+
+        # template = job_object.get_template()
+        return self.template.render(job=pod)
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
new file mode 100644
index 000000000..98f5aa7a7
--- /dev/null
+++ b/src/ClusterManager/test_pod_template.py
@@ -0,0 +1,95 @@
+import unittest
+import json
+import yaml
+import sys
+import os
+from job import Job, JobSchema
+from pod_template import PodTemplate
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+from config import config
+
+VALID_JOB_ATTRIBUTES = {
+    "cluster": config,
+    "jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c",
+    "userName": "user@foo.com",
+    "jobPath": "user_alias/jobs/date/job_id"
+}
+
+job, errors = JobSchema().load(VALID_JOB_ATTRIBUTES)
+assert(not errors)
+
+
+class TestPodTemplate(unittest.TestCase):
+
+    def test_pod_template_without_custer_scheduler(self):
+        enable_custom_scheduler = False
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        pod = {"resourcegpu": 2}
+        pod_yaml = pod_template.generate_pod_yaml(pod)
+        data = yaml.load(pod_yaml)
+
+        # not eanbled custom scheduler, set the resource limits: spec.containers[].resources.limits
+        self.assertEqual(pod["resourcegpu"], data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
+        # metadata.annotations["pod.alpha/DeviceInformation"] should be empty
+        self.assertTrue(("annotations" not in data["metadata"]) or ("pod.alpha/DeviceInformation" not in data["metadata"]["annotations"]))
+
+    def test_pod_template_with_custom_scheduler(self):
+        enable_custom_scheduler = True
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        gpu_num = 2
+        pod = {
+            "podName": "790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0",
+            "resourcegpu": gpu_num,
+        }
+        pod_yaml = pod_template.generate_pod_yaml(pod)
+        data = yaml.load(pod_yaml)
+
+        # eanbled custom scheduler would clear the resource limits: spec.containers[].resources.limits
+        self.assertEqual(0, data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
+
+        # metadata.annotations["pod.alpha/DeviceInformation"] should be set
+        # annotations = data["metadata"]["annotations"]
+        device_annotation = json.loads(data["metadata"]["annotations"]["pod.alpha/DeviceInformation"])
+        self.assertEqual(gpu_num, device_annotation["runningcontainer"][pod["podName"]]["requests"]["alpha.gpu/numgpu"])
+        # disabled topology
+        self.assertEqual(0, device_annotation["requests"]["alpha.gpu/gpu-generate-topology"])
+
+    def test_pod_template_with_custom_scheduler_use_topology(self):
+        enable_custom_scheduler = True
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        gpu_num = 2
+        pod = {
+            "podName": "790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0",
+            "resourcegpu": gpu_num,
+            "useGPUTopology": True
+        }
+        pod_yaml = pod_template.generate_pod_yaml(pod)
+        data = yaml.load(pod_yaml)
+
+        # eanbled custom scheduler, clear the resource limits: spec.containers[].resources.limits
+        self.assertEqual(0, data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
+
+        # metadata.annotations["pod.alpha/DeviceInformation"] should be set:
+        # {
+        #    "requests":{
+        #       "alpha.gpu/gpu-generate-topology":1
+        #    },
+        #    "runningcontainer":{
+        #       "790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0":{
+        #          "requests":{
+        #             "alpha.gpu/numgpu":2
+        #          }
+        #       }
+        #    },
+        #    "podname":"790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0"
+        # }
+
+        # annotations = data["metadata"]["annotations"]
+        device_annotation = json.loads(data["metadata"]["annotations"]["pod.alpha/DeviceInformation"])
+        self.assertEqual(gpu_num, device_annotation["runningcontainer"][pod["podName"]]["requests"]["alpha.gpu/numgpu"])
+        # enabled topology
+        self.assertEqual(1, device_annotation["requests"]["alpha.gpu/gpu-generate-topology"])

From d2c44048653538de5132eae7068898c9a0381cc3 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 09:09:41 +0000
Subject: [PATCH 018/194] cleanup job_manager.py

---
 src/ClusterManager/job_manager.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index c034ead5a..bc5dbcf35 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -110,7 +110,6 @@ def SubmitRegularJob(job):
         localJobPath = job_object.get_job_path_hostpath()
         script_file = job_object.generate_launch_script(localJobPath, jobParams["userId"], jobParams["resourcegpu"], jobParams["cmd"])
         jobParams["LaunchCMD"] = "[\"bash\", \"/job/%s\"]" % script_file
-        jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
         jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
 
         # TODO will need to move it out of jobParams
@@ -168,32 +167,30 @@ def SubmitRegularJob(job):
             job_description_list.append(job_description)
         job_description = "\n---\n".join(job_description_list)
 
-        jobDescriptionPath = os.path.join(config["storage-mount-path"], jobParams["jobDescriptionPath"])
-        if not os.path.exists(os.path.dirname(os.path.realpath(jobDescriptionPath))):
-            os.makedirs(os.path.dirname(os.path.realpath(jobDescriptionPath)))
-        if os.path.isfile(jobDescriptionPath):
-            output = k8sUtils.kubectl_delete(jobDescriptionPath)
+        job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
+        local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
 
-        with open(jobDescriptionPath, 'w') as f:
+        if os.path.isfile(local_jobDescriptionPath):
+            output = k8sUtils.kubectl_delete(local_jobDescriptionPath)
+
+        if not os.path.exists(os.path.dirname(local_jobDescriptionPath)):
+            os.makedirs(os.path.dirname(os.path.realpath(local_jobDescriptionPath)))
+        with open(local_jobDescriptionPath, 'w') as f:
             f.write(job_description)
 
-        output = k8sUtils.kubectl_create(jobDescriptionPath)
+        output = k8sUtils.kubectl_create(local_jobDescriptionPath)
         logging.info("Submitted job %s to k8s, returned with status %s" % (job["jobId"], output))
 
         ret["output"] = output
 
         ret["jobId"] = jobParams["jobId"]
 
-        # TODO why?
-        if "userName" not in jobParams:
-            jobParams["userName"] = ""
-
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", jobParams["jobDescriptionPath"])
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", job_description_path)
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(job_description))
 
         jobMeta = {}
-        jobMeta["jobDescriptionPath"] = jobParams["jobDescriptionPath"]
+        jobMeta["jobDescriptionPath"] = job_description_path
         jobMeta["jobPath"] = jobParams["jobPath"]
         jobMeta["workPath"] = jobParams["workPath"]
         jobMeta["jobPath"] = jobParams["jobPath"]

From b6c67283dc652b7006c7c874b78d7441ef007db9 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 09:41:27 +0000
Subject: [PATCH 019/194] refactoring

---
 src/ClusterManager/job.py         |  5 ++++-
 src/ClusterManager/job_manager.py | 30 +++++++++++-------------------
 src/ClusterManager/test_job.py    | 13 +++++++++++--
 3 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index e1d00bab1..cccd82bbf 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -90,7 +90,7 @@ def get_alias(self):
     def get_homefolder_hostpath(self):
         return os.path.join(self.cluster["storage-mount-path"], "work/", self.get_alias())
 
-    def get_job_path_hostpath(self):
+    def get_local_job_path(self):
         return os.path.join(self.cluster["storage-mount-path"], "work/", self.job_path)
 
     def job_path_mountpoint(self):
@@ -130,6 +130,9 @@ def get_template(self):
         assert(isinstance(template, Template))
         return template
 
+    def is_custom_scheduler_eanbled(self):
+        return "kube_custom_scheduler" in self.cluster and self.cluster["kube_custom_scheduler"]
+
 
 class JobSchema(Schema):
     cluster = fields.Dict(required=True)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index bc5dbcf35..bd4d2980d 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -94,7 +94,7 @@ def SubmitRegularJob(job):
                     "userId",
                     "resourcegpu",
                 ]):
-            dataHandler.SetJobError(jobParams["jobId"], "ERROR: required fileds missing in jobParams.")
+            dataHandler.SetJobError(job_object.job_id, "ERROR: required fileds missing in jobParams.")
             return False
 
         job_object.job_path = jobParams["jobPath"]
@@ -107,8 +107,8 @@ def SubmitRegularJob(job):
         job_object.add_mountpoints(job_object.work_path_mountpoint())
         job_object.add_mountpoints(job_object.data_path_mountpoint())
 
-        localJobPath = job_object.get_job_path_hostpath()
-        script_file = job_object.generate_launch_script(localJobPath, jobParams["userId"], jobParams["resourcegpu"], jobParams["cmd"])
+        local_job_path = job_object.get_local_job_path()
+        script_file = job_object.generate_launch_script(local_job_path, jobParams["userId"], jobParams["resourcegpu"], jobParams["cmd"])
         jobParams["LaunchCMD"] = "[\"bash\", \"/job/%s\"]" % script_file
         jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
 
@@ -141,23 +141,16 @@ def SubmitRegularJob(job):
 
             for idx, val in enumerate(range(start, end, step)):
                 pod = jobParams.copy()
-                pod["podName"] = "{0}-pod-{1}".format(jobParams["jobId"], idx)
+                pod["podName"] = "{0}-pod-{1}".format(job_object.job_id, idx)
                 pod["envs"] = [{"name": env_name, "value": val}]
                 pods.append(pod)
         else:
                 pod = jobParams.copy()
-                pod["podName"] = jobParams["jobId"]
+                pod["podName"] = job_object.job_id
                 pod["envs"] = []
                 pods.append(pod)
 
-        if "env" not in jobParams:
-            jobParams["env"] = []
-        jobParams["commonenv"] = copy.copy(jobParams["env"])
-
-        if "kube_custom_scheduler" in config and config["kube_custom_scheduler"]:
-            enable_custom_scheduler = True
-        else:
-            enable_custom_scheduler = False
+        enable_custom_scheduler = job_object.is_custom_scheduler_eanbled()
         pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
 
         print("Render Job: %s" % jobParams)
@@ -167,7 +160,7 @@ def SubmitRegularJob(job):
             job_description_list.append(job_description)
         job_description = "\n---\n".join(job_description_list)
 
-        job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
+        job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
         local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
 
         if os.path.isfile(local_jobDescriptionPath):
@@ -179,11 +172,11 @@ def SubmitRegularJob(job):
             f.write(job_description)
 
         output = k8sUtils.kubectl_create(local_jobDescriptionPath)
-        logging.info("Submitted job %s to k8s, returned with status %s" % (job["jobId"], output))
+        logging.info("Submitted job %s to k8s, returned with status %s" % (job_object.job_id, output))
 
         ret["output"] = output
 
-        ret["jobId"] = jobParams["jobId"]
+        ret["jobId"] = job_object.job_id
 
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", job_description_path)
@@ -191,9 +184,8 @@ def SubmitRegularJob(job):
 
         jobMeta = {}
         jobMeta["jobDescriptionPath"] = job_description_path
-        jobMeta["jobPath"] = jobParams["jobPath"]
-        jobMeta["workPath"] = jobParams["workPath"]
-        jobMeta["jobPath"] = jobParams["jobPath"]
+        jobMeta["jobPath"] = job_object.job_path
+        jobMeta["workPath"] = job_object.work_path
         jobMeta["LaunchCMD"] = jobParams["LaunchCMD"]
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 60665db13..8633cea4c 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -130,10 +130,10 @@ def test_get_homefolder_hostpath(self):
         job = self.create_a_job()
         self.assertEqual("/dlwsdata/work/user", job.get_homefolder_hostpath())
 
-    def test_get_job_path_hostpath(self):
+    def test_get_local_job_path(self):
         job = self.create_a_job()
         self.assertEqual("user_alias/jobs/date/job_id", job.job_path)
-        self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.get_job_path_hostpath())
+        self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.get_local_job_path())
 
     def test_job_work_data_mountpoints(self):
         job = self.create_a_job()
@@ -167,3 +167,12 @@ def test_get_template(self):
         job = self.create_a_job()
 
         self.assertIsNotNone(job.get_template())
+
+    def test_is_custom_scheduler_eanbled(self):
+        job = self.create_a_job()
+
+        self.assertFalse(job.is_custom_scheduler_eanbled())
+
+        # TODO !!! notice, it would change all the 'cluster' settings
+        job.cluster["kube_custom_scheduler"] = True
+        self.assertTrue(job.is_custom_scheduler_eanbled())

From 4a77fb3b470e4c2bcac7e7d66a825d83a98ad089 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 10:03:53 +0000
Subject: [PATCH 020/194] minor refactoring

---
 src/ClusterManager/job.py          |  5 +++++
 src/ClusterManager/job_manager.py  | 23 +++++++++++++----------
 src/ClusterManager/pod_template.py |  1 +
 src/ClusterManager/test_job.py     |  7 ++++++-
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index cccd82bbf..4a43fcd0f 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -133,6 +133,11 @@ def get_template(self):
     def is_custom_scheduler_eanbled(self):
         return "kube_custom_scheduler" in self.cluster and self.cluster["kube_custom_scheduler"]
 
+    def get_rest_api_url(self):
+        if "rest-api" in self.cluster:
+            return self.cluster["rest-api"]
+        return None
+
 
 class JobSchema(Schema):
     cluster = fields.Dict(required=True)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index bd4d2980d..45ff6dbf1 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -88,11 +88,14 @@ def SubmitRegularJob(job):
         if any(required_field not in jobParams for required_field in
                 [
                     "jobtrainingtype",
+                    "jobName",
                     "jobPath",
                     "workPath",
+                    "dataPath",
                     "cmd",
                     "userId",
                     "resourcegpu",
+                    "userName",
                 ]):
             dataHandler.SetJobError(job_object.job_id, "ERROR: required fileds missing in jobParams.")
             return False
@@ -109,11 +112,7 @@ def SubmitRegularJob(job):
 
         local_job_path = job_object.get_local_job_path()
         script_file = job_object.generate_launch_script(local_job_path, jobParams["userId"], jobParams["resourcegpu"], jobParams["cmd"])
-        jobParams["LaunchCMD"] = "[\"bash\", \"/job/%s\"]" % script_file
-        jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
-
-        # TODO will need to move it out of jobParams
-        jobParams["rest-api"] = config["rest-api"]
+        luanch_cmd = "[\"bash\", \"/job/%s\"]" % script_file
 
         jobParams["mountpoints"] = job_object.mountpoints
         jobParams["user_email"] = jobParams["userName"]
@@ -132,6 +131,10 @@ def SubmitRegularJob(job):
         jobParams["uid"] = user_info["uid"]
         jobParams["user"] = job_object.get_alias()
 
+        # assign parameters to generate the pod yaml
+        jobParams["LaunchCMD"] = luanch_cmd
+        jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
+        jobParams["rest-api"] = job_object.get_rest_api_url()
         pods = []
         if all(hyper_parameter in jobParams for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
             env_name = jobParams["hyperparametername"]
@@ -178,18 +181,18 @@ def SubmitRegularJob(job):
 
         ret["jobId"] = job_object.job_id
 
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", job_description_path)
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(job_description))
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobStatus", "scheduling")
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobDescriptionPath", job_description_path)
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobDescription", base64.b64encode(job_description))
 
         jobMeta = {}
         jobMeta["jobDescriptionPath"] = job_description_path
         jobMeta["jobPath"] = job_object.job_path
         jobMeta["workPath"] = job_object.work_path
-        jobMeta["LaunchCMD"] = jobParams["LaunchCMD"]
+        jobMeta["LaunchCMD"] = luanch_cmd
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobMeta", jobMetaStr)
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
     except Exception as e:
         logging.error("Submit job failed: %s" % job)
         traceback.print_exc()
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index ac3d010bc..d6feeeca0 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -33,6 +33,7 @@ def generate_pod_yaml(self, pod):
             if "annotations" not in pod:
                 pod["annotations"] = {}
             pod["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
+            # TODO it's not safe to update pod["resourcegpu"]
             pod["resourcegpu"] = 0  # gpu requests specified through annotation
 
         if "nodeSelector" not in pod:
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 8633cea4c..f3d782d21 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -12,7 +12,7 @@
     "cluster": config,
     "jobId": "ce7dca49-28df-450a-a03b-51b9c2ecc69c",
     "userName": "user@foo.com",
-    "jobPath": "user_alias/jobs/date/job_id"
+    "jobPath": "user_alias/jobs/date/job_id",
 }
 
 
@@ -176,3 +176,8 @@ def test_is_custom_scheduler_eanbled(self):
         # TODO !!! notice, it would change all the 'cluster' settings
         job.cluster["kube_custom_scheduler"] = True
         self.assertTrue(job.is_custom_scheduler_eanbled())
+
+    def test_get_rest_api_url(self):
+        job = self.create_a_job()
+
+        self.assertEqual("http://faked.uri/", job.get_rest_api_url())

From ff4df26eb90d03e3a7495bb14b43373f7fb7f360 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 11:15:16 +0000
Subject: [PATCH 021/194] refactoring and fix typo

---
 src/ClusterManager/job.py      | 20 ++++++++++++++++----
 src/ClusterManager/test_job.py |  6 +++---
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 4a43fcd0f..18773679e 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -31,6 +31,7 @@ def __init__(self,
                  job_path="",
                  work_path="",
                  data_path="",
+                 params=None,
                  ):
         """
         job_id: an unique string for the job.
@@ -46,6 +47,7 @@ def __init__(self,
         self.job_path = job_path
         self.work_path = work_path
         self.data_path = data_path
+        self.params = params
 
     def add_mountpoints(self, mountpoint):
         '''
@@ -130,12 +132,21 @@ def get_template(self):
         assert(isinstance(template, Template))
         return template
 
-    def is_custom_scheduler_eanbled(self):
-        return "kube_custom_scheduler" in self.cluster and self.cluster["kube_custom_scheduler"]
+    def is_custom_scheduler_enabled(self):
+        return self._get_cluster_config("kube_custom_scheduler")
 
     def get_rest_api_url(self):
-        if "rest-api" in self.cluster:
-            return self.cluster["rest-api"]
+        return self._get_cluster_config("rest-api")
+
+    def get_pod_ip_range(self):
+        return self._get_cluster_config("pod_ip_range")
+
+    def is_user_flow_enabled(self):
+        return self._get_cluster_config("usefreeflow")
+
+    def _get_cluster_config(self, key):
+        if key in self.cluster:
+            return self.cluster[key]
         return None
 
 
@@ -156,6 +167,7 @@ class JobSchema(Schema):
     job_path = fields.String(required=False, dump_to="jobPath", load_from="jobPath")
     work_path = fields.String(required=False, dump_to="workPath", load_from="workPath")
     data_path = fields.String(required=False, dump_to="dataPath", load_from="dataPath")
+    params = fields.Dict(required=False)
 
     @post_load
     def make_user(self, data, **kwargs):
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index f3d782d21..3bc9516c1 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -168,14 +168,14 @@ def test_get_template(self):
 
         self.assertIsNotNone(job.get_template())
 
-    def test_is_custom_scheduler_eanbled(self):
+    def test_is_custom_scheduler_enabled(self):
         job = self.create_a_job()
 
-        self.assertFalse(job.is_custom_scheduler_eanbled())
+        self.assertFalse(job.is_custom_scheduler_enabled())
 
         # TODO !!! notice, it would change all the 'cluster' settings
         job.cluster["kube_custom_scheduler"] = True
-        self.assertTrue(job.is_custom_scheduler_eanbled())
+        self.assertTrue(job.is_custom_scheduler_enabled())
 
     def test_get_rest_api_url(self):
         job = self.create_a_job()

From ba441e11e5c488e2d18dc58a6e42b9214a311219 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 28 Jun 2019 11:32:25 +0000
Subject: [PATCH 022/194] generate job description in PodTemplate

---
 src/ClusterManager/job_manager.py       | 85 ++++---------------------
 src/ClusterManager/pod_template.py      | 77 +++++++++++++++++++++-
 src/ClusterManager/test_pod_template.py | 66 +++++++++++++++++++
 3 files changed, 154 insertions(+), 74 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 45ff6dbf1..af33b4908 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -84,85 +84,24 @@ def SubmitRegularJob(job):
         # TODO assert job_object is a Job
         assert(isinstance(job_object, Job))
 
-        jobParams = json.loads(base64.b64decode(job["jobParams"]))
-        if any(required_field not in jobParams for required_field in
-                [
-                    "jobtrainingtype",
-                    "jobName",
-                    "jobPath",
-                    "workPath",
-                    "dataPath",
-                    "cmd",
-                    "userId",
-                    "resourcegpu",
-                    "userName",
-                ]):
-            dataHandler.SetJobError(job_object.job_id, "ERROR: required fileds missing in jobParams.")
-            return False
-
-        job_object.job_path = jobParams["jobPath"]
-        job_object.work_path = jobParams["workPath"]
-        job_object.data_path = jobParams["dataPath"]
-        # TODO user's mountpoints first, but should after 'job_path'
-        job_object.add_mountpoints(job_object.job_path_mountpoint())
-        if "mountpoints" in jobParams:
-            job_object.add_mountpoints(jobParams["mountpoints"])
-        job_object.add_mountpoints(job_object.work_path_mountpoint())
-        job_object.add_mountpoints(job_object.data_path_mountpoint())
-
-        local_job_path = job_object.get_local_job_path()
-        script_file = job_object.generate_launch_script(local_job_path, jobParams["userId"], jobParams["resourcegpu"], jobParams["cmd"])
-        luanch_cmd = "[\"bash\", \"/job/%s\"]" % script_file
-
-        jobParams["mountpoints"] = job_object.mountpoints
-        jobParams["user_email"] = jobParams["userName"]
-        jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
-
-        jobParams["pod_ip_range"] = config["pod_ip_range"]
-        if "usefreeflow" in config:
-            jobParams["usefreeflow"] = config["usefreeflow"]
-        else:
-            jobParams["usefreeflow"] = False
+        job_object.params = json.loads(base64.b64decode(job["jobParams"]))
 
         # inject gid, uid and user
         # TODO it should return only one entry
-        user_info = dataHandler.GetIdentityInfo(jobParams["userName"])[0]
-        jobParams["gid"] = user_info["gid"]
-        jobParams["uid"] = user_info["uid"]
-        jobParams["user"] = job_object.get_alias()
-
-        # assign parameters to generate the pod yaml
-        jobParams["LaunchCMD"] = luanch_cmd
-        jobParams["jobNameLabel"] = ''.join(e for e in jobParams["jobName"] if e.isalnum())
-        jobParams["rest-api"] = job_object.get_rest_api_url()
-        pods = []
-        if all(hyper_parameter in jobParams for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
-            env_name = jobParams["hyperparametername"]
-            start = int(jobParams["hyperparameterstartvalue"])
-            end = int(jobParams["hyperparameterendvalue"])
-            step = int(jobParams["hyperparameterstep"])
-
-            for idx, val in enumerate(range(start, end, step)):
-                pod = jobParams.copy()
-                pod["podName"] = "{0}-pod-{1}".format(job_object.job_id, idx)
-                pod["envs"] = [{"name": env_name, "value": val}]
-                pods.append(pod)
-        else:
-                pod = jobParams.copy()
-                pod["podName"] = job_object.job_id
-                pod["envs"] = []
-                pods.append(pod)
+        user_info = dataHandler.GetIdentityInfo(job_object.params["userName"])[0]
+        job_object.params["gid"] = user_info["gid"]
+        job_object.params["uid"] = user_info["uid"]
+        job_object.params["user"] = job_object.get_alias()
 
-        enable_custom_scheduler = job_object.is_custom_scheduler_eanbled()
+        enable_custom_scheduler = job_object.is_custom_scheduler_enabled()
         pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
+        ret, error = pod_template.generate_job_description(job_object)
+        if error:
+            dataHandler.SetJobError(job_object.job_id, "ERROR: %s" % error)
+            return False
 
-        print("Render Job: %s" % jobParams)
-        job_description_list = []
-        for pod in pods:
-            job_description = pod_template.generate_pod_yaml(pod)
-            job_description_list.append(job_description)
-        job_description = "\n---\n".join(job_description_list)
-
+        job_description = ret["job_description"]
+        luanch_cmd = ret["luanch_cmd"]
         job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
         local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
 
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index d6feeeca0..887716d90 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -1,5 +1,6 @@
 import json
 from jinja2 import Template
+from job import Job
 
 
 class PodTemplate():
@@ -41,5 +42,79 @@ def generate_pod_yaml(self, pod):
         if "gpuType" in pod:
             pod["nodeSelector"]["gpuType"] = pod["gpuType"]
 
-        # template = job_object.get_template()
         return self.template.render(job=pod)
+
+    def generate_job_description(self, job):
+        """
+        Return (job_description, errors)
+        """
+
+        assert(isinstance(job, Job))
+        params = job.params
+        if any(required_field not in params for required_field in
+                [
+                    "jobtrainingtype",
+                    "jobName",
+                    "jobPath",
+                    "workPath",
+                    "dataPath",
+                    "cmd",
+                    "userId",
+                    "resourcegpu",
+                    "userName",
+                ]):
+            return None, "Missing required parameters!"
+
+        job.job_path = params["jobPath"]
+        job.work_path = params["workPath"]
+        job.data_path = params["dataPath"]
+        # TODO user's mountpoints first, but should after 'job_path'
+        job.add_mountpoints(job.job_path_mountpoint())
+        if "mountpoints" in params:
+            job.add_mountpoints(params["mountpoints"])
+        job.add_mountpoints(job.work_path_mountpoint())
+        job.add_mountpoints(job.data_path_mountpoint())
+
+        params["mountpoints"] = job.mountpoints
+
+        local_job_path = job.get_local_job_path()
+        script_file = job.generate_launch_script(local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
+        luanch_cmd = "[\"bash\", \"/job/%s\"]" % script_file
+
+        params["user_email"] = params["userName"]
+        params["homeFolderHostpath"] = job.get_homefolder_hostpath()
+
+        params["pod_ip_range"] = job.get_pod_ip_range()
+        params["usefreeflow"] = job.is_user_flow_enabled()
+
+        # assign parameters to generate the pod yaml
+        params["LaunchCMD"] = luanch_cmd
+        params["jobNameLabel"] = ''.join(e for e in params["jobName"] if e.isalnum())
+        params["rest-api"] = job.get_rest_api_url()
+        pods = []
+        if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
+            env_name = params["hyperparametername"]
+            start = int(params["hyperparameterstartvalue"])
+            end = int(params["hyperparameterendvalue"])
+            step = int(params["hyperparameterstep"])
+
+            for idx, val in enumerate(range(start, end, step)):
+                pod = params.copy()
+                pod["podName"] = "{0}-pod-{1}".format(job.job_id, idx)
+                pod["envs"] = [{"name": env_name, "value": val}]
+                pods.append(pod)
+        else:
+                pod = params.copy()
+                pod["podName"] = job.job_id
+                pod["envs"] = []
+                pods.append(pod)
+
+        enable_custom_scheduler = job.is_custom_scheduler_enabled()
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        job_description_list = []
+        for pod in pods:
+            job_description = pod_template.generate_pod_yaml(pod)
+            job_description_list.append(job_description)
+        job_description = "\n---\n".join(job_description_list)
+        return {"job_description": job_description, "luanch_cmd": luanch_cmd}, None
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index 98f5aa7a7..ed2ab2d98 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -93,3 +93,69 @@ def test_pod_template_with_custom_scheduler_use_topology(self):
         self.assertEqual(gpu_num, device_annotation["runningcontainer"][pod["podName"]]["requests"]["alpha.gpu/numgpu"])
         # enabled topology
         self.assertEqual(1, device_annotation["requests"]["alpha.gpu/gpu-generate-topology"])
+
+    def test_generate_job_description_missing_required_params(self):
+        enable_custom_scheduler = True
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        job.params = {}
+        job_description, error = pod_template.generate_job_description(job)
+
+        self.assertIsNone(job_description)
+        self.assertTrue(error)
+        self.assertEqual("Missing required parameters!", error)
+
+    def test_generate_job_description(self):
+        enable_custom_scheduler = True
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        job.params = {
+            "gid": "20000",
+            "uid": "20000",
+            "user": "user",
+            "mountpoints": [
+                {
+                    "description": "NFS (remote file share)",
+                    "enabled": True,
+                    "containerPath": "/home/user",
+                    "hostPath": "/dlwsdata/work/user",
+                    "name": "homefolder"
+                }
+            ],
+            "image": "indexserveregistry.azurecr.io/deepscale:1.0",
+            "userId": "20000",
+            "dataPath": "",
+            "jobId": "140782a0-7f6d-4039-9801-fd6294c7c88a",
+            "isParent": 1,
+            "jobType": "training",
+            "jobPath": "user/jobs/190627/140782a0-7f6d-4039-9801-fd6294c7c88a",
+            "containerUserId": "0",
+            "resourcegpu": 1,
+            "env": [
+            ],
+            "enabledatapath": True,
+            "runningasroot": True,
+            "interactivePorts": [
+
+            ],
+            "preemptionAllowed": False,
+            "jobtrainingtype": "RegularJob",
+            "do_log": False,
+            "is_interactive": False,
+            "familyToken": "72fc61265bcb4416b68b44c82d120b3b",
+            "enableworkpath": True,
+            "vcName": "vc1",
+            "userName": "user@foo.com",
+            "workPath": "user",
+            "cmd": "sleep infinity",
+            "jobName": "test-job",
+            "enablejobpath": True,
+            "gpuType": "P40",
+            "ssh": True
+        }
+
+        ret, error = pod_template.generate_job_description(job)
+
+        self.assertFalse(error)
+        self.assertTrue(dict, type(ret))
+        self.assertTrue(["job_description", "launch_cmd"])

From 0eb7a308fa7d58c57b8c1dd73cdbde2810b2f316 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 29 Jun 2019 06:13:01 +0000
Subject: [PATCH 023/194] return job_description list and fix typo

---
 src/ClusterManager/job_manager.py       | 7 ++++---
 src/ClusterManager/pod_template.py      | 3 +--
 src/ClusterManager/test_pod_template.py | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index af33b4908..4f4ce5785 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -100,8 +100,9 @@ def SubmitRegularJob(job):
             dataHandler.SetJobError(job_object.job_id, "ERROR: %s" % error)
             return False
 
-        job_description = ret["job_description"]
-        luanch_cmd = ret["luanch_cmd"]
+        job_description_list = ret["job_descriptions"]
+        job_description = "\n---\n".join(job_description_list)
+        launch_cmd = ret["launch_cmd"]
         job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
         local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
 
@@ -128,7 +129,7 @@ def SubmitRegularJob(job):
         jobMeta["jobDescriptionPath"] = job_description_path
         jobMeta["jobPath"] = job_object.job_path
         jobMeta["workPath"] = job_object.work_path
-        jobMeta["LaunchCMD"] = luanch_cmd
+        jobMeta["LaunchCMD"] = launch_cmd
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
         dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 887716d90..6276a6018 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -116,5 +116,4 @@ def generate_job_description(self, job):
         for pod in pods:
             job_description = pod_template.generate_pod_yaml(pod)
             job_description_list.append(job_description)
-        job_description = "\n---\n".join(job_description_list)
-        return {"job_description": job_description, "luanch_cmd": luanch_cmd}, None
+        return {"job_descriptions": job_description_list, "launch_cmd": luanch_cmd}, None
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index ed2ab2d98..976a3524e 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -158,4 +158,4 @@ def test_generate_job_description(self):
 
         self.assertFalse(error)
         self.assertTrue(dict, type(ret))
-        self.assertTrue(["job_description", "launch_cmd"])
+        self.assertListEqual(["job_descriptions", "launch_cmd"], ret.keys())

From ea3ce4e02739a046f3403e25589559f22f3c5859 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Mon, 1 Jul 2019 08:58:56 +0000
Subject: [PATCH 024/194] extract job_deployer for create/delete pod

---
 src/ClusterManager/job_deployer.py      | 54 +++++++++++++++++++++++++
 src/ClusterManager/job_manager.py       | 16 ++++----
 src/ClusterManager/test_job_deployer.py | 49 ++++++++++++++++++++++
 3 files changed, 111 insertions(+), 8 deletions(-)
 create mode 100644 src/ClusterManager/job_deployer.py
 create mode 100644 src/ClusterManager/test_job_deployer.py

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
new file mode 100644
index 000000000..d3ee56b64
--- /dev/null
+++ b/src/ClusterManager/job_deployer.py
@@ -0,0 +1,54 @@
+import yaml
+import os
+import logging
+from kubernetes import client, config
+
+
+class JobDeployer:
+
+    def __init__(self):
+        # The config will be loaded from default location.
+        config.load_kube_config()
+        self.v1 = client.CoreV1Api()
+        self.namespace = "default"
+        self.pretty = "pretty_example"
+
+    def create_pod(self, body, dry_run=None):
+        api_response = self.v1.create_namespaced_pod(
+            namespace=self.namespace,
+            body=body,
+            pretty=self.pretty,
+            dry_run=dry_run,
+        )
+        return api_response
+
+    def delete_pod(self, name, dry_run=None):
+        api_response = self.v1.delete_namespaced_pod(
+            name=name,
+            namespace=self.namespace,
+            pretty=self.pretty,
+            body=client.V1DeleteOptions(),
+            dry_run=dry_run,
+            #  grace_period_seconds=grace_period_seconds,
+            #  orphan_dependents=orphan_dependents,
+            #  propagation_policy=propagation_policy,
+        )
+        pass
+
+    def cleanup_pods(self, pods):
+        for pod in pods:
+            try:
+                self.delete_pod(pod["metadata"]["name"])
+            except Exception as e:
+                logging.warning(e, exc_info=True)
+
+    def create_pods(self, pods):
+        # TODO instead of delete, we could check update existiong ones. During refactoring, keeping the old way.
+        self.cleanup_pods(pods)
+        created = []
+        for pod in pods:
+            self.create_pod(pod)
+            pod_name = pod["metadata"]["name"]
+            created.append(pod_name)
+            logging.info("Successfully created pod: %s" % pod_name)
+        return created
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 4f4ce5785..9a878b9bf 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -38,6 +38,7 @@
 import logging.config
 from job import Job, JobSchema
 from pod_template import PodTemplate
+from job_deployer import JobDeployer
 
 
 def printlog(msg):
@@ -103,21 +104,20 @@ def SubmitRegularJob(job):
         job_description_list = ret["job_descriptions"]
         job_description = "\n---\n".join(job_description_list)
         launch_cmd = ret["launch_cmd"]
+
         job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
         local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
-
-        if os.path.isfile(local_jobDescriptionPath):
-            output = k8sUtils.kubectl_delete(local_jobDescriptionPath)
-
         if not os.path.exists(os.path.dirname(local_jobDescriptionPath)):
             os.makedirs(os.path.dirname(os.path.realpath(local_jobDescriptionPath)))
         with open(local_jobDescriptionPath, 'w') as f:
             f.write(job_description)
 
-        output = k8sUtils.kubectl_create(local_jobDescriptionPath)
-        logging.info("Submitted job %s to k8s, returned with status %s" % (job_object.job_id, output))
-
-        ret["output"] = output
+        job_deployer = JobDeployer()
+        try:
+            ret["output"] = job_deployer.create_pods(job_description_list)
+        except Exception as e:
+            ret["output"] = "Error: %s" % e.message
+            logging.error(e, exc_info=True)
 
         ret["jobId"] = job_object.job_id
 
diff --git a/src/ClusterManager/test_job_deployer.py b/src/ClusterManager/test_job_deployer.py
new file mode 100644
index 000000000..f0731cabd
--- /dev/null
+++ b/src/ClusterManager/test_job_deployer.py
@@ -0,0 +1,49 @@
+import unittest
+import kubernetes
+import yaml
+from kubernetes.client.rest import ApiException
+
+from job_deployer import JobDeployer
+
+
+class TestJobDeployer(unittest.TestCase):
+
+    def create_job_deployer(self):
+        job_deployer = JobDeployer()
+        self.assertIsNotNone(job_deployer)
+        return job_deployer
+
+    def test_create_pod(self):
+        job_deployer = self.create_job_deployer()
+        raw_yaml = """
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-pod
+spec:
+  containers:
+  - name: busybox
+    image: busybox
+    args:
+    - sleep
+    - "1000000"
+    """
+        body = yaml.load(raw_yaml)
+
+        # with self.assertRaises(ApiException):
+        job_deployer.create_pod(body)
+
+    def test_delete_pod(self):
+        job_deployer = self.create_job_deployer()
+
+        job_deployer.delete_pod("test-pod")
+
+    def test_cleanup_pods(self):
+        job_deployer = self.create_job_deployer()
+
+        pods = [
+            {"metadata": {"name": "pod-1"}},
+            {"metadata": {"name": "pod-2"}},
+        ]
+
+        job_deployer.cleanup_pods(pods)

From 6e97c77800154bdf86f841b29824e615e76db230 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 2 Jul 2019 07:48:50 +0000
Subject: [PATCH 025/194] better funciton name for PodTemlate

---
 src/ClusterManager/job_manager.py       | 12 +++++-------
 src/ClusterManager/pod_template.py      | 16 +++++++++-------
 src/ClusterManager/test_pod_template.py | 23 +++++++++++------------
 3 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 9a878b9bf..3b553098b 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -96,15 +96,12 @@ def SubmitRegularJob(job):
 
         enable_custom_scheduler = job_object.is_custom_scheduler_enabled()
         pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
-        ret, error = pod_template.generate_job_description(job_object)
+        pods, error = pod_template.generate_pods(job_object)
         if error:
             dataHandler.SetJobError(job_object.job_id, "ERROR: %s" % error)
             return False
 
-        job_description_list = ret["job_descriptions"]
-        job_description = "\n---\n".join(job_description_list)
-        launch_cmd = ret["launch_cmd"]
-
+        job_description = "\n---\n".join([yaml.dump(pod) for pod in pods])
         job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
         local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
         if not os.path.exists(os.path.dirname(local_jobDescriptionPath)):
@@ -114,7 +111,7 @@ def SubmitRegularJob(job):
 
         job_deployer = JobDeployer()
         try:
-            ret["output"] = job_deployer.create_pods(job_description_list)
+            ret["output"] = job_deployer.create_pods(pods)
         except Exception as e:
             ret["output"] = "Error: %s" % e.message
             logging.error(e, exc_info=True)
@@ -129,7 +126,8 @@ def SubmitRegularJob(job):
         jobMeta["jobDescriptionPath"] = job_description_path
         jobMeta["jobPath"] = job_object.job_path
         jobMeta["workPath"] = job_object.work_path
-        jobMeta["LaunchCMD"] = launch_cmd
+        # the command of the first container
+        jobMeta["LaunchCMD"] = pods[0]["spec"]["containers"][0]["command"]
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
         dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 6276a6018..68ed8a123 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -1,4 +1,5 @@
 import json
+import yaml
 from jinja2 import Template
 from job import Job
 
@@ -8,7 +9,7 @@ def __init__(self, template, enable_custom_scheduler):
         self.template = template
         self.enable_custom_scheduler = enable_custom_scheduler
 
-    def generate_pod_yaml(self, pod):
+    def generate_pod(self, pod):
         assert(isinstance(self.template, Template))
         if self.enable_custom_scheduler:
             if "useGPUTopology" in pod and pod["useGPUTopology"]:
@@ -42,9 +43,10 @@ def generate_pod_yaml(self, pod):
         if "gpuType" in pod:
             pod["nodeSelector"]["gpuType"] = pod["gpuType"]
 
-        return self.template.render(job=pod)
+        pod_yaml = self.template.render(job=pod)
+        return yaml.load(pod_yaml)
 
-    def generate_job_description(self, job):
+    def generate_pods(self, job):
         """
         Return (job_description, errors)
         """
@@ -112,8 +114,8 @@ def generate_job_description(self, job):
         enable_custom_scheduler = job.is_custom_scheduler_enabled()
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
 
-        job_description_list = []
+        k8s_pods = []
         for pod in pods:
-            job_description = pod_template.generate_pod_yaml(pod)
-            job_description_list.append(job_description)
-        return {"job_descriptions": job_description_list, "launch_cmd": luanch_cmd}, None
+            k8s_pod = pod_template.generate_pod(pod)
+            k8s_pods.append(k8s_pod)
+        return k8s_pods, None
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index 976a3524e..a49c62768 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -27,8 +27,7 @@ def test_pod_template_without_custer_scheduler(self):
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
 
         pod = {"resourcegpu": 2}
-        pod_yaml = pod_template.generate_pod_yaml(pod)
-        data = yaml.load(pod_yaml)
+        data = pod_template.generate_pod(pod)
 
         # not eanbled custom scheduler, set the resource limits: spec.containers[].resources.limits
         self.assertEqual(pod["resourcegpu"], data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
@@ -44,8 +43,7 @@ def test_pod_template_with_custom_scheduler(self):
             "podName": "790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0",
             "resourcegpu": gpu_num,
         }
-        pod_yaml = pod_template.generate_pod_yaml(pod)
-        data = yaml.load(pod_yaml)
+        data = pod_template.generate_pod(pod)
 
         # eanbled custom scheduler would clear the resource limits: spec.containers[].resources.limits
         self.assertEqual(0, data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
@@ -67,8 +65,7 @@ def test_pod_template_with_custom_scheduler_use_topology(self):
             "resourcegpu": gpu_num,
             "useGPUTopology": True
         }
-        pod_yaml = pod_template.generate_pod_yaml(pod)
-        data = yaml.load(pod_yaml)
+        data = pod_template.generate_pod(pod)
 
         # eanbled custom scheduler, clear the resource limits: spec.containers[].resources.limits
         self.assertEqual(0, data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
@@ -94,18 +91,18 @@ def test_pod_template_with_custom_scheduler_use_topology(self):
         # enabled topology
         self.assertEqual(1, device_annotation["requests"]["alpha.gpu/gpu-generate-topology"])
 
-    def test_generate_job_description_missing_required_params(self):
+    def test_generate_pods_missing_required_params(self):
         enable_custom_scheduler = True
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
 
         job.params = {}
-        job_description, error = pod_template.generate_job_description(job)
+        job_description, error = pod_template.generate_pods(job)
 
         self.assertIsNone(job_description)
         self.assertTrue(error)
         self.assertEqual("Missing required parameters!", error)
 
-    def test_generate_job_description(self):
+    def test_generate_pods(self):
         enable_custom_scheduler = True
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
 
@@ -154,8 +151,10 @@ def test_generate_job_description(self):
             "ssh": True
         }
 
-        ret, error = pod_template.generate_job_description(job)
+        pods, error = pod_template.generate_pods(job)
 
         self.assertFalse(error)
-        self.assertTrue(dict, type(ret))
-        self.assertListEqual(["job_descriptions", "launch_cmd"], ret.keys())
+        # generate list of pod yamls
+        self.assertTrue(list, type(pods))
+        self.assertEqual(1, len(pods))
+        self.assertIsNotNone(pods[0]["spec"]["containers"][0]["command"])

From 14587eeef34320156d66492a6f73f724945ca511 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 2 Jul 2019 07:50:10 +0000
Subject: [PATCH 026/194] better log format

---
 src/ClusterManager/job_deployer.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index d3ee56b64..b9f502754 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -38,9 +38,10 @@ def delete_pod(self, name, dry_run=None):
     def cleanup_pods(self, pods):
         for pod in pods:
             try:
-                self.delete_pod(pod["metadata"]["name"])
+                pod_name = pod["metadata"]["name"]
+                self.delete_pod(pod_name)
             except Exception as e:
-                logging.warning(e, exc_info=True)
+                logging.warning("Delete pod failed: %s!" % pod_name, exc_info=True)
 
     def create_pods(self, pods):
         # TODO instead of delete, we could check update existiong ones. During refactoring, keeping the old way.
@@ -50,5 +51,5 @@ def create_pods(self, pods):
             self.create_pod(pod)
             pod_name = pod["metadata"]["name"]
             created.append(pod_name)
-            logging.info("Successfully created pod: %s" % pod_name)
+            logging.info("Create pod succeed: %s" % pod_name)
         return created

From 579e304c2c183245ce8a9794d303b7795f4cd3e8 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 2 Jul 2019 07:55:54 +0000
Subject: [PATCH 027/194] refine log

---
 src/ClusterManager/job_manager.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 3b553098b..a703e3006 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -132,9 +132,7 @@ def SubmitRegularJob(job):
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
         dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
     except Exception as e:
-        logging.error("Submit job failed: %s" % job)
-        traceback.print_exc()
-        logging.error(e)
+        logging.error("Submit job failed: %s" % job, exc_info=True)
         ret["error"] = str(e)
         retries = dataHandler.AddandGetJobRetries(job["jobId"])
         if retries >= 5:

From dcbcf33e9d726703dfcc17920f1ae112dad73eb0 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 2 Jul 2019 08:41:15 +0000
Subject: [PATCH 028/194] refactoring dist job

---
 src/ClusterManager/job_manager.py | 123 ++++++++++++++++--------------
 1 file changed, 66 insertions(+), 57 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index a703e3006..d4f40e6e2 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -147,52 +147,76 @@ def SubmitPSDistJob(job):
     dataHandler = DataHandler()
 
     try:
+        job["cluster"] = config
+        job_object, errors = JobSchema().load(job)
+        # TODO assert job_object is a Job
+        assert(isinstance(job_object, Job))
+
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
+
+        job_object.params = json.loads(base64.b64decode(job["jobParams"]))
+
+        # TODO
         jobParams["rest-api"] = config["rest-api"]
+
         distJobParams = {}
         distJobParams["ps"] = []
         distJobParams["worker"] = []
+
+        # TODO
         assignedRack = None
         if len(config["racks"]) > 0:
             assignedRack = random.choice(config["racks"])
 
-        userAlias = getAlias(jobParams["userName"])
+        # inject gid, uid and user
+        # TODO it should return only one entry
+        user_info = dataHandler.GetIdentityInfo(job_object.params["userName"])[0]
+        jobParams["gid"] = user_info["gid"]
+        jobParams["uid"] = user_info["uid"]
+        jobParams["user"] = job_object.get_alias()
+
+        userAlias = job_object.get_alias()
         jobParams["user_email"] = jobParams["userName"]
 
-        jobParams["homeFolderHostpath"] = os.path.join(config["storage-mount-path"], GetWorkPath(userAlias))
+        job_object.job_path = jobParams["jobPath"]
+        jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
+
+        if any(required_field not in jobParams for required_field in
+               [
+                   "jobtrainingtype",
+                   "jobName",
+                   "jobPath",
+                   "workPath",
+                   "dataPath",
+                   "cmd",
+                   "userId",
+                   "resourcegpu",
+                   "userName",
+               ]):
+            return None, "Missing required parameters!"
 
         if jobParams["jobtrainingtype"] == "PSDistJob":
             jobDescriptionList = []
-            nums = {"ps":int(jobParams["numps"]),"worker":int(jobParams["numpsworker"])}
-            for role in ["ps","worker"]:
+            nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
+            for role in ["ps", "worker"]:
                 for i in range(nums[role]):
-                    distJobParam=copy.deepcopy(jobParams)
-                    distJobParam["distId"] = "%s%d" % (role,i)
+                    distJobParam = copy.deepcopy(jobParams)
+                    distJobParam["distId"] = "%s%d" % (role, i)
                     distJobParam["distRole"] = role
                     distJobParam["distRoleIdx"] = i
 
-                    if "jobPath" not in distJobParam or len(distJobParam["jobPath"].strip()) == 0:
-                        dataHandler.SetJobError(distJobParam["jobId"],"ERROR: job-path does not exist")
-                        return False
-                    if "workPath" not in distJobParam or len(distJobParam["workPath"].strip()) == 0:
-                        dataHandler.SetJobError(distJobParam["jobId"],"ERROR: work-path does not exist")
-                        return False
-                    #if "dataPath" not in distJobParam or len(distJobParam["dataPath"].strip()) == 0:
-                    #    dataHandler.SetJobError(distJobParam["jobId"],"ERROR: data-path does not exist")
-                    #    return False
-                    distJobParam["distJobPath"] = os.path.join(distJobParam["jobPath"],distJobParam["distId"])
-                    jobPath,workPath,dataPath = GetStoragePath(distJobParam["distJobPath"],distJobParam["workPath"],distJobParam["dataPath"])
-
-                    localJobPath = os.path.join(config["storage-mount-path"],jobPath)
+                    # TODO
+                    distJobParam["distJobPath"] = os.path.join(job_object.job_path, distJobParam["distId"])
+                    jobPath = "work/" + distJobParam["distJobPath"]
+                    workPath = "work/" + distJobParam["workPath"]
+                    dataPath = "storage/" + distJobParam["dataPath"]
+
+                    localJobPath = os.path.join(config["storage-mount-path"], jobPath)
                     if not os.path.exists(localJobPath):
                         if "userId" in distJobParam:
-                            mkdirsAsUser(localJobPath,distJobParam["userId"])
+                            mkdirsAsUser(localJobPath, distJobParam["userId"])
                         else:
-                            mkdirsAsUser(localJobPath,0)
-
-                    # TODO ???
-                    if "cmd" not in distJobParam:
-                        distJobParam["cmd"] = ""
+                            mkdirsAsUser(localJobPath, 0)
 
 #change ssh folder permission here because the setup permission script in launch_ps_job function may have race condition with init_user.sh script. results in no such user error
                     if role == "ps":
@@ -238,7 +262,7 @@ def SubmitPSDistJob(job):
 
 echo "[DLWorkspace System]: All containers are ready, launching training job..."
 %s
-""" % (userAlias,userAlias,userAlias,userAlias,userAlias,distJobParam["jobPath"],jobParams["numpsworker"],distJobParam["cmd"])
+""" % (userAlias, userAlias, userAlias, userAlias, userAlias, distJobParam["jobPath"], jobParams["numpsworker"], distJobParam["cmd"])
                     else:
                         launchCMD = """
 while [ ! -f /opt/run_dist_job ]; do
@@ -255,24 +279,22 @@ def SubmitPSDistJob(job):
 sudo touch /job/WORKER_READY
 
 sleep infinity
-""" % (userAlias,userAlias,userAlias,userAlias,userAlias)
+""" % (userAlias, userAlias, userAlias, userAlias, userAlias)
 
-
-                    launchScriptPath = os.path.join(localJobPath,"launch-%s-%s%d.sh" % (distJobParam["jobId"],role,i))
+                    launchScriptPath = os.path.join(localJobPath, "launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i))
                     # TODO need to set up user for distribute jobs
                     with open(launchScriptPath, 'w') as f:
                         f.write(launchCMD)
                     f.close()
 
-
-                    launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"],role,i)
+                    launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i)
 
                     distJobParam["LaunchCMD"] = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
 
                     distJobParam["jobNameLabel"] = ''.join(e for e in distJobParam["jobName"] if e.isalnum())
                     ENV = Environment(loader=FileSystemLoader("/"))
 
-                    jobTempDir = os.path.join(config["root-path"],"Jobs_Templete")
+                    jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
                     jobTemp = os.path.join(jobTempDir, "DistJob.yaml.template")
 
                     distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
@@ -282,14 +304,13 @@ def SubmitPSDistJob(job):
                     if "mountpoints" not in distJobParam:
                         distJobParam["mountpoints"] = []
 
-                    distJobParam["mountpoints"].append({"name":"job","containerPath":"/job","hostPath":distJobParam["hostjobPath"]})
-                    distJobParam["mountpoints"].append({"name":"work","containerPath":"/work","hostPath":distJobParam["hostworkPath"]})
-                    distJobParam["mountpoints"].append({"name":"data","containerPath":"/data","hostPath":distJobParam["hostdataPath"]})
+                    distJobParam["mountpoints"].append({"name": "job", "containerPath": "/job", "hostPath": distJobParam["hostjobPath"]})
+                    distJobParam["mountpoints"].append({"name": "work", "containerPath": "/work", "hostPath": distJobParam["hostworkPath"]})
+                    distJobParam["mountpoints"].append({"name": "data", "containerPath": "/data", "hostPath": distJobParam["hostdataPath"]})
 
                     for idx in range(len(distJobParam["mountpoints"])):
                         if "name" not in distJobParam["mountpoints"][idx]:
-                            distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-","")
-
+                            distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
 
                     distJobParam["pod_ip_range"] = config["pod_ip_range"]
                     if "usefreeflow" in config:
@@ -300,13 +321,11 @@ def SubmitPSDistJob(job):
                     distJobParam["numworker"] = int(jobParams["numpsworker"])
                     distJobParam["numps"] = int(jobParams["numps"])
 
-
-
                     random.seed(datetime.datetime.now())
                     if "hostNetwork" in jobParams and jobParams["hostNetwork"]:
                         distJobParam["containerPort"] = random.randint(40000, 49999)
                     else:
-                        distJobParam["containerPort"] = int(random.random()*1000+3000)
+                        distJobParam["containerPort"] = int(random.random() * 1000 + 3000)
 
                     if assignedRack is not None:
                         if "nodeSelector" not in distJobParam:
@@ -318,13 +337,6 @@ def SubmitPSDistJob(job):
                             distJobParam["nodeSelector"] = {}
                         distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
 
-                    # inject gid, uid and user
-                    # TODO it should return only one entry
-                    user_info = dataHandler.GetIdentityInfo(jobParams["userName"])[0]
-                    distJobParam["gid"] = user_info["gid"]
-                    distJobParam["uid"] = user_info["uid"]
-                    distJobParam["user"] = userAlias
-
                     template = ENV.get_template(os.path.abspath(jobTemp))
                     job_description = template.render(job=distJobParam)
 
@@ -332,11 +344,9 @@ def SubmitPSDistJob(job):
 
                     distJobParams[role].append(distJobParam)
 
-
             jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
             jobDescription = "\n---\n".join(jobDescriptionList)
 
-
         jobDescriptionPath = os.path.join(config["storage-mount-path"], jobParams["jobDescriptionPath"])
         if not os.path.exists(os.path.dirname(os.path.realpath(jobDescriptionPath))):
             os.makedirs(os.path.dirname(os.path.realpath(jobDescriptionPath)))
@@ -352,14 +362,12 @@ def SubmitPSDistJob(job):
 
         ret["jobId"] = jobParams["jobId"]
 
-
         if "userName" not in jobParams:
             jobParams["userName"] = ""
 
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobStatus","scheduling")
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobDescriptionPath",jobParams["jobDescriptionPath"])
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobDescription",base64.b64encode(jobDescription))
-
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", jobParams["jobDescriptionPath"])
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(jobDescription))
 
         jobMeta = {}
         jobMeta["jobDescriptionPath"] = jobParams["jobDescriptionPath"]
@@ -370,7 +378,7 @@ def SubmitPSDistJob(job):
         jobMeta["distJobParams"] = distJobParams
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
-        dataHandler.UpdateJobTextField(jobParams["jobId"],"jobMeta",jobMetaStr)
+        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobMeta", jobMetaStr)
     except Exception as e:
         import traceback
         traceback.print_exc()
@@ -378,11 +386,12 @@ def SubmitPSDistJob(job):
         ret["error"] = str(e)
         retries = dataHandler.AddandGetJobRetries(jobParams["jobId"])
         if retries >= 5:
-            dataHandler.UpdateJobTextField(jobParams["jobId"],"jobStatus","error")
-            dataHandler.UpdateJobTextField(jobParams["jobId"],"errorMsg","Cannot submit job!" + str(e))
+            dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "error")
+            dataHandler.UpdateJobTextField(jobParams["jobId"], "errorMsg", "Cannot submit job!" + str(e))
     dataHandler.Close()
     return ret
 
+
 def KillJob(job, desiredState="killed"):
     dataHandler = DataHandler()
     result, detail = k8sUtils.GetJobStatus(job["jobId"])

From 1d4dccd1d5d446e60bfa50eb37a48526f63da1f1 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 3 Jul 2019 11:52:44 +0000
Subject: [PATCH 029/194] add default value: enable_custom_scheduler=False

---
 src/ClusterManager/pod_template.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 68ed8a123..c8000a4b3 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -5,7 +5,7 @@
 
 
 class PodTemplate():
-    def __init__(self, template, enable_custom_scheduler):
+    def __init__(self, template, enable_custom_scheduler=False):
         self.template = template
         self.enable_custom_scheduler = enable_custom_scheduler
 

From d7e2eb5c95aaa5fa6ccd4ba926b71e7e730be567 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 03:17:35 +0000
Subject: [PATCH 030/194] fix comments

---
 src/ClusterManager/pod_template.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index c8000a4b3..50c00c434 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -48,7 +48,7 @@ def generate_pod(self, pod):
 
     def generate_pods(self, job):
         """
-        Return (job_description, errors)
+        Return (pods, errors)
         """
 
         assert(isinstance(job, Job))

From be6b4498237096cb22cae76c03bfdb52ea7397a4 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 04:32:15 +0000
Subject: [PATCH 031/194] refine SubmitPSDistJob()

---
 src/ClusterManager/dist_pod_template.py |  78 ++++++++
 src/ClusterManager/job.py               |   8 +
 src/ClusterManager/job_manager.py       | 236 +++++++++---------------
 src/ClusterManager/test_job.py          |   5 +
 4 files changed, 178 insertions(+), 149 deletions(-)
 create mode 100644 src/ClusterManager/dist_pod_template.py

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
new file mode 100644
index 000000000..7ede35304
--- /dev/null
+++ b/src/ClusterManager/dist_pod_template.py
@@ -0,0 +1,78 @@
+import json
+import yaml
+from jinja2 import Template
+from job import Job
+
+
+class DistPodTemplate():
+    def __init__(self, template, enable_custom_scheduler=False):
+        self.template = template
+        self.enable_custom_scheduler = enable_custom_scheduler
+
+    @staticmethod
+    def generate_launch_cmd(role, user_alias, job_path, worker_num, cmd):
+        # change ssh folder permission here because the setup permission
+        #  script in launch_ps_job function may have race condition with init_user.sh script.
+        # results in no such user error
+        if role == "ps":
+            launchCMD = """
+#!/bin/bash
+echo "[DLWorkspace System]: Waiting for all containers are ready..."
+while [ ! -f /opt/run_dist_job ]; do
+    sleep 3
+done
+
+sudo chmod 600 -R /home/{0}/.ssh &>/dev/null;
+sudo chmod 700 /home/{0}/.ssh &>/dev/null;
+sudo chown -R {0} /home/{0}/.ssh &>/dev/null;
+
+sudo mkdir -p /root/.ssh  &>/dev/null ;
+sudo ln -s /home/{0}/.ssh/config /root/.ssh/config  &>/dev/null;
+sudo mkdir -p /opt  &>/dev/null;
+sudo ln -s /job/hostfile /opt/hostfile &>/dev/null;
+
+JOB_DIR='/home/{1}'
+WORKER_NUM={2}
+echo $JOB_DIR $WORKER_NUM
+
+all_workers_ready=false
+while [ "$all_workers_ready" != true ]
+do
+  # update it to false if any woker is not ready
+  all_workers_ready=true
+
+  for i in $(seq 0 $(( ${{WORKER_NUM}} - 1)) )
+  do
+    worker="worker${{i}}"
+    file="$JOB_DIR/${{worker}}/WORKER_READY"
+    #echo $file
+
+    if [ ! -f $file ]; then
+      echo "${{worker}} not ready!"
+      all_workers_ready=false
+      sleep 10
+    fi
+  done
+done
+
+echo "[DLWorkspace System]: All containers are ready, launching training job..."
+{3}
+""".format(user_alias, job_path, worker_num, cmd)
+        else:
+            launchCMD = """
+while [ ! -f /opt/run_dist_job ]; do
+    sleep 3
+done
+sudo chmod 600 -R /home/{0}/.ssh &>/dev/null;
+sudo chmod 700 /home/{0}/.ssh &>/dev/null;
+sudo chown -R {0} /home/{0}/.ssh  &>/dev/null;
+sudo mkdir -p /root/.ssh  &>/dev/null;
+sudo ln -s /home/{0}/.ssh/config /root/.ssh/config &>/dev/null;
+sudo mkdir -p /opt && sudo ln -s /job/hostfile /opt/hostfile  &>/dev/null;
+
+# TODO mark the worker as 'READY', better to change to '/pod/READY' later
+sudo touch /job/WORKER_READY
+
+sleep infinity
+""".format(user_alias)
+        return launchCMD
diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 18773679e..77a73aec3 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -1,5 +1,6 @@
 import sys
 import os
+import random
 from datetime import date
 from marshmallow import Schema, fields, pprint, post_load, validate
 from jinja2 import Environment, FileSystemLoader, Template
@@ -144,6 +145,13 @@ def get_pod_ip_range(self):
     def is_user_flow_enabled(self):
         return self._get_cluster_config("usefreeflow")
 
+    def get_rack(self):
+        racks = self._get_cluster_config("racks")
+        if racks is None or len(racks) == 0:
+            return None
+        # TODO why random.choice?
+        return random.choice(racks)
+
     def _get_cluster_config(self, key):
         if key in self.cluster:
             return self.cluster[key]
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index d4f40e6e2..f42426f2d 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -38,6 +38,7 @@
 import logging.config
 from job import Job, JobSchema
 from pod_template import PodTemplate
+from dist_pod_template import DistPodTemplate
 from job_deployer import JobDeployer
 
 
@@ -156,18 +157,6 @@ def SubmitPSDistJob(job):
 
         job_object.params = json.loads(base64.b64decode(job["jobParams"]))
 
-        # TODO
-        jobParams["rest-api"] = config["rest-api"]
-
-        distJobParams = {}
-        distJobParams["ps"] = []
-        distJobParams["worker"] = []
-
-        # TODO
-        assignedRack = None
-        if len(config["racks"]) > 0:
-            assignedRack = random.choice(config["racks"])
-
         # inject gid, uid and user
         # TODO it should return only one entry
         user_info = dataHandler.GetIdentityInfo(job_object.params["userName"])[0]
@@ -175,12 +164,6 @@ def SubmitPSDistJob(job):
         jobParams["uid"] = user_info["uid"]
         jobParams["user"] = job_object.get_alias()
 
-        userAlias = job_object.get_alias()
-        jobParams["user_email"] = jobParams["userName"]
-
-        job_object.job_path = jobParams["jobPath"]
-        jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
-
         if any(required_field not in jobParams for required_field in
                [
                    "jobtrainingtype",
@@ -194,158 +177,113 @@ def SubmitPSDistJob(job):
                    "userName",
                ]):
             return None, "Missing required parameters!"
+        assert(jobParams["jobtrainingtype"] == "PSDistJob")
 
-        if jobParams["jobtrainingtype"] == "PSDistJob":
-            jobDescriptionList = []
-            nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
-            for role in ["ps", "worker"]:
-                for i in range(nums[role]):
-                    distJobParam = copy.deepcopy(jobParams)
-                    distJobParam["distId"] = "%s%d" % (role, i)
-                    distJobParam["distRole"] = role
-                    distJobParam["distRoleIdx"] = i
-
-                    # TODO
-                    distJobParam["distJobPath"] = os.path.join(job_object.job_path, distJobParam["distId"])
-                    jobPath = "work/" + distJobParam["distJobPath"]
-                    workPath = "work/" + distJobParam["workPath"]
-                    dataPath = "storage/" + distJobParam["dataPath"]
-
-                    localJobPath = os.path.join(config["storage-mount-path"], jobPath)
-                    if not os.path.exists(localJobPath):
-                        if "userId" in distJobParam:
-                            mkdirsAsUser(localJobPath, distJobParam["userId"])
-                        else:
-                            mkdirsAsUser(localJobPath, 0)
-
-#change ssh folder permission here because the setup permission script in launch_ps_job function may have race condition with init_user.sh script. results in no such user error
-                    if role == "ps":
-                        launchCMD = """
-#!/bin/bash
-echo "[DLWorkspace System]: Waiting for all containers are ready..."
-while [ ! -f /opt/run_dist_job ]; do
-    sleep 3
-done
-
-sudo chmod 600 -R /home/%s/.ssh &>/dev/null;
-sudo chmod 700 /home/%s/.ssh &>/dev/null;
-sudo chown -R %s /home/%s/.ssh &>/dev/null;
-
-sudo mkdir -p /root/.ssh  &>/dev/null ;
-sudo ln -s /home/%s/.ssh/config /root/.ssh/config  &>/dev/null;
-sudo mkdir -p /opt  &>/dev/null;
-sudo ln -s /job/hostfile /opt/hostfile &>/dev/null;
-
-JOB_DIR='/home/%s'
-WORKER_NUM=%s
-echo $JOB_DIR $WORKER_NUM
-
-all_workers_ready=false
-while [ "$all_workers_ready" != true ]
-do
-  # update it to false if any woker is not ready
-  all_workers_ready=true
-
-  for i in $(seq 0 $(( ${WORKER_NUM} - 1)) )
-  do
-    worker="worker${i}"
-    file="$JOB_DIR/${worker}/WORKER_READY"
-    #echo $file
-
-    if [ ! -f $file ]; then
-      echo "${worker} not ready!"
-      all_workers_ready=false
-      sleep 10
-    fi
-  done
-done
-
-echo "[DLWorkspace System]: All containers are ready, launching training job..."
-%s
-""" % (userAlias, userAlias, userAlias, userAlias, userAlias, distJobParam["jobPath"], jobParams["numpsworker"], distJobParam["cmd"])
-                    else:
-                        launchCMD = """
-while [ ! -f /opt/run_dist_job ]; do
-    sleep 3
-done
-sudo chmod 600 -R /home/%s/.ssh &>/dev/null;
-sudo chmod 700 /home/%s/.ssh &>/dev/null;
-sudo chown -R %s /home/%s/.ssh  &>/dev/null;
-sudo mkdir -p /root/.ssh  &>/dev/null;
-sudo ln -s /home/%s/.ssh/config /root/.ssh/config &>/dev/null;
-sudo mkdir -p /opt && sudo ln -s /job/hostfile /opt/hostfile  &>/dev/null;
+        jobParams["rest-api"] = job_object.get_rest_api_url()
+        assignedRack = job_object.get_rack()
 
-# TODO mark the worker as 'READY', better to change to '/pod/READY' later
-sudo touch /job/WORKER_READY
+        distJobParams = {}
+        distJobParams["ps"] = []
+        distJobParams["worker"] = []
 
-sleep infinity
-""" % (userAlias, userAlias, userAlias, userAlias, userAlias)
+        userAlias = job_object.get_alias()
+        jobParams["user_email"] = jobParams["userName"]
 
-                    launchScriptPath = os.path.join(localJobPath, "launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i))
-                    # TODO need to set up user for distribute jobs
-                    with open(launchScriptPath, 'w') as f:
-                        f.write(launchCMD)
-                    f.close()
+        job_object.job_path = jobParams["jobPath"]
+        jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
 
-                    launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i)
+        jobDescriptionList = []
+        nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
+        for role in ["ps", "worker"]:
+            for i in range(nums[role]):
+                distJobParam = copy.deepcopy(jobParams)
+                distJobParam["distId"] = "%s%d" % (role, i)
+                distJobParam["distRole"] = role
+                distJobParam["distRoleIdx"] = i
+
+                # TODO
+                distJobParam["distJobPath"] = os.path.join(job_object.job_path, distJobParam["distId"])
+                jobPath = "work/" + distJobParam["distJobPath"]
+                workPath = "work/" + distJobParam["workPath"]
+                dataPath = "storage/" + distJobParam["dataPath"]
+
+                localJobPath = os.path.join(config["storage-mount-path"], jobPath)
+                if not os.path.exists(localJobPath):
+                    if "userId" in distJobParam:
+                        mkdirsAsUser(localJobPath, distJobParam["userId"])
+                    else:
+                        mkdirsAsUser(localJobPath, 0)
 
-                    distJobParam["LaunchCMD"] = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
+                job_path = distJobParam["jobPath"]
+                worker_num = distJobParam["numpsworker"]
+                cmd = distJobParam["cmd"]
+                launchCMD = DistPodTemplate.generate_launch_cmd(role, userAlias, job_path, worker_num, cmd)
 
-                    distJobParam["jobNameLabel"] = ''.join(e for e in distJobParam["jobName"] if e.isalnum())
-                    ENV = Environment(loader=FileSystemLoader("/"))
+                launchScriptPath = os.path.join(localJobPath, "launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i))
+                # TODO need to set up user for distribute jobs
+                with open(launchScriptPath, 'w') as f:
+                    f.write(launchCMD)
+                f.close()
 
-                    jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
-                    jobTemp = os.path.join(jobTempDir, "DistJob.yaml.template")
+                launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i)
 
-                    distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
-                    distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
-                    distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
+                distJobParam["LaunchCMD"] = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
 
-                    if "mountpoints" not in distJobParam:
-                        distJobParam["mountpoints"] = []
+                distJobParam["jobNameLabel"] = ''.join(e for e in distJobParam["jobName"] if e.isalnum())
+                ENV = Environment(loader=FileSystemLoader("/"))
 
-                    distJobParam["mountpoints"].append({"name": "job", "containerPath": "/job", "hostPath": distJobParam["hostjobPath"]})
-                    distJobParam["mountpoints"].append({"name": "work", "containerPath": "/work", "hostPath": distJobParam["hostworkPath"]})
-                    distJobParam["mountpoints"].append({"name": "data", "containerPath": "/data", "hostPath": distJobParam["hostdataPath"]})
+                jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
+                jobTemp = os.path.join(jobTempDir, "DistJob.yaml.template")
 
-                    for idx in range(len(distJobParam["mountpoints"])):
-                        if "name" not in distJobParam["mountpoints"][idx]:
-                            distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
+                distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
+                distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
+                distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
 
-                    distJobParam["pod_ip_range"] = config["pod_ip_range"]
-                    if "usefreeflow" in config:
-                        distJobParam["usefreeflow"] = config["usefreeflow"]
-                    else:
-                        distJobParam["usefreeflow"] = False
+                if "mountpoints" not in distJobParam:
+                    distJobParam["mountpoints"] = []
 
-                    distJobParam["numworker"] = int(jobParams["numpsworker"])
-                    distJobParam["numps"] = int(jobParams["numps"])
+                distJobParam["mountpoints"].append({"name": "job", "containerPath": "/job", "hostPath": distJobParam["hostjobPath"]})
+                distJobParam["mountpoints"].append({"name": "work", "containerPath": "/work", "hostPath": distJobParam["hostworkPath"]})
+                distJobParam["mountpoints"].append({"name": "data", "containerPath": "/data", "hostPath": distJobParam["hostdataPath"]})
 
-                    random.seed(datetime.datetime.now())
-                    if "hostNetwork" in jobParams and jobParams["hostNetwork"]:
-                        distJobParam["containerPort"] = random.randint(40000, 49999)
-                    else:
-                        distJobParam["containerPort"] = int(random.random() * 1000 + 3000)
+                for idx in range(len(distJobParam["mountpoints"])):
+                    if "name" not in distJobParam["mountpoints"][idx]:
+                        distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
+
+                distJobParam["pod_ip_range"] = config["pod_ip_range"]
+                if "usefreeflow" in config:
+                    distJobParam["usefreeflow"] = config["usefreeflow"]
+                else:
+                    distJobParam["usefreeflow"] = False
+
+                distJobParam["numworker"] = int(jobParams["numpsworker"])
+                distJobParam["numps"] = int(jobParams["numps"])
+
+                random.seed(datetime.datetime.now())
+                if "hostNetwork" in jobParams and jobParams["hostNetwork"]:
+                    distJobParam["containerPort"] = random.randint(40000, 49999)
+                else:
+                    distJobParam["containerPort"] = int(random.random() * 1000 + 3000)
 
-                    if assignedRack is not None:
-                        if "nodeSelector" not in distJobParam:
-                            distJobParam["nodeSelector"] = {}
-                        distJobParam["nodeSelector"]["rack"] = assignedRack
+                if assignedRack is not None:
+                    if "nodeSelector" not in distJobParam:
+                        distJobParam["nodeSelector"] = {}
+                    distJobParam["nodeSelector"]["rack"] = assignedRack
 
-                    if "gpuType" in distJobParam:
-                        if "nodeSelector" not in distJobParam:
-                            distJobParam["nodeSelector"] = {}
-                        distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
+                if "gpuType" in distJobParam:
+                    if "nodeSelector" not in distJobParam:
+                        distJobParam["nodeSelector"] = {}
+                    distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
 
-                    template = ENV.get_template(os.path.abspath(jobTemp))
-                    job_description = template.render(job=distJobParam)
+                template = ENV.get_template(os.path.abspath(jobTemp))
+                job_description = template.render(job=distJobParam)
 
-                    jobDescriptionList.append(job_description)
+                jobDescriptionList.append(job_description)
 
-                    distJobParams[role].append(distJobParam)
+                distJobParams[role].append(distJobParam)
 
-            jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
-            jobDescription = "\n---\n".join(jobDescriptionList)
+        jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
+        jobDescription = "\n---\n".join(jobDescriptionList)
 
         jobDescriptionPath = os.path.join(config["storage-mount-path"], jobParams["jobDescriptionPath"])
         if not os.path.exists(os.path.dirname(os.path.realpath(jobDescriptionPath))):
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 3bc9516c1..02672bb23 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -181,3 +181,8 @@ def test_get_rest_api_url(self):
         job = self.create_a_job()
 
         self.assertEqual("http://faked.uri/", job.get_rest_api_url())
+
+    def test_get_rack(self):
+        job = self.create_a_job()
+
+        self.assertEqual(None, job.get_rack())

From 412e3e302e8e98830cdffa6bf38e4dbd47c345a7 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 05:42:40 +0000
Subject: [PATCH 032/194] a better func name

---
 src/ClusterManager/job.py          | 2 +-
 src/ClusterManager/pod_template.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 77a73aec3..d889153aa 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -142,7 +142,7 @@ def get_rest_api_url(self):
     def get_pod_ip_range(self):
         return self._get_cluster_config("pod_ip_range")
 
-    def is_user_flow_enabled(self):
+    def is_freeflow_enabled(self):
         return self._get_cluster_config("usefreeflow")
 
     def get_rack(self):
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 50c00c434..aece37371 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -87,7 +87,7 @@ def generate_pods(self, job):
         params["homeFolderHostpath"] = job.get_homefolder_hostpath()
 
         params["pod_ip_range"] = job.get_pod_ip_range()
-        params["usefreeflow"] = job.is_user_flow_enabled()
+        params["usefreeflow"] = job.is_freeflow_enabled()
 
         # assign parameters to generate the pod yaml
         params["LaunchCMD"] = luanch_cmd

From e4b0c2024193ec0a668afdf80187cacb21020077 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 06:29:01 +0000
Subject: [PATCH 033/194] dist job: add label sshPort

---
 src/ClusterManager/job_manager.py       | 30 ++++++++++++-------------
 src/Jobs_Templete/DistJob.yaml.template |  4 +---
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index f42426f2d..353160685 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -197,17 +197,14 @@ def SubmitPSDistJob(job):
         for role in ["ps", "worker"]:
             for i in range(nums[role]):
                 distJobParam = copy.deepcopy(jobParams)
+
                 distJobParam["distId"] = "%s%d" % (role, i)
                 distJobParam["distRole"] = role
                 distJobParam["distRoleIdx"] = i
 
                 # TODO
                 distJobParam["distJobPath"] = os.path.join(job_object.job_path, distJobParam["distId"])
-                jobPath = "work/" + distJobParam["distJobPath"]
-                workPath = "work/" + distJobParam["workPath"]
-                dataPath = "storage/" + distJobParam["dataPath"]
-
-                localJobPath = os.path.join(config["storage-mount-path"], jobPath)
+                localJobPath = os.path.join(config["storage-mount-path"], "work/", distJobParam["distJobPath"])
                 if not os.path.exists(localJobPath):
                     if "userId" in distJobParam:
                         mkdirsAsUser(localJobPath, distJobParam["userId"])
@@ -235,6 +232,10 @@ def SubmitPSDistJob(job):
                 jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
                 jobTemp = os.path.join(jobTempDir, "DistJob.yaml.template")
 
+                jobPath = "work/" + distJobParam["distJobPath"]
+                workPath = "work/" + distJobParam["workPath"]
+                dataPath = "storage/" + distJobParam["dataPath"]
+
                 distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
                 distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
                 distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
@@ -250,20 +251,17 @@ def SubmitPSDistJob(job):
                     if "name" not in distJobParam["mountpoints"][idx]:
                         distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
 
-                distJobParam["pod_ip_range"] = config["pod_ip_range"]
-                if "usefreeflow" in config:
-                    distJobParam["usefreeflow"] = config["usefreeflow"]
-                else:
-                    distJobParam["usefreeflow"] = False
+                distJobParam["pod_ip_range"] = job_object.get_pod_ip_range()
+                distJobParam["usefreeflow"] = job_object.is_freeflow_enabled()
 
                 distJobParam["numworker"] = int(jobParams["numpsworker"])
                 distJobParam["numps"] = int(jobParams["numps"])
 
                 random.seed(datetime.datetime.now())
                 if "hostNetwork" in jobParams and jobParams["hostNetwork"]:
-                    distJobParam["containerPort"] = random.randint(40000, 49999)
+                    distJobParam["sshPort"] = random.randint(40000, 49999)
                 else:
-                    distJobParam["containerPort"] = int(random.random() * 1000 + 3000)
+                    distJobParam["sshPort"] = int(random.random() * 1000 + 3000)
 
                 if assignedRack is not None:
                     if "nodeSelector" not in distJobParam:
@@ -545,9 +543,9 @@ def launch_ps_dist_job(jobParams):
     # setup ssh server
     for [idx, pod] in enumerate(pods["items"]):
         pod_name = pod["metadata"]["name"]
-        dist_port = pod["metadata"]["labels"]["distPort"]
+        ssh_port = pod["metadata"]["labels"]["sshPort"]
         # quit if can't setup ssh server
-        ssh_port = start_ssh_server(pod_name, user_name, host_network, dist_port)
+        ssh_port = start_ssh_server(pod_name, user_name, host_network, ssh_port)
 
     # generate ssh config
     ssh_config = """
@@ -561,13 +559,13 @@ def launch_ps_dist_job(jobParams):
     sshconfigstr = ""
     for [idx, pod] in enumerate(pods["items"]):
         pod_ip = pod["status"]["podIP"]
-        dist_port = pod["metadata"]["labels"]["distPort"]
+        ssh_port = pod["metadata"]["labels"]["sshPort"]
         role = pod["metadata"]["labels"]["distRole"]
         role_idx = pod["metadata"]["labels"]["distRoleIdx"]
 
         # TODO hostNetwork
         if host_network:
-            sshconfigstr += (ssh_config % (role + "-"+str(role_idx), pod_ip, str(dist_port), user_name) + "\n")
+            sshconfigstr += (ssh_config % (role + "-"+str(role_idx), pod_ip, str(ssh_port), user_name) + "\n")
         else:
             sshconfigstr += (ssh_config % (role + "-"+str(role_idx), pod_ip, 22, user_name) + "\n")
 
diff --git a/src/Jobs_Templete/DistJob.yaml.template b/src/Jobs_Templete/DistJob.yaml.template
index 816fd50ef..8c9267934 100755
--- a/src/Jobs_Templete/DistJob.yaml.template
+++ b/src/Jobs_Templete/DistJob.yaml.template
@@ -8,7 +8,7 @@ metadata:
     jobName: {{ job["jobNameLabel"] }}
     distRole: {{ job["distRole"] }}
     distRoleIdx: "{{ job["distRoleIdx"] }}"
-    distPort: "{{job["containerPort"]}}"
+    sshPort: "{{job["sshPort"]}}"
     userName: {{ job["user"] }}
     vcName: {{ job["vcName"] }}
   {% if "gpuType" in job %}
@@ -50,8 +50,6 @@ spec:
         - IPC_LOCK
         - SYS_ADMIN
     ports:
-    - containerPort: {{job["containerPort"]}}
-      hostPort: {{job["containerPort"]}}
     {% if job["distRole"] =="worker" %}
     resources:
       limits:

From 1631366a0babaeab4ba11b069680e429da113b29 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 06:56:10 +0000
Subject: [PATCH 034/194] extract DistPodTemplate.generate_pod()

---
 src/ClusterManager/dist_pod_template.py | 80 +++++++++++++++++++++++++
 src/ClusterManager/job.py               | 11 +++-
 src/ClusterManager/job_manager.py       | 74 ++---------------------
 3 files changed, 96 insertions(+), 69 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 7ede35304..82e45d53e 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -1,8 +1,16 @@
+import os
+import sys
+import uuid
+import datetime
+import random
 import json
 import yaml
 from jinja2 import Template
 from job import Job
 
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+from config import config
+
 
 class DistPodTemplate():
     def __init__(self, template, enable_custom_scheduler=False):
@@ -76,3 +84,75 @@ def generate_launch_cmd(role, user_alias, job_path, worker_num, cmd):
 sleep infinity
 """.format(user_alias)
         return launchCMD
+
+    def generate_pod(self, job, pod):
+        # TODO
+        distJobParam = pod
+
+        role = distJobParam["distRole"]
+        localJobPath = os.path.join(config["storage-mount-path"], "work/", distJobParam["distJobPath"])
+        userAlias = distJobParam["user"]
+        idx = distJobParam["distRoleIdx"]
+
+        job_path = distJobParam["jobPath"]
+        worker_num = distJobParam["numpsworker"]
+        cmd = distJobParam["cmd"]
+        launchCMD = DistPodTemplate.generate_launch_cmd(role, userAlias, job_path, worker_num, cmd)
+
+        launchScriptPath = os.path.join(localJobPath, "launch-%s-%s%d.sh" % (distJobParam["jobId"], role, idx))
+        # TODO need to set up user for distribute jobs
+        with open(launchScriptPath, 'w') as f:
+            f.write(launchCMD)
+        f.close()
+
+        launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"], role, idx)
+
+        distJobParam["LaunchCMD"] = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
+
+        distJobParam["jobNameLabel"] = ''.join(e for e in distJobParam["jobName"] if e.isalnum())
+
+        jobPath = "work/" + distJobParam["distJobPath"]
+        workPath = "work/" + distJobParam["workPath"]
+        dataPath = "storage/" + distJobParam["dataPath"]
+
+        distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
+        distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
+        distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
+
+        if "mountpoints" not in distJobParam:
+            distJobParam["mountpoints"] = []
+
+        distJobParam["mountpoints"].append({"name": "job", "containerPath": "/job", "hostPath": distJobParam["hostjobPath"]})
+        distJobParam["mountpoints"].append({"name": "work", "containerPath": "/work", "hostPath": distJobParam["hostworkPath"]})
+        distJobParam["mountpoints"].append({"name": "data", "containerPath": "/data", "hostPath": distJobParam["hostdataPath"]})
+
+        for idx in range(len(distJobParam["mountpoints"])):
+            if "name" not in distJobParam["mountpoints"][idx]:
+                distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
+
+        distJobParam["pod_ip_range"] = job.get_pod_ip_range()
+        distJobParam["usefreeflow"] = job.is_freeflow_enabled()
+
+        distJobParam["numworker"] = int(distJobParam["numpsworker"])
+        distJobParam["numps"] = int(distJobParam["numps"])
+
+        random.seed(datetime.datetime.now())
+        if "hostNetwork" in distJobParam and distJobParam["hostNetwork"]:
+            distJobParam["sshPort"] = random.randint(40000, 49999)
+        else:
+            distJobParam["sshPort"] = int(random.random() * 1000 + 3000)
+
+        assignedRack = job.get_rack()
+        if assignedRack is not None:
+            if "nodeSelector" not in distJobParam:
+                distJobParam["nodeSelector"] = {}
+            distJobParam["nodeSelector"]["rack"] = assignedRack
+
+        if "gpuType" in distJobParam:
+            if "nodeSelector" not in distJobParam:
+                distJobParam["nodeSelector"] = {}
+            distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
+
+        job_description = self.template.render(job=distJobParam)
+
+        return job_description
diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index d889153aa..87d1e47ee 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -126,13 +126,22 @@ def generate_launch_script(self, path_to_save, user_id, gpu_num, user_script):
         return file_name
 
     def get_template(self):
-        """Return abosulutely path of job template file on the node."""
+        """Return jinja template."""
         path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "RegularJob.yaml.template"))
         ENV = Environment(loader=FileSystemLoader("/"))
         template = ENV.get_template(path)
         assert(isinstance(template, Template))
         return template
 
+    # TODO remove it later
+    def get_dist_template(self):
+        """Return jinja template."""
+        path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "DistJob.yaml.template"))
+        ENV = Environment(loader=FileSystemLoader("/"))
+        template = ENV.get_template(path)
+        assert(isinstance(template, Template))
+        return template
+
     def is_custom_scheduler_enabled(self):
         return self._get_cluster_config("kube_custom_scheduler")
 
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 353160685..8140c5add 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -180,7 +180,7 @@ def SubmitPSDistJob(job):
         assert(jobParams["jobtrainingtype"] == "PSDistJob")
 
         jobParams["rest-api"] = job_object.get_rest_api_url()
-        assignedRack = job_object.get_rack()
+
 
         distJobParams = {}
         distJobParams["ps"] = []
@@ -195,12 +195,12 @@ def SubmitPSDistJob(job):
         jobDescriptionList = []
         nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
         for role in ["ps", "worker"]:
-            for i in range(nums[role]):
+            for idx in range(nums[role]):
                 distJobParam = copy.deepcopy(jobParams)
 
-                distJobParam["distId"] = "%s%d" % (role, i)
+                distJobParam["distId"] = "%s%d" % (role, idx)
                 distJobParam["distRole"] = role
-                distJobParam["distRoleIdx"] = i
+                distJobParam["distRoleIdx"] = idx
 
                 # TODO
                 distJobParam["distJobPath"] = os.path.join(job_object.job_path, distJobParam["distId"])
@@ -211,70 +211,8 @@ def SubmitPSDistJob(job):
                     else:
                         mkdirsAsUser(localJobPath, 0)
 
-                job_path = distJobParam["jobPath"]
-                worker_num = distJobParam["numpsworker"]
-                cmd = distJobParam["cmd"]
-                launchCMD = DistPodTemplate.generate_launch_cmd(role, userAlias, job_path, worker_num, cmd)
-
-                launchScriptPath = os.path.join(localJobPath, "launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i))
-                # TODO need to set up user for distribute jobs
-                with open(launchScriptPath, 'w') as f:
-                    f.write(launchCMD)
-                f.close()
-
-                launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"], role, i)
-
-                distJobParam["LaunchCMD"] = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
-
-                distJobParam["jobNameLabel"] = ''.join(e for e in distJobParam["jobName"] if e.isalnum())
-                ENV = Environment(loader=FileSystemLoader("/"))
-
-                jobTempDir = os.path.join(config["root-path"], "Jobs_Templete")
-                jobTemp = os.path.join(jobTempDir, "DistJob.yaml.template")
-
-                jobPath = "work/" + distJobParam["distJobPath"]
-                workPath = "work/" + distJobParam["workPath"]
-                dataPath = "storage/" + distJobParam["dataPath"]
-
-                distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
-                distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
-                distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
-
-                if "mountpoints" not in distJobParam:
-                    distJobParam["mountpoints"] = []
-
-                distJobParam["mountpoints"].append({"name": "job", "containerPath": "/job", "hostPath": distJobParam["hostjobPath"]})
-                distJobParam["mountpoints"].append({"name": "work", "containerPath": "/work", "hostPath": distJobParam["hostworkPath"]})
-                distJobParam["mountpoints"].append({"name": "data", "containerPath": "/data", "hostPath": distJobParam["hostdataPath"]})
-
-                for idx in range(len(distJobParam["mountpoints"])):
-                    if "name" not in distJobParam["mountpoints"][idx]:
-                        distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
-
-                distJobParam["pod_ip_range"] = job_object.get_pod_ip_range()
-                distJobParam["usefreeflow"] = job_object.is_freeflow_enabled()
-
-                distJobParam["numworker"] = int(jobParams["numpsworker"])
-                distJobParam["numps"] = int(jobParams["numps"])
-
-                random.seed(datetime.datetime.now())
-                if "hostNetwork" in jobParams and jobParams["hostNetwork"]:
-                    distJobParam["sshPort"] = random.randint(40000, 49999)
-                else:
-                    distJobParam["sshPort"] = int(random.random() * 1000 + 3000)
-
-                if assignedRack is not None:
-                    if "nodeSelector" not in distJobParam:
-                        distJobParam["nodeSelector"] = {}
-                    distJobParam["nodeSelector"]["rack"] = assignedRack
-
-                if "gpuType" in distJobParam:
-                    if "nodeSelector" not in distJobParam:
-                        distJobParam["nodeSelector"] = {}
-                    distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
-
-                template = ENV.get_template(os.path.abspath(jobTemp))
-                job_description = template.render(job=distJobParam)
+                pod_template = DistPodTemplate(job_object.get_dist_template())
+                job_description = pod_template.generate_pod(job_object, distJobParam)
 
                 jobDescriptionList.append(job_description)
 

From 0801c07c2464a305b18489784bd334fc1f07292d Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 07:12:22 +0000
Subject: [PATCH 035/194] extract DistPodTemplate.generate_pods()

---
 src/ClusterManager/dist_pod_template.py | 33 +++++++++++++++++++++++++
 src/ClusterManager/job_manager.py       | 31 +++--------------------
 2 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 82e45d53e..9fb2ee466 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -4,12 +4,14 @@
 import datetime
 import random
 import json
+import copy
 import yaml
 from jinja2 import Template
 from job import Job
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
 from config import config
+from osUtils import mkdirsAsUser
 
 
 class DistPodTemplate():
@@ -156,3 +158,34 @@ def generate_pod(self, job, pod):
         job_description = self.template.render(job=distJobParam)
 
         return job_description
+
+    def generate_pods(self, job):
+        # TODO
+        jobParams = job.params
+
+        pods = []
+
+        nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
+        for role in ["ps", "worker"]:
+            for idx in range(nums[role]):
+                distJobParam = copy.deepcopy(jobParams)
+
+                distJobParam["distId"] = "%s%d" % (role, idx)
+                distJobParam["distRole"] = role
+                distJobParam["distRoleIdx"] = idx
+
+                # TODO
+                distJobParam["distJobPath"] = os.path.join(job.job_path, distJobParam["distId"])
+                localJobPath = os.path.join(config["storage-mount-path"], "work/", distJobParam["distJobPath"])
+                if not os.path.exists(localJobPath):
+                    if "userId" in distJobParam:
+                        mkdirsAsUser(localJobPath, distJobParam["userId"])
+                    else:
+                        mkdirsAsUser(localJobPath, 0)
+
+                pod_template = DistPodTemplate(job.get_dist_template())
+                job_description = pod_template.generate_pod(job, distJobParam)
+
+                pods.append(job_description)
+
+        return pods
\ No newline at end of file
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 8140c5add..391275297 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -155,7 +155,7 @@ def SubmitPSDistJob(job):
 
         jobParams = json.loads(base64.b64decode(job["jobParams"]))
 
-        job_object.params = json.loads(base64.b64decode(job["jobParams"]))
+        job_object.params = jobParams
 
         # inject gid, uid and user
         # TODO it should return only one entry
@@ -192,31 +192,8 @@ def SubmitPSDistJob(job):
         job_object.job_path = jobParams["jobPath"]
         jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
 
-        jobDescriptionList = []
-        nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
-        for role in ["ps", "worker"]:
-            for idx in range(nums[role]):
-                distJobParam = copy.deepcopy(jobParams)
-
-                distJobParam["distId"] = "%s%d" % (role, idx)
-                distJobParam["distRole"] = role
-                distJobParam["distRoleIdx"] = idx
-
-                # TODO
-                distJobParam["distJobPath"] = os.path.join(job_object.job_path, distJobParam["distId"])
-                localJobPath = os.path.join(config["storage-mount-path"], "work/", distJobParam["distJobPath"])
-                if not os.path.exists(localJobPath):
-                    if "userId" in distJobParam:
-                        mkdirsAsUser(localJobPath, distJobParam["userId"])
-                    else:
-                        mkdirsAsUser(localJobPath, 0)
-
-                pod_template = DistPodTemplate(job_object.get_dist_template())
-                job_description = pod_template.generate_pod(job_object, distJobParam)
-
-                jobDescriptionList.append(job_description)
-
-                distJobParams[role].append(distJobParam)
+        dist_pod_template = DistPodTemplate(job_object.get_dist_template())
+        jobDescriptionList = dist_pod_template.generate_pods(job_object)
 
         jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
         jobDescription = "\n---\n".join(jobDescriptionList)
@@ -249,7 +226,7 @@ def SubmitPSDistJob(job):
         jobMeta["workPath"] = jobParams["workPath"]
         jobMeta["jobPath"] = jobParams["jobPath"]
         jobMeta["LaunchCMD"] = jobParams["cmd"]
-        jobMeta["distJobParams"] = distJobParams
+        jobMeta["distJobParams"] = jobDescriptionList
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
         dataHandler.UpdateJobTextField(jobParams["jobId"], "jobMeta", jobMetaStr)

From a7fd72994cf636062645449172d0eea2dd10fcaa Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 07:36:36 +0000
Subject: [PATCH 036/194] refactoring jobmanager

---
 src/ClusterManager/dist_pod_template.py |  30 ++++++-
 src/ClusterManager/job_manager.py       | 100 ++++++++----------------
 2 files changed, 61 insertions(+), 69 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 9fb2ee466..36ecee1e8 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -155,14 +155,38 @@ def generate_pod(self, job, pod):
                 distJobParam["nodeSelector"] = {}
             distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
 
-        job_description = self.template.render(job=distJobParam)
+        pod_yaml = self.template.render(job=distJobParam)
 
-        return job_description
+        return yaml.load(pod_yaml)
 
     def generate_pods(self, job):
+        """
+        Return (pods, errors)
+        """
+
         # TODO
         jobParams = job.params
 
+        if any(required_field not in jobParams for required_field in
+               [
+                   "jobtrainingtype",
+                   "jobName",
+                   "jobPath",
+                   "workPath",
+                   "dataPath",
+                   "cmd",
+                   "userId",
+                   "resourcegpu",
+                   "userName",
+               ]):
+            return None, "Missing required parameters!"
+        assert(jobParams["jobtrainingtype"] == "PSDistJob")
+
+        jobParams["rest-api"] = job.get_rest_api_url()
+        jobParams["user_email"] = jobParams["userName"]
+        jobParams["homeFolderHostpath"] = job.get_homefolder_hostpath()
+        job.job_path = jobParams["jobPath"]
+
         pods = []
 
         nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
@@ -188,4 +212,4 @@ def generate_pods(self, job):
 
                 pods.append(job_description)
 
-        return pods
\ No newline at end of file
+        return pods, None
\ No newline at end of file
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 391275297..fe0828609 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -153,92 +153,60 @@ def SubmitPSDistJob(job):
         # TODO assert job_object is a Job
         assert(isinstance(job_object, Job))
 
-        jobParams = json.loads(base64.b64decode(job["jobParams"]))
-
-        job_object.params = jobParams
+        job_object.params = json.loads(base64.b64decode(job["jobParams"]))
 
         # inject gid, uid and user
         # TODO it should return only one entry
         user_info = dataHandler.GetIdentityInfo(job_object.params["userName"])[0]
-        jobParams["gid"] = user_info["gid"]
-        jobParams["uid"] = user_info["uid"]
-        jobParams["user"] = job_object.get_alias()
-
-        if any(required_field not in jobParams for required_field in
-               [
-                   "jobtrainingtype",
-                   "jobName",
-                   "jobPath",
-                   "workPath",
-                   "dataPath",
-                   "cmd",
-                   "userId",
-                   "resourcegpu",
-                   "userName",
-               ]):
-            return None, "Missing required parameters!"
-        assert(jobParams["jobtrainingtype"] == "PSDistJob")
-
-        jobParams["rest-api"] = job_object.get_rest_api_url()
-
-
-        distJobParams = {}
-        distJobParams["ps"] = []
-        distJobParams["worker"] = []
-
-        userAlias = job_object.get_alias()
-        jobParams["user_email"] = jobParams["userName"]
-
-        job_object.job_path = jobParams["jobPath"]
-        jobParams["homeFolderHostpath"] = job_object.get_homefolder_hostpath()
+        job_object.params["gid"] = user_info["gid"]
+        job_object.params["uid"] = user_info["uid"]
+        job_object.params["user"] = job_object.get_alias()
 
         dist_pod_template = DistPodTemplate(job_object.get_dist_template())
-        jobDescriptionList = dist_pod_template.generate_pods(job_object)
-
-        jobParams["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
-        jobDescription = "\n---\n".join(jobDescriptionList)
-
-        jobDescriptionPath = os.path.join(config["storage-mount-path"], jobParams["jobDescriptionPath"])
-        if not os.path.exists(os.path.dirname(os.path.realpath(jobDescriptionPath))):
-            os.makedirs(os.path.dirname(os.path.realpath(jobDescriptionPath)))
-        if os.path.isfile(jobDescriptionPath):
-            output = k8sUtils.kubectl_delete(jobDescriptionPath)
-
-        with open(jobDescriptionPath, 'w') as f:
-            f.write(jobDescription)
-
-        output = k8sUtils.kubectl_create(jobDescriptionPath)
+        pods, error = dist_pod_template.generate_pods(job_object)
+        if error:
+            dataHandler.SetJobError(job_object.job_id, "ERROR: %s" % error)
+            return False
 
-        ret["output"] = output
+        job_description = "\n---\n".join([yaml.dump(pod) for pod in pods])
+        job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
+        local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
+        if not os.path.exists(local_jobDescriptionPath):
+            os.makedirs(os.path.dirname(local_jobDescriptionPath))
+        with open(local_jobDescriptionPath, 'w') as f:
+            f.write(job_description)
 
-        ret["jobId"] = jobParams["jobId"]
+        job_deployer = JobDeployer()
+        try:
+            ret["output"] = job_deployer.create_pods(pods)
+        except Exception as e:
+            ret["output"] = "Error: %s" % e.message
+            logging.error(e, exc_info=True)
 
-        if "userName" not in jobParams:
-            jobParams["userName"] = ""
+        ret["jobId"] = job_object.job_id
 
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "scheduling")
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescriptionPath", jobParams["jobDescriptionPath"])
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobDescription", base64.b64encode(jobDescription))
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobStatus", "scheduling")
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobDescriptionPath", job_description_path)
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobDescription", base64.b64encode(job_description))
 
         jobMeta = {}
-        jobMeta["jobDescriptionPath"] = jobParams["jobDescriptionPath"]
-        jobMeta["jobPath"] = jobParams["jobPath"]
-        jobMeta["workPath"] = jobParams["workPath"]
-        jobMeta["jobPath"] = jobParams["jobPath"]
-        jobMeta["LaunchCMD"] = jobParams["cmd"]
-        jobMeta["distJobParams"] = jobDescriptionList
+        jobMeta["jobDescriptionPath"] = job_description_path
+        jobMeta["jobPath"] = job_object.job_path
+        jobMeta["workPath"] = job_object.work_path
+        # the command of the first container
+        jobMeta["LaunchCMD"] = pods[0]["spec"]["containers"][0]["command"]
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
-        dataHandler.UpdateJobTextField(jobParams["jobId"], "jobMeta", jobMetaStr)
+        dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
     except Exception as e:
         import traceback
         traceback.print_exc()
         print(e)
         ret["error"] = str(e)
-        retries = dataHandler.AddandGetJobRetries(jobParams["jobId"])
+        retries = dataHandler.AddandGetJobRetries(job["jobId"])
         if retries >= 5:
-            dataHandler.UpdateJobTextField(jobParams["jobId"], "jobStatus", "error")
-            dataHandler.UpdateJobTextField(jobParams["jobId"], "errorMsg", "Cannot submit job!" + str(e))
+            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
+            dataHandler.UpdateJobTextField(job["jobId"], "errorMsg", "Cannot submit job!" + str(e))
     dataHandler.Close()
     return ret
 

From eba316034dacb954945e1ac3165f1edc425ff0a8 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 07:58:31 +0000
Subject: [PATCH 037/194] cleanup

---
 src/ClusterManager/job_manager.py | 124 ++----------------------------
 1 file changed, 8 insertions(+), 116 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index fe0828609..60b53cb5b 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -45,38 +45,8 @@
 def printlog(msg):
     print("%s - %s" % (datetime.datetime.utcnow().strftime("%x %X"),msg))
 
-def LoadJobParams(jobParamsJsonStr):
-    return json.loads(jobParamsJsonStr)
-
-def cmd_exec(cmdStr):
-    try:
-        output = subprocess.check_output(["bash","-c", cmdStr])
-    except Exception as e:
-        print(e)
-        output = ""
-    return output
-
-
-
-
-
 
 def SubmitJob(job):
-    jobParams = json.loads(base64.b64decode(job["jobParams"]))
-    if jobParams["jobtrainingtype"] == "RegularJob":
-        SubmitRegularJob(job)
-    elif jobParams["jobtrainingtype"] == "PSDistJob":
-        SubmitPSDistJob(job)
-
-def CheckMountPoints(mplist, mp):
-    ret = True
-    for item in mplist:
-        if item["name"] == mp["name"] or item["containerPath"] == mp["containerPath"] or item["hostPath"] == mp["hostPath"]:
-            ret = False
-    return ret
-
-
-def SubmitRegularJob(job):
     ret = {}
     dataHandler = DataHandler()
 
@@ -96,7 +66,14 @@ def SubmitRegularJob(job):
         job_object.params["user"] = job_object.get_alias()
 
         enable_custom_scheduler = job_object.is_custom_scheduler_enabled()
-        pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
+        if job_object.params["jobtrainingtype"] == "RegularJob":
+            pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
+        elif job_object.params["jobtrainingtype"] == "PSDistJob":
+            pod_template = DistPodTemplate(job_object.get_dist_template())
+        else:
+            dataHandler.SetJobError(job_object.job_id, "ERROR: invalid jobtrainingtype: %s" % job_object.params["jobtrainingtype"])
+            return False
+
         pods, error = pod_template.generate_pods(job_object)
         if error:
             dataHandler.SetJobError(job_object.job_id, "ERROR: %s" % error)
@@ -143,74 +120,6 @@ def SubmitRegularJob(job):
     return ret
 
 
-def SubmitPSDistJob(job):
-    ret = {}
-    dataHandler = DataHandler()
-
-    try:
-        job["cluster"] = config
-        job_object, errors = JobSchema().load(job)
-        # TODO assert job_object is a Job
-        assert(isinstance(job_object, Job))
-
-        job_object.params = json.loads(base64.b64decode(job["jobParams"]))
-
-        # inject gid, uid and user
-        # TODO it should return only one entry
-        user_info = dataHandler.GetIdentityInfo(job_object.params["userName"])[0]
-        job_object.params["gid"] = user_info["gid"]
-        job_object.params["uid"] = user_info["uid"]
-        job_object.params["user"] = job_object.get_alias()
-
-        dist_pod_template = DistPodTemplate(job_object.get_dist_template())
-        pods, error = dist_pod_template.generate_pods(job_object)
-        if error:
-            dataHandler.SetJobError(job_object.job_id, "ERROR: %s" % error)
-            return False
-
-        job_description = "\n---\n".join([yaml.dump(pod) for pod in pods])
-        job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
-        local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
-        if not os.path.exists(local_jobDescriptionPath):
-            os.makedirs(os.path.dirname(local_jobDescriptionPath))
-        with open(local_jobDescriptionPath, 'w') as f:
-            f.write(job_description)
-
-        job_deployer = JobDeployer()
-        try:
-            ret["output"] = job_deployer.create_pods(pods)
-        except Exception as e:
-            ret["output"] = "Error: %s" % e.message
-            logging.error(e, exc_info=True)
-
-        ret["jobId"] = job_object.job_id
-
-        dataHandler.UpdateJobTextField(job_object.job_id, "jobStatus", "scheduling")
-        dataHandler.UpdateJobTextField(job_object.job_id, "jobDescriptionPath", job_description_path)
-        dataHandler.UpdateJobTextField(job_object.job_id, "jobDescription", base64.b64encode(job_description))
-
-        jobMeta = {}
-        jobMeta["jobDescriptionPath"] = job_description_path
-        jobMeta["jobPath"] = job_object.job_path
-        jobMeta["workPath"] = job_object.work_path
-        # the command of the first container
-        jobMeta["LaunchCMD"] = pods[0]["spec"]["containers"][0]["command"]
-
-        jobMetaStr = base64.b64encode(json.dumps(jobMeta))
-        dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        print(e)
-        ret["error"] = str(e)
-        retries = dataHandler.AddandGetJobRetries(job["jobId"])
-        if retries >= 5:
-            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
-            dataHandler.UpdateJobTextField(job["jobId"], "errorMsg", "Cannot submit job!" + str(e))
-    dataHandler.Close()
-    return ret
-
-
 def KillJob(job, desiredState="killed"):
     dataHandler = DataHandler()
     result, detail = k8sUtils.GetJobStatus(job["jobId"])
@@ -349,23 +258,6 @@ def UpdateJobStatus(job):
 
     dataHandler.Close()
 
-def run_dist_cmd_on_pod(podId, cmd, outputfile):
-    remotecmd = "exec %s -- %s" % (podId,cmd)
-    print(remotecmd)
-    k8sUtils.kubectl_exec_output_to_file(remotecmd,outputfile)
-
-
-
-class Kube_RemoteCMD_Thread(threading.Thread):
-    def __init__(self, jobId, podId, cmd, outputfile):
-        threading.Thread.__init__(self)
-        self.jobId = jobId
-        self.podId = podId
-        self.cmd = cmd
-        self.outputfile = outputfile
-    def run(self):
-        run_dist_cmd_on_pod(self.podId, self.cmd, self.outputfile)
-
 
 # TODO remove duplicate code later
 def is_ssh_server_ready(pod_name):

From b67486e45a14c72dbf39dd0170a597f3ed9059e1 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 08:00:28 +0000
Subject: [PATCH 038/194] update field lastUpdated

---
 src/ClusterManager/job_manager.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 60b53cb5b..f8db999fd 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -99,6 +99,7 @@ def SubmitJob(job):
         dataHandler.UpdateJobTextField(job_object.job_id, "jobStatus", "scheduling")
         dataHandler.UpdateJobTextField(job_object.job_id, "jobDescriptionPath", job_description_path)
         dataHandler.UpdateJobTextField(job_object.job_id, "jobDescription", base64.b64encode(job_description))
+        dataHandler.UpdateJobTextField(job_object.job_id, "lastUpdated", datetime.datetime.now().isoformat())
 
         jobMeta = {}
         jobMeta["jobDescriptionPath"] = job_description_path

From 7ac1a5d91450c9dad0f3da571e722a78ee284ed7 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 09:08:36 +0000
Subject: [PATCH 039/194] refactoring PodTemplate

---
 src/ClusterManager/job.py               | 14 --------
 src/ClusterManager/pod_template.py      | 45 +++++++++++++++++--------
 src/ClusterManager/test_job.py          | 12 -------
 src/ClusterManager/test_pod_template.py | 12 +++++++
 4 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 87d1e47ee..ffe223c02 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -111,20 +111,6 @@ def data_path_mountpoint(self):
         data_host_path = os.path.join(self.cluster["storage-mount-path"], "storage", self.data_path)
         return {"name": "data", "containerPath": "/data", "hostPath": data_host_path, "enabled": True}
 
-    def generate_launch_script(self, path_to_save, user_id, gpu_num, user_script):
-        if not os.path.exists(path_to_save):
-            mkdirsAsUser(path_to_save, user_id)
-
-        file_name = "launch-%s.sh" % self.job_id
-        launch_script_file = os.path.join(path_to_save, file_name)
-        with open(launch_script_file, 'w') as f:
-            f.write("#!/bin/bash -x\n")
-            f.write("mkdir /opt; \n")
-            f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % gpu_num)
-            f.write("bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
-        os.system("sudo chown %s %s" % (user_id, launch_script_file))
-        return file_name
-
     def get_template(self):
         """Return jinja template."""
         path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "RegularJob.yaml.template"))
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index aece37371..dd497ea92 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -1,14 +1,34 @@
+import os
+import sys
 import json
 import yaml
 from jinja2 import Template
 from job import Job
 
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+from osUtils import mkdirsAsUser
+
 
 class PodTemplate():
     def __init__(self, template, enable_custom_scheduler=False):
         self.template = template
         self.enable_custom_scheduler = enable_custom_scheduler
 
+    @staticmethod
+    def generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script):
+        if not os.path.exists(path_to_save):
+            mkdirsAsUser(path_to_save, user_id)
+
+        file_name = "launch-%s.sh" % job_id
+        launch_script_file = os.path.join(path_to_save, file_name)
+        with open(launch_script_file, 'w') as f:
+            f.write("#!/bin/bash -x\n")
+            f.write("mkdir /opt; \n")
+            f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % gpu_num)
+            f.write("bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
+        os.system("sudo chown %s %s" % (user_id, launch_script_file))
+        return file_name
+
     def generate_pod(self, pod):
         assert(isinstance(self.template, Template))
         if self.enable_custom_scheduler:
@@ -38,11 +58,6 @@ def generate_pod(self, pod):
             # TODO it's not safe to update pod["resourcegpu"]
             pod["resourcegpu"] = 0  # gpu requests specified through annotation
 
-        if "nodeSelector" not in pod:
-            pod["nodeSelector"] = {}
-        if "gpuType" in pod:
-            pod["nodeSelector"]["gpuType"] = pod["gpuType"]
-
         pod_yaml = self.template.render(job=pod)
         return yaml.load(pod_yaml)
 
@@ -76,23 +91,25 @@ def generate_pods(self, job):
             job.add_mountpoints(params["mountpoints"])
         job.add_mountpoints(job.work_path_mountpoint())
         job.add_mountpoints(job.data_path_mountpoint())
-
         params["mountpoints"] = job.mountpoints
 
-        local_job_path = job.get_local_job_path()
-        script_file = job.generate_launch_script(local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
-        luanch_cmd = "[\"bash\", \"/job/%s\"]" % script_file
-
         params["user_email"] = params["userName"]
         params["homeFolderHostpath"] = job.get_homefolder_hostpath()
-
         params["pod_ip_range"] = job.get_pod_ip_range()
         params["usefreeflow"] = job.is_freeflow_enabled()
-
-        # assign parameters to generate the pod yaml
-        params["LaunchCMD"] = luanch_cmd
         params["jobNameLabel"] = ''.join(e for e in params["jobName"] if e.isalnum())
         params["rest-api"] = job.get_rest_api_url()
+
+        if "nodeSelector" not in params:
+            params["nodeSelector"] = {}
+        if "gpuType" in params:
+            params["nodeSelector"]["gpuType"] = params["gpuType"]
+
+        local_job_path = job.get_local_job_path()
+        script_file = PodTemplate.generate_launch_script(params["jobId"], local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
+        luanch_cmd = "[\"bash\", \"/job/%s\"]" % script_file
+        params["LaunchCMD"] = luanch_cmd
+
         pods = []
         if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
             env_name = params["hyperparametername"]
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index 02672bb23..eff8c5c22 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -151,18 +151,6 @@ def test_job_work_data_mountpoints(self):
         job.add_mountpoints(job.data_path_mountpoint())
         self.assertEquals(3, len(job.mountpoints))
 
-    def test_generate_launch_script(self):
-        job = self.create_a_job()
-
-        path_to_save = "/tmp"
-        user_id = "20000"
-        gpu_num = 3
-        user_script = "sleep infinity"
-        script_file = job.generate_launch_script(path_to_save, user_id, gpu_num, user_script)
-
-        # "launch-%s.sh" % self.job_id
-        self.assertEqual("launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh", script_file)
-
     def test_get_template(self):
         job = self.create_a_job()
 
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index a49c62768..e336c473a 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -22,6 +22,18 @@
 
 class TestPodTemplate(unittest.TestCase):
 
+    def test_generate_launch_script(self):
+        job_id = "ce7dca49-28df-450a-a03b-51b9c2ecc69c"
+        path_to_save = "/tmp"
+        user_id = "20000"
+        gpu_num = 3
+        user_script = "sleep infinity"
+
+        script_file = PodTemplate.generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script)
+
+        # "launch-%s.sh" % self.job_id
+        self.assertEqual("launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh", script_file)
+
     def test_pod_template_without_custer_scheduler(self):
         enable_custom_scheduler = False
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)

From 827037d98831eaefc5eb8470c7aede86f958d548 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 09:27:11 +0000
Subject: [PATCH 040/194] refine

---
 src/ClusterManager/pod_template.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index dd497ea92..667b83c6c 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -128,11 +128,8 @@ def generate_pods(self, job):
                 pod["envs"] = []
                 pods.append(pod)
 
-        enable_custom_scheduler = job.is_custom_scheduler_enabled()
-        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
-
         k8s_pods = []
         for pod in pods:
-            k8s_pod = pod_template.generate_pod(pod)
+            k8s_pod = self.generate_pod(pod)
             k8s_pods.append(k8s_pod)
         return k8s_pods, None

From e0bde4db195443af75fb73a9dbda30731403eb5e Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 09:41:25 +0000
Subject: [PATCH 041/194] refine PodTemplate

---
 src/ClusterManager/pod_template.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 667b83c6c..b24525e7c 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -27,7 +27,8 @@ def generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script):
             f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % gpu_num)
             f.write("bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
         os.system("sudo chown %s %s" % (user_id, launch_script_file))
-        return file_name
+        luanch_cmd = "[\"bash\", \"/job/%s\"]" % file_name
+        return luanch_cmd
 
     def generate_pod(self, pod):
         assert(isinstance(self.template, Template))
@@ -106,9 +107,7 @@ def generate_pods(self, job):
             params["nodeSelector"]["gpuType"] = params["gpuType"]
 
         local_job_path = job.get_local_job_path()
-        script_file = PodTemplate.generate_launch_script(params["jobId"], local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
-        luanch_cmd = "[\"bash\", \"/job/%s\"]" % script_file
-        params["LaunchCMD"] = luanch_cmd
+        params["LaunchCMD"] = PodTemplate.generate_launch_script(params["jobId"], local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
 
         pods = []
         if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):

From 2c29a20d646c7827f94f0c3aaaca2b9837d43f26 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 11:34:52 +0000
Subject: [PATCH 042/194] refactoring DistPodTemplate

---
 src/ClusterManager/dist_pod_template.py | 199 +++++++++++-------------
 1 file changed, 92 insertions(+), 107 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 36ecee1e8..021289b5e 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -20,12 +20,12 @@ def __init__(self, template, enable_custom_scheduler=False):
         self.enable_custom_scheduler = enable_custom_scheduler
 
     @staticmethod
-    def generate_launch_cmd(role, user_alias, job_path, worker_num, cmd):
+    def generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cmd):
         # change ssh folder permission here because the setup permission
         #  script in launch_ps_job function may have race condition with init_user.sh script.
         # results in no such user error
-        if role == "ps":
-            launchCMD = """
+        if dist_id.startswith("ps"):
+            script = """
 #!/bin/bash
 echo "[DLWorkspace System]: Waiting for all containers are ready..."
 while [ ! -f /opt/run_dist_job ]; do
@@ -48,28 +48,28 @@ def generate_launch_cmd(role, user_alias, job_path, worker_num, cmd):
 all_workers_ready=false
 while [ "$all_workers_ready" != true ]
 do
-  # update it to false if any woker is not ready
-  all_workers_ready=true
-
-  for i in $(seq 0 $(( ${{WORKER_NUM}} - 1)) )
-  do
-    worker="worker${{i}}"
-    file="$JOB_DIR/${{worker}}/WORKER_READY"
-    #echo $file
-
-    if [ ! -f $file ]; then
-      echo "${{worker}} not ready!"
-      all_workers_ready=false
-      sleep 10
-    fi
-  done
+    # update it to false if any woker is not ready
+    all_workers_ready=true
+
+    for i in $(seq 0 $(( ${{WORKER_NUM}} - 1)) )
+    do
+        worker="worker${{i}}"
+        file="$JOB_DIR/${{worker}}/WORKER_READY"
+        #echo $file
+
+        if [ ! -f $file ]; then
+        echo "${{worker}} not ready!"
+        all_workers_ready=false
+        sleep 10
+        fi
+    done
 done
 
 echo "[DLWorkspace System]: All containers are ready, launching training job..."
 {3}
 """.format(user_alias, job_path, worker_num, cmd)
         else:
-            launchCMD = """
+            script = """
 while [ ! -f /opt/run_dist_job ]; do
     sleep 3
 done
@@ -85,89 +85,48 @@ def generate_launch_cmd(role, user_alias, job_path, worker_num, cmd):
 
 sleep infinity
 """.format(user_alias)
-        return launchCMD
-
-    def generate_pod(self, job, pod):
-        # TODO
-        distJobParam = pod
-
-        role = distJobParam["distRole"]
-        localJobPath = os.path.join(config["storage-mount-path"], "work/", distJobParam["distJobPath"])
-        userAlias = distJobParam["user"]
-        idx = distJobParam["distRoleIdx"]
 
-        job_path = distJobParam["jobPath"]
-        worker_num = distJobParam["numpsworker"]
-        cmd = distJobParam["cmd"]
-        launchCMD = DistPodTemplate.generate_launch_cmd(role, userAlias, job_path, worker_num, cmd)
-
-        launchScriptPath = os.path.join(localJobPath, "launch-%s-%s%d.sh" % (distJobParam["jobId"], role, idx))
-        # TODO need to set up user for distribute jobs
-        with open(launchScriptPath, 'w') as f:
-            f.write(launchCMD)
+        local_job_path = os.path.join(config["storage-mount-path"], "work/", job_path, dist_id)
+        file_name = "launch-%s-%s.sh" % (job_id, dist_id)
+        launch_script_file = os.path.join(local_job_path, file_name)
+        with open(launch_script_file, 'w') as f:
+            f.write(script)
         f.close()
 
-        launchScriptInContainer = "bash /job/launch-%s-%s%d.sh" % (distJobParam["jobId"], role, idx)
-
-        distJobParam["LaunchCMD"] = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
+        launchScriptInContainer = "bash /job/launch-%s-%s.sh" % (job_id, dist_id)
 
-        distJobParam["jobNameLabel"] = ''.join(e for e in distJobParam["jobName"] if e.isalnum())
-
-        jobPath = "work/" + distJobParam["distJobPath"]
-        workPath = "work/" + distJobParam["workPath"]
-        dataPath = "storage/" + distJobParam["dataPath"]
-
-        distJobParam["hostjobPath"] = os.path.join(config["storage-mount-path"], jobPath)
-        distJobParam["hostworkPath"] = os.path.join(config["storage-mount-path"], workPath)
-        distJobParam["hostdataPath"] = os.path.join(config["storage-mount-path"], dataPath)
-
-        if "mountpoints" not in distJobParam:
-            distJobParam["mountpoints"] = []
-
-        distJobParam["mountpoints"].append({"name": "job", "containerPath": "/job", "hostPath": distJobParam["hostjobPath"]})
-        distJobParam["mountpoints"].append({"name": "work", "containerPath": "/work", "hostPath": distJobParam["hostworkPath"]})
-        distJobParam["mountpoints"].append({"name": "data", "containerPath": "/data", "hostPath": distJobParam["hostdataPath"]})
+        launchCMD = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
+        return launchCMD
 
-        for idx in range(len(distJobParam["mountpoints"])):
-            if "name" not in distJobParam["mountpoints"][idx]:
-                distJobParam["mountpoints"][idx]["name"] = str(uuid.uuid4()).replace("-", "")
+    def generate_pod(self, pod):
+        assert(isinstance(self.template, Template))
 
-        distJobParam["pod_ip_range"] = job.get_pod_ip_range()
-        distJobParam["usefreeflow"] = job.is_freeflow_enabled()
+        dist_id = pod["distId"]
+        job_id = pod["jobId"]
+        user_alias = pod["user"]
+        job_path = pod["jobPath"]
+        worker_num = pod["numpsworker"]
+        cmd = pod["cmd"]
 
-        distJobParam["numworker"] = int(distJobParam["numpsworker"])
-        distJobParam["numps"] = int(distJobParam["numps"])
+        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cmd)
 
         random.seed(datetime.datetime.now())
-        if "hostNetwork" in distJobParam and distJobParam["hostNetwork"]:
-            distJobParam["sshPort"] = random.randint(40000, 49999)
+        if "hostNetwork" in pod and pod["hostNetwork"]:
+            pod["sshPort"] = random.randint(40000, 49999)
         else:
-            distJobParam["sshPort"] = int(random.random() * 1000 + 3000)
-
-        assignedRack = job.get_rack()
-        if assignedRack is not None:
-            if "nodeSelector" not in distJobParam:
-                distJobParam["nodeSelector"] = {}
-            distJobParam["nodeSelector"]["rack"] = assignedRack
-
-        if "gpuType" in distJobParam:
-            if "nodeSelector" not in distJobParam:
-                distJobParam["nodeSelector"] = {}
-            distJobParam["nodeSelector"]["gpuType"] = distJobParam["gpuType"]
-
-        pod_yaml = self.template.render(job=distJobParam)
+            pod["sshPort"] = int(random.random() * 1000 + 3000)
 
+        pod_yaml = self.template.render(job=pod)
         return yaml.load(pod_yaml)
 
     def generate_pods(self, job):
         """
         Return (pods, errors)
         """
+        assert(isinstance(job, Job))
+        params = job.params
 
-        # TODO
-        jobParams = job.params
-
-        if any(required_field not in jobParams for required_field in
+        if any(required_field not in params for required_field in
                [
                    "jobtrainingtype",
                    "jobName",
@@ -180,36 +139,62 @@ def generate_pods(self, job):
                    "userName",
                ]):
             return None, "Missing required parameters!"
-        assert(jobParams["jobtrainingtype"] == "PSDistJob")
+        assert(params["jobtrainingtype"] == "PSDistJob")
+
+        job.job_path = params["jobPath"]
+        job.work_path = params["workPath"]
+        job.data_path = params["dataPath"]
+        # TODO user's mountpoints first, but should after 'job_path'
+        job.add_mountpoints(job.job_path_mountpoint())
+        if "mountpoints" in params:
+            job.add_mountpoints(params["mountpoints"])
+        job.add_mountpoints(job.work_path_mountpoint())
+        job.add_mountpoints(job.data_path_mountpoint())
+        params["mountpoints"] = job.mountpoints
+
+        params["user_email"] = params["userName"]
+        params["homeFolderHostpath"] = job.get_homefolder_hostpath()
+        params["pod_ip_range"] = job.get_pod_ip_range()
+        params["usefreeflow"] = job.is_freeflow_enabled()
+        params["jobNameLabel"] = ''.join(e for e in params["jobName"] if e.isalnum())
+        params["rest-api"] = job.get_rest_api_url()
+
+        if "nodeSelector" not in params:
+            params["nodeSelector"] = {}
+        if "gpuType" in params:
+            params["nodeSelector"]["gpuType"] = params["gpuType"]
+        assignedRack = job.get_rack()
+        if assignedRack is not None:
+            params["nodeSelector"]["rack"] = assignedRack
 
-        jobParams["rest-api"] = job.get_rest_api_url()
-        jobParams["user_email"] = jobParams["userName"]
-        jobParams["homeFolderHostpath"] = job.get_homefolder_hostpath()
-        job.job_path = jobParams["jobPath"]
+        params["numworker"] = int(params["numpsworker"])
+        params["numps"] = int(params["numps"])
 
         pods = []
-
-        nums = {"ps": int(jobParams["numps"]), "worker": int(jobParams["numpsworker"])}
+        nums = {"ps": int(params["numps"]), "worker": int(params["numpsworker"])}
         for role in ["ps", "worker"]:
             for idx in range(nums[role]):
-                distJobParam = copy.deepcopy(jobParams)
+                pod = copy.deepcopy(params)
+
+                pod["distId"] = "%s%d" % (role, idx)
+                pod["distRole"] = role
+                pod["distRoleIdx"] = idx
 
-                distJobParam["distId"] = "%s%d" % (role, idx)
-                distJobParam["distRole"] = role
-                distJobParam["distRoleIdx"] = idx
+                # TODO refine later
+                dist_job_path = os.path.join(job.job_path, pod["distId"])
+                for mp in pod["mountpoints"]:
+                    if mp["name"] == "job":
+                        mp["hostPath"] = mp["hostPath"] + "/" + pod["distId"]
 
-                # TODO
-                distJobParam["distJobPath"] = os.path.join(job.job_path, distJobParam["distId"])
-                localJobPath = os.path.join(config["storage-mount-path"], "work/", distJobParam["distJobPath"])
-                if not os.path.exists(localJobPath):
-                    if "userId" in distJobParam:
-                        mkdirsAsUser(localJobPath, distJobParam["userId"])
-                    else:
-                        mkdirsAsUser(localJobPath, 0)
+                local_job_path = os.path.join(config["storage-mount-path"], "work/", dist_job_path)
+                if not os.path.exists(local_job_path):
+                    mkdirsAsUser(local_job_path, pod["userId"])
 
-                pod_template = DistPodTemplate(job.get_dist_template())
-                job_description = pod_template.generate_pod(job, distJobParam)
+                pods.append(pod)
 
-                pods.append(job_description)
+        k8s_pods = []
+        for pod in pods:
+            k8s_pod = self.generate_pod(pod)
+            k8s_pods.append(k8s_pod)
 
-        return pods, None
\ No newline at end of file
+        return k8s_pods, None

From 211142f86b5e95c9eca894cca78e765033f30693 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 11:48:39 +0000
Subject: [PATCH 043/194] using yaml.sfull_load for safety

---
 src/ClusterManager/cluster_manager.py   | 2 +-
 src/ClusterManager/dist_pod_template.py | 2 +-
 src/ClusterManager/job.py               | 2 +-
 src/ClusterManager/job_manager.py       | 2 +-
 src/ClusterManager/pod_template.py      | 2 +-
 src/ClusterManager/requirements.txt     | 3 +++
 src/ClusterManager/test_job_deployer.py | 2 +-
 src/utils/config.py                     | 6 +++---
 8 files changed, 12 insertions(+), 9 deletions(-)
 create mode 100644 src/ClusterManager/requirements.txt

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index ae8427d66..ab9aa7d68 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -32,7 +32,7 @@ def create_log(logdir='/var/log/dlworkspace'):
     if not os.path.exists(logdir):
         os.system("mkdir -p " + logdir)
     with open('logging.yaml') as f:
-        logging_config = yaml.load(f)
+        logging_config = yaml.full_load(f)
         f.close()
         logging_config["handlers"]["file"]["filename"] = logdir+"/clustermanager.log"
         logging.config.dictConfig(logging_config)
diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 021289b5e..fc6516213 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -117,7 +117,7 @@ def generate_pod(self, pod):
             pod["sshPort"] = int(random.random() * 1000 + 3000)
 
         pod_yaml = self.template.render(job=pod)
-        return yaml.load(pod_yaml)
+        return yaml.full_load(pod_yaml)
 
     def generate_pods(self, job):
         """
diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index ffe223c02..70f74e8b8 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -17,7 +17,7 @@ def create_log(logdir='.'):
     if not os.path.exists(logdir):
         os.system("mkdir -p " + logdir)
     with open('logging.yaml') as f:
-        logging_config = yaml.load(f)
+        logging_config = yaml.full_load(f)
         f.close()
         logging_config["handlers"]["file"]["filename"] = logdir + "/jobmanager.log"
         logging.config.dictConfig(logging_config)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index f8db999fd..7731c57df 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -391,7 +391,7 @@ def create_log( logdir = '/var/log/dlworkspace' ):
     if not os.path.exists( logdir ):
         os.system("mkdir -p " + logdir )
     with open('logging.yaml') as f:
-        logging_config = yaml.load(f)
+        logging_config = yaml.full_load(f)
         f.close()
         logging_config["handlers"]["file"]["filename"] = logdir+"/jobmanager.log"
         logging.config.dictConfig(logging_config)
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index b24525e7c..6cab85f7b 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -60,7 +60,7 @@ def generate_pod(self, pod):
             pod["resourcegpu"] = 0  # gpu requests specified through annotation
 
         pod_yaml = self.template.render(job=pod)
-        return yaml.load(pod_yaml)
+        return yaml.full_load(pod_yaml)
 
     def generate_pods(self, job):
         """
diff --git a/src/ClusterManager/requirements.txt b/src/ClusterManager/requirements.txt
new file mode 100644
index 000000000..111149103
--- /dev/null
+++ b/src/ClusterManager/requirements.txt
@@ -0,0 +1,3 @@
+marshmallow==2.19.5
+kubernetes==9.0.0
+PyYAML>=5.1.1
diff --git a/src/ClusterManager/test_job_deployer.py b/src/ClusterManager/test_job_deployer.py
index f0731cabd..d466e596b 100644
--- a/src/ClusterManager/test_job_deployer.py
+++ b/src/ClusterManager/test_job_deployer.py
@@ -28,7 +28,7 @@ def test_create_pod(self):
     - sleep
     - "1000000"
     """
-        body = yaml.load(raw_yaml)
+        body = yaml.full_load(raw_yaml)
 
         # with self.assertRaises(ApiException):
         job_deployer.create_pod(body)
diff --git a/src/utils/config.py b/src/utils/config.py
index 7c19c84ad..25d6e572e 100755
--- a/src/utils/config.py
+++ b/src/utils/config.py
@@ -1,11 +1,11 @@
 import yaml
 import os
-from Queue import Queue 
+from Queue import Queue
 import threading
 
 try:
-    f = open(os.path.join(os.path.dirname(os.path.realpath(__file__)),"config.yaml"))
-    config = yaml.load(f)
+    f = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.yaml"))
+    config = yaml.full_load(f)
 except Exception:
     config = {}
     ()

From 03716950354d6fe535f8759eb6db05339704b592 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 4 Jul 2019 20:19:48 +0800
Subject: [PATCH 044/194] Add password validation and team checking

---
 .../WebPortal/Controllers/dlwsController.cs   | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index 91d2a71de..eadc37f87 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -161,12 +161,16 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
                     var db = pair.Value;
 
 
-                    var priorEntrys = db.User.Where(b => b.Email == email).Where(b => b.Password == password).ToAsyncEnumerable();
+                    var priorEntrys = db.User.Where(b => b.Email == email).ToAsyncEnumerable();
 
                     await priorEntrys.ForEachAsync(userEntry =>
                     {
                         authorizedClusters.Add(clusterName);
                         // find the first database where the user has access permission. 
+                        if (!userEntry.Password.Equals(password))
+                        {
+                            return;
+                        }
                         if (!passwdLogin)
                         {
                             HttpContext.Session.SetString("Email", userEntry.Alias);
@@ -189,7 +193,10 @@ await priorEntrys.ForEachAsync(userEntry =>
                 if (passwdLogin)
                 {
                     HttpContext.Session.SetString("AuthorizedClusters", JsonConvert.SerializeObject(authorizedClusters));
-                    HttpContext.Session.SetString("Team", HttpContext.Request.Query["Team"]);
+                    var team = HttpContext.Request.Query["Team"];
+                    HttpContext.Session.SetString("Team", team);
+                    var teamClusters = await HomeController.GetTeamClusters(HttpContext, team);
+                    HttpContext.Session.SetString("TeamClusters", JsonConvert.SerializeObject(teamClusters));
                 }
                 if ( !bFindUser )
                 {
@@ -509,6 +516,13 @@ public async Task<ActionResult> postJob(TemplateParams templateParams)
             }
             var restapi = Startup.Clusters[cluster].Restapi;
 
+            var team = HttpContext.Session.GetString("Team");
+            var teamClusters = JsonConvert.DeserializeObject<List<string>>(HttpContext.Session.GetString("TeamClusters"));
+            if (!teamClusters.Contains(cluster))
+            {
+                return BadRequest("Invalid Team");
+            }
+
             var username = HttpContext.Session.GetString("Username");
             ViewData["Username"] = username;
             var uid = HttpContext.Session.GetString("uid");
@@ -518,7 +532,7 @@ public async Task<ActionResult> postJob(TemplateParams templateParams)
             jobObject["userName"] = HttpContext.Session.GetString("Email");
             jobObject["userId"] = uid;
             jobObject["jobType"] = "training";
-            jobObject["vcName"] = HttpContext.Session.GetString("Team");
+            jobObject["vcName"] = team;
 
             var runningasroot = jobObject["runningasroot"];
             if (

From a3d8f1448c7db0e324edde276e86cb57e4c75e37 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 13:05:42 +0000
Subject: [PATCH 045/194] refine job template file

---
 src/ClusterManager/dist_pod_template.py    |  1 +
 src/Jobs_Templete/DistJob.yaml.template    | 53 ++++++++++++----------
 src/Jobs_Templete/RegularJob.yaml.template | 31 ++++++-------
 3 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index fc6516213..9945e667c 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -179,6 +179,7 @@ def generate_pods(self, job):
                 pod["distId"] = "%s%d" % (role, idx)
                 pod["distRole"] = role
                 pod["distRoleIdx"] = idx
+                pod["podName"] = "{}-{}".format(pod["jobId"], pod["distId"])
 
                 # TODO refine later
                 dist_job_path = os.path.join(job.job_path, pod["distId"])
diff --git a/src/Jobs_Templete/DistJob.yaml.template b/src/Jobs_Templete/DistJob.yaml.template
index 8c9267934..79f747207 100755
--- a/src/Jobs_Templete/DistJob.yaml.template
+++ b/src/Jobs_Templete/DistJob.yaml.template
@@ -1,24 +1,26 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: {{ job["jobId"] }}-{{ job["distId"] }}
+  name: {{ job["podName"] }}
   labels:
     run: {{ job["jobId"] }}
-    podName: {{ job["jobId"] }}-{{ job["distId"] }}
+    podName: {{ job["podName"] }}
     jobName: {{ job["jobNameLabel"] }}
+    jobId: {{ job["jobId"] }}
+    userName: {{ job["user"] }}
+    vcName: {{ job["vcName"] }}
+
     distRole: {{ job["distRole"] }}
     distRoleIdx: "{{ job["distRoleIdx"] }}"
     sshPort: "{{job["sshPort"]}}"
-    userName: {{ job["user"] }}
-    vcName: {{ job["vcName"] }}
+
   {% if "gpuType" in job %}
-   {% if job["gpuType"]|length > 0 %}
+    {% if job["gpuType"]|length > 0 %}
     gpuType: {{ job["gpuType"] }}
-   {% endif %}
+    {% endif %}
   {% endif %}
     preemptionAllowed: "{{ job["preemptionAllowed"] }}"
 spec:
-  #hostNetwork: true
   nodeSelector:
     worker: active
   {% if job["nodeSelector"]|length > 0 %}
@@ -36,12 +38,12 @@ spec:
   hostIPC: true
   {% endif %}
   containers:
-  - name: {{ job["jobId"] }}
+  - name: {{ job["podName"] }}
     image: {{ job["image"] }}
     imagePullPolicy: Always
     command: {{ job["LaunchCMD"] }}
-    #container port and host port should be same.
     securityContext:
+      runAsUser: {{ job["containerUserId"] }}
     {% if job["isPrivileged"] %}
       privileged: true
     {% endif %}
@@ -49,7 +51,6 @@ spec:
         add:
         - IPC_LOCK
         - SYS_ADMIN
-    ports:
     {% if job["distRole"] =="worker" %}
     resources:
       limits:
@@ -81,17 +82,22 @@ spec:
     - name: authorized-keys-volume
       mountPath: /home/{{ job["user"] }}/.ssh/authorized_keys
       readOnly: true
-    {% if job["usefreeflow"] %}
-    - mountPath: /freeflow
-      name: freeflow
+    {% if not job["dnsPolicy"] %}
+    - mountPath: /etc/resolv.conf
+      name: resolv
     {% endif %}
     {% for mp in job["mountpoints"] %}
+    {% if mp.enabled %}
     - mountPath: {{ mp.containerPath }}
       name: {{ mp.name }}
+      {% if mp.readOnly %}
+      readOnly: true
+      {% endif %}
+    {% endif %}
     {% endfor %}
-    {% if not job["dnsPolicy"] %}
-    - mountPath: /etc/resolv.conf
-      name: resolv
+    {% if job["usefreeflow"] %}
+    - mountPath: /freeflow
+      name: freeflow
     {% endif %}
     - mountPath: /dev/shm
       name: dshm
@@ -142,7 +148,7 @@ spec:
       value: "{{ job["distRoleIdx"] }}"
     {% for env in job["env"] %}
     - name: {{ env.name }}
-      value: {{ env.value }}
+      value: "{{ env.value }}"
     {% endfor %}
 
   imagePullSecrets:
@@ -164,18 +170,13 @@ spec:
   - name: authorized-keys-volume
     hostPath:
       path: {{ job["homeFolderHostpath"] }}/.ssh/authorized_keys
-  {% if job["usefreeflow"] %}
-  - name: freeflow
-    hostPath:
-      path: /freeflow
-  {% endif %}
   {% if not job["dnsPolicy"] %}
   - name: resolv
     hostPath:
       path: /etc/resolv.conf
   {% endif %}
-
   {% for mp in job["mountpoints"] %}
+  {% if mp.enabled %}
   - name: {{ mp.name }}
     {% if mp.emptydir %}
     emptyDir: {}
@@ -186,7 +187,13 @@ spec:
       type: {{ mp.type }}
       {% endif %}
     {% endif %}
+  {% endif %}
   {% endfor %}
+  {% if job["usefreeflow"] %}
+  - name: freeflow
+    hostPath:
+      path: /freeflow
+  {% endif %}
   - name: dshm
     emptyDir:
       medium: Memory
diff --git a/src/Jobs_Templete/RegularJob.yaml.template b/src/Jobs_Templete/RegularJob.yaml.template
index f0a66884f..1fc6daeb5 100755
--- a/src/Jobs_Templete/RegularJob.yaml.template
+++ b/src/Jobs_Templete/RegularJob.yaml.template
@@ -3,24 +3,26 @@ kind: Pod
 metadata:
   name: {{ job["podName"] }}
   labels:
-     run: {{ job["jobId"] }}
-     podName: {{ job["podName"] }}
-     jobName: {{ job["jobNameLabel"] }}
-     jobId: {{job["jobId"]}}
-     userName: {{ job["user"] }}
-     vcName: {{ job["vcName"] }}
-   {% if "gpuType" in job %}
+    run: {{ job["jobId"] }}
+    podName: {{ job["podName"] }}
+    jobName: {{ job["jobNameLabel"] }}
+    jobId: {{ job["jobId"] }}
+    userName: {{ job["user"] }}
+    vcName: {{ job["vcName"] }}
+  {% if "gpuType" in job %}
     {% if job["gpuType"]|length > 0 %}
-     gpuType: {{ job["gpuType"] }}
+    gpuType: {{ job["gpuType"] }}
     {% endif %}
-   {% endif %}
-     preemptionAllowed: "{{ job["preemptionAllowed"] }}"
+  {% endif %}
+    preemptionAllowed: "{{ job["preemptionAllowed"] }}"
+
   {% if "annotations" in job %}
   annotations:
     {% for annotationKey,annotationVal in job["annotations"].items() %}
       {{ annotationKey }}: {{ annotationVal }}
     {% endfor %}
   {% endif %}
+
 spec:
   nodeSelector:
     worker: active
@@ -48,9 +50,9 @@ spec:
     command: {{ job["LaunchCMD"] }}
     securityContext:
       runAsUser: {{ job["containerUserId"] }}
-      {% if job["isPrivileged"] %}
+    {% if job["isPrivileged"] %}
       privileged: true
-      {% endif %}
+    {% endif %}
       capabilities:
         add:
         - IPC_LOCK
@@ -69,7 +71,6 @@ spec:
       requests:
         memory: job["memoryrequest"]
     {% endif %}
-
     volumeMounts:
     - name: "init-user-script"
       mountPath: /dlws/init_user.sh
@@ -97,7 +98,6 @@ spec:
       readOnly: true
       {% endif %}
     {% endif %}
-
     {% endfor %}
     {% if job["usefreeflow"] %}
     - mountPath: /freeflow
@@ -110,8 +110,6 @@ spec:
       value: {{ job["familyToken"] }}
     - name: DLWS_REST_API
       value: {{ job["rest-api"] }}
-    - name: JOB_ID
-      value: {{ job["jobId"] }}
     - name: DLWS_JOB_ID
       value: {{ job["jobId"] }}
     - name: DLWS_NUM_WORKER
@@ -156,7 +154,6 @@ spec:
 
   restartPolicy: Never
   volumes:
-  # TODO need to create the configmap during installation: kubectl create configmap init-user-script --from-file=init_user.sh
   - name: "init-user-script"
     configMap:
       name: "init-user-script"

From f207da8eeac2d210727a39803617fa3fb24ee841 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 13:53:49 +0000
Subject: [PATCH 046/194] refine job template again

---
 src/ClusterManager/dist_pod_template.py    | 33 ++++++++++++++++++----
 src/ClusterManager/pod_template.py         |  6 +++-
 src/Jobs_Templete/DistJob.yaml.template    | 31 +++++++++++---------
 src/Jobs_Templete/RegularJob.yaml.template | 19 ++++++++-----
 4 files changed, 61 insertions(+), 28 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 9945e667c..5847a95bc 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -110,12 +110,6 @@ def generate_pod(self, pod):
 
         pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cmd)
 
-        random.seed(datetime.datetime.now())
-        if "hostNetwork" in pod and pod["hostNetwork"]:
-            pod["sshPort"] = random.randint(40000, 49999)
-        else:
-            pod["sshPort"] = int(random.random() * 1000 + 3000)
-
         pod_yaml = self.template.render(job=pod)
         return yaml.full_load(pod_yaml)
 
@@ -170,6 +164,10 @@ def generate_pods(self, job):
         params["numworker"] = int(params["numpsworker"])
         params["numps"] = int(params["numps"])
 
+        if "envs" not in params:
+            params["envs"] = []
+        params["envs"].append({"name": "DLWS_NUM_GPU_PER_WORKER", "value": params["resourcegpu"]})
+
         pods = []
         nums = {"ps": int(params["numps"]), "worker": int(params["numpsworker"])}
         for role in ["ps", "worker"]:
@@ -195,6 +193,29 @@ def generate_pods(self, job):
 
         k8s_pods = []
         for pod in pods:
+
+            random.seed(datetime.datetime.now())
+            if "hostNetwork" in pod and pod["hostNetwork"]:
+                pod["sshPort"] = random.randint(40000, 49999)
+            else:
+                pod["sshPort"] = int(random.random() * 1000 + 3000)
+
+            if (pod["distRole"] == "worker"):
+                pod["gpuLimit"] = pod["resourcegpu"]
+            else:
+                pod["gpuLimit"] = 0
+
+            if "envs" not in pod:
+                pod["envs"] = []
+            pod["envs"].append({"name": "DLWS_ROLE_NAME", "value": pod["distRole"]})
+            pod["envs"].append({"name": "DLWS_ROLE_IDX", "value": pod["distRoleIdx"]})
+
+            if "labels" not in pod:
+                pod["labels"] = []
+            pod["labels"].append({"name": "distRole", "value": pod["distRole"]})
+            pod["labels"].append({"name": "distRoleIdx", "value": pod["distRoleIdx"]})
+            pod["labels"].append({"name": "sshPort", "value": pod["sshPort"]})
+
             k8s_pod = self.generate_pod(pod)
             k8s_pods.append(k8s_pod)
 
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 6cab85f7b..5dbddf0c4 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -57,7 +57,7 @@ def generate_pod(self, pod):
                 pod["annotations"] = {}
             pod["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
             # TODO it's not safe to update pod["resourcegpu"]
-            pod["resourcegpu"] = 0  # gpu requests specified through annotation
+            pod["gpuLimit"] = 0  # gpu requests specified through annotation
 
         pod_yaml = self.template.render(job=pod)
         return yaml.full_load(pod_yaml)
@@ -129,6 +129,10 @@ def generate_pods(self, job):
 
         k8s_pods = []
         for pod in pods:
+            pod["numps"] = 0
+            pod["numworker"] = 1
+            pod["fragmentGpuJob"] = True
+            pod["gpuLimit"] = pod["resourcegpu"]
             k8s_pod = self.generate_pod(pod)
             k8s_pods.append(k8s_pod)
         return k8s_pods, None
diff --git a/src/Jobs_Templete/DistJob.yaml.template b/src/Jobs_Templete/DistJob.yaml.template
index 79f747207..f78e2bced 100755
--- a/src/Jobs_Templete/DistJob.yaml.template
+++ b/src/Jobs_Templete/DistJob.yaml.template
@@ -10,9 +10,9 @@ metadata:
     userName: {{ job["user"] }}
     vcName: {{ job["vcName"] }}
 
-    distRole: {{ job["distRole"] }}
-    distRoleIdx: "{{ job["distRoleIdx"] }}"
-    sshPort: "{{job["sshPort"]}}"
+  {% for label in job["labels"] %}
+    {{label.name}}: "{{label.value}}"
+  {% endfor %}
 
   {% if "gpuType" in job %}
     {% if job["gpuType"]|length > 0 %}
@@ -20,6 +20,14 @@ metadata:
     {% endif %}
   {% endif %}
     preemptionAllowed: "{{ job["preemptionAllowed"] }}"
+
+  {% if "annotations" in job %}
+  annotations:
+    {% for annotationKey,annotationVal in job["annotations"].items() %}
+      {{ annotationKey }}: {{ annotationVal }}
+    {% endfor %}
+  {% endif %}
+
 spec:
   nodeSelector:
     worker: active
@@ -28,6 +36,9 @@ spec:
     {{key}}: {{value}}
   {% endfor %}
   {% endif %}
+  {% if job["fragmentGpuJob"] %}
+    FragmentGPUJob: active
+  {% endif %}
   {% if job["dnsPolicy"] %}
   dnsPolicy: {{ job["dnsPolicy" ]}}
   {% endif %}
@@ -51,10 +62,9 @@ spec:
         add:
         - IPC_LOCK
         - SYS_ADMIN
-    {% if job["distRole"] =="worker" %}
     resources:
       limits:
-        nvidia.com/gpu: {{ job["resourcegpu"] }}
+        nvidia.com/gpu: {{ job["gpuLimit"] }}
     {% if not job["cpurequest"] %}
       requests:
         cpu: 1.0
@@ -66,7 +76,6 @@ spec:
       requests:
         memory: job["memoryrequest"]
     {% endif %}
-    {% endif %}
     volumeMounts:
     - name: "init-user-script"
       mountPath: /dlws/init_user.sh
@@ -112,9 +121,7 @@ spec:
       value: "{{ job["numps"] }}"
     - name: DLWS_NUM_WORKER
       value: "{{ job["numworker"] }}"
-    - name: DLWS_NUM_GPU_PER_WORKER
-      value: "{{ job["resourcegpu"] }}"
-    {% if job["distRole"] =="ps" or not job["resourcegpu"] is defined or job["resourcegpu"]|int < 1  %}
+    {% if job["gpuLimit"]|int < 1  %}
     - name: NVIDIA_VISIBLE_DEVICES
       value: ""
     {% endif %}
@@ -142,11 +149,7 @@ spec:
       value: "{{ job["user_email"] }}"
     - name: DLWS_VC_NAME
       value: {{ job["vcName"] }}
-    - name: DLWS_ROLE_NAME
-      value: {{ job["distRole"] }}
-    - name: DLWS_ROLE_IDX
-      value: "{{ job["distRoleIdx"] }}"
-    {% for env in job["env"] %}
+    {% for env in job["envs"] %}
     - name: {{ env.name }}
       value: "{{ env.value }}"
     {% endfor %}
diff --git a/src/Jobs_Templete/RegularJob.yaml.template b/src/Jobs_Templete/RegularJob.yaml.template
index 1fc6daeb5..f78e2bced 100755
--- a/src/Jobs_Templete/RegularJob.yaml.template
+++ b/src/Jobs_Templete/RegularJob.yaml.template
@@ -9,6 +9,11 @@ metadata:
     jobId: {{ job["jobId"] }}
     userName: {{ job["user"] }}
     vcName: {{ job["vcName"] }}
+
+  {% for label in job["labels"] %}
+    {{label.name}}: "{{label.value}}"
+  {% endfor %}
+
   {% if "gpuType" in job %}
     {% if job["gpuType"]|length > 0 %}
     gpuType: {{ job["gpuType"] }}
@@ -31,7 +36,7 @@ spec:
     {{key}}: {{value}}
   {% endfor %}
   {% endif %}
-  {% if job["resourcegpu"]|int < 8  %}
+  {% if job["fragmentGpuJob"] %}
     FragmentGPUJob: active
   {% endif %}
   {% if job["dnsPolicy"] %}
@@ -59,7 +64,7 @@ spec:
         - SYS_ADMIN
     resources:
       limits:
-        nvidia.com/gpu: {{ job["resourcegpu"] }}
+        nvidia.com/gpu: {{ job["gpuLimit"] }}
     {% if not job["cpurequest"] %}
       requests:
         cpu: 1.0
@@ -112,11 +117,11 @@ spec:
       value: {{ job["rest-api"] }}
     - name: DLWS_JOB_ID
       value: {{ job["jobId"] }}
+    - name: DLWS_NUM_PS
+      value: "{{ job["numps"] }}"
     - name: DLWS_NUM_WORKER
-      value: "1"
-    - name: DLWS_NUM_GPU_PER_WORKER
-      value: "{{ job["resourcegpu"] }}"
-    {% if job["resourcegpu"]|int < 1  %}
+      value: "{{ job["numworker"] }}"
+    {% if job["gpuLimit"]|int < 1  %}
     - name: NVIDIA_VISIBLE_DEVICES
       value: ""
     {% endif %}
@@ -144,7 +149,7 @@ spec:
       value: "{{ job["user_email"] }}"
     - name: DLWS_VC_NAME
       value: {{ job["vcName"] }}
-    {% for env in job["env"] %}
+    {% for env in job["envs"] %}
     - name: {{ env.name }}
       value: "{{ env.value }}"
     {% endfor %}

From 441b8d89b6a75268207ef5432976cc91a02da080 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 14:02:36 +0000
Subject: [PATCH 047/194] merge Regular/Dist job template

---
 src/ClusterManager/job.py                     |  11 +-
 src/ClusterManager/job_manager.py             |   2 +-
 src/ClusterManager/pod_template.py            |   6 +-
 src/ClusterManager/test_pod_template.py       |  12 +-
 src/Jobs_Templete/RegularJob.yaml.template    | 202 ------------------
 ...istJob.yaml.template => pod.yaml.template} |   0
 6 files changed, 11 insertions(+), 222 deletions(-)
 delete mode 100755 src/Jobs_Templete/RegularJob.yaml.template
 rename src/Jobs_Templete/{DistJob.yaml.template => pod.yaml.template} (100%)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 70f74e8b8..54fb07b47 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -113,16 +113,7 @@ def data_path_mountpoint(self):
 
     def get_template(self):
         """Return jinja template."""
-        path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "RegularJob.yaml.template"))
-        ENV = Environment(loader=FileSystemLoader("/"))
-        template = ENV.get_template(path)
-        assert(isinstance(template, Template))
-        return template
-
-    # TODO remove it later
-    def get_dist_template(self):
-        """Return jinja template."""
-        path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "DistJob.yaml.template"))
+        path = os.path.abspath(os.path.join(self.cluster["root-path"], "Jobs_Templete", "pod.yaml.template"))
         ENV = Environment(loader=FileSystemLoader("/"))
         template = ENV.get_template(path)
         assert(isinstance(template, Template))
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 7731c57df..8c07ae989 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -69,7 +69,7 @@ def SubmitJob(job):
         if job_object.params["jobtrainingtype"] == "RegularJob":
             pod_template = PodTemplate(job_object.get_template(), enable_custom_scheduler)
         elif job_object.params["jobtrainingtype"] == "PSDistJob":
-            pod_template = DistPodTemplate(job_object.get_dist_template())
+            pod_template = DistPodTemplate(job_object.get_template())
         else:
             dataHandler.SetJobError(job_object.job_id, "ERROR: invalid jobtrainingtype: %s" % job_object.params["jobtrainingtype"])
             return False
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 5dbddf0c4..fcdde913c 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -39,7 +39,7 @@ def generate_pod(self, pod):
                 # for cases when desired topology is explictly given or not desired
                 gpu_topology_flag = 0
             pod_name = pod["podName"]
-            request_gpu = int(pod["resourcegpu"])
+            request_gpu = int(pod["gpuLimit"])
 
             podInfo = {
                 "podname": pod_name,
@@ -56,8 +56,8 @@ def generate_pod(self, pod):
             if "annotations" not in pod:
                 pod["annotations"] = {}
             pod["annotations"]["pod.alpha/DeviceInformation"] = "'" + json.dumps(podInfo) + "'"
-            # TODO it's not safe to update pod["resourcegpu"]
-            pod["gpuLimit"] = 0  # gpu requests specified through annotation
+            # gpu requests specified through annotation
+            pod["gpuLimit"] = 0
 
         pod_yaml = self.template.render(job=pod)
         return yaml.full_load(pod_yaml)
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index e336c473a..9e57276f7 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -31,18 +31,18 @@ def test_generate_launch_script(self):
 
         script_file = PodTemplate.generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script)
 
-        # "launch-%s.sh" % self.job_id
-        self.assertEqual("launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh", script_file)
+        # return the container command
+        self.assertEqual('["bash", "/job/launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh"]', script_file)
 
     def test_pod_template_without_custer_scheduler(self):
         enable_custom_scheduler = False
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
 
-        pod = {"resourcegpu": 2}
+        pod = {"gpuLimit": 2}
         data = pod_template.generate_pod(pod)
 
         # not eanbled custom scheduler, set the resource limits: spec.containers[].resources.limits
-        self.assertEqual(pod["resourcegpu"], data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
+        self.assertEqual(pod["gpuLimit"], data["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"])
         # metadata.annotations["pod.alpha/DeviceInformation"] should be empty
         self.assertTrue(("annotations" not in data["metadata"]) or ("pod.alpha/DeviceInformation" not in data["metadata"]["annotations"]))
 
@@ -53,7 +53,7 @@ def test_pod_template_with_custom_scheduler(self):
         gpu_num = 2
         pod = {
             "podName": "790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0",
-            "resourcegpu": gpu_num,
+            "gpuLimit": gpu_num,
         }
         data = pod_template.generate_pod(pod)
 
@@ -74,7 +74,7 @@ def test_pod_template_with_custom_scheduler_use_topology(self):
         gpu_num = 2
         pod = {
             "podName": "790a6b30-560f-44a4-a9f0-5d1458dcb0d1-pod-0",
-            "resourcegpu": gpu_num,
+            "gpuLimit": gpu_num,
             "useGPUTopology": True
         }
         data = pod_template.generate_pod(pod)
diff --git a/src/Jobs_Templete/RegularJob.yaml.template b/src/Jobs_Templete/RegularJob.yaml.template
deleted file mode 100755
index f78e2bced..000000000
--- a/src/Jobs_Templete/RegularJob.yaml.template
+++ /dev/null
@@ -1,202 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: {{ job["podName"] }}
-  labels:
-    run: {{ job["jobId"] }}
-    podName: {{ job["podName"] }}
-    jobName: {{ job["jobNameLabel"] }}
-    jobId: {{ job["jobId"] }}
-    userName: {{ job["user"] }}
-    vcName: {{ job["vcName"] }}
-
-  {% for label in job["labels"] %}
-    {{label.name}}: "{{label.value}}"
-  {% endfor %}
-
-  {% if "gpuType" in job %}
-    {% if job["gpuType"]|length > 0 %}
-    gpuType: {{ job["gpuType"] }}
-    {% endif %}
-  {% endif %}
-    preemptionAllowed: "{{ job["preemptionAllowed"] }}"
-
-  {% if "annotations" in job %}
-  annotations:
-    {% for annotationKey,annotationVal in job["annotations"].items() %}
-      {{ annotationKey }}: {{ annotationVal }}
-    {% endfor %}
-  {% endif %}
-
-spec:
-  nodeSelector:
-    worker: active
-  {% if job["nodeSelector"]|length > 0 %}
-  {% for key, value in job["nodeSelector"].items() %}
-    {{key}}: {{value}}
-  {% endfor %}
-  {% endif %}
-  {% if job["fragmentGpuJob"] %}
-    FragmentGPUJob: active
-  {% endif %}
-  {% if job["dnsPolicy"] %}
-  dnsPolicy: {{ job["dnsPolicy" ]}}
-  {% endif %}
-  {% if job["hostNetwork"] %}
-  hostNetwork: true
-  {% endif %}
-  {% if job["hostIPC"] %}
-  hostIPC: true
-  {% endif %}
-  containers:
-  - name: {{ job["podName"] }}
-    image: {{ job["image"] }}
-    imagePullPolicy: Always
-    command: {{ job["LaunchCMD"] }}
-    securityContext:
-      runAsUser: {{ job["containerUserId"] }}
-    {% if job["isPrivileged"] %}
-      privileged: true
-    {% endif %}
-      capabilities:
-        add:
-        - IPC_LOCK
-        - SYS_ADMIN
-    resources:
-      limits:
-        nvidia.com/gpu: {{ job["gpuLimit"] }}
-    {% if not job["cpurequest"] %}
-      requests:
-        cpu: 1.0
-    {% else %}
-      requests:
-        cpu: job["cpurequest"]
-    {% endif %}
-    {% if job["memoryrequest"] %}
-      requests:
-        memory: job["memoryrequest"]
-    {% endif %}
-    volumeMounts:
-    - name: "init-user-script"
-      mountPath: /dlws/init_user.sh
-      subPath: init_user.sh
-    - name: ssh-volume
-      mountPath: /home/{{ job["user"] }}/.ssh
-    - name: id-rsa-volume
-      mountPath: /home/{{ job["user"] }}/.ssh/id_rsa
-      readOnly: true
-    - name: id-rsa-pub-volume
-      mountPath: /home/{{ job["user"] }}/.ssh/id_rsa.pub
-      readOnly: true
-    - name: authorized-keys-volume
-      mountPath: /home/{{ job["user"] }}/.ssh/authorized_keys
-      readOnly: true
-    {% if not job["dnsPolicy"] %}
-    - mountPath: /etc/resolv.conf
-      name: resolv
-    {% endif %}
-    {% for mp in job["mountpoints"] %}
-    {% if mp.enabled %}
-    - mountPath: {{ mp.containerPath }}
-      name: {{ mp.name }}
-      {% if mp.readOnly %}
-      readOnly: true
-      {% endif %}
-    {% endif %}
-    {% endfor %}
-    {% if job["usefreeflow"] %}
-    - mountPath: /freeflow
-      name: freeflow
-    {% endif %}
-    - mountPath: /dev/shm
-      name: dshm
-    env:
-    - name: FAMILY_TOKEN
-      value: {{ job["familyToken"] }}
-    - name: DLWS_REST_API
-      value: {{ job["rest-api"] }}
-    - name: DLWS_JOB_ID
-      value: {{ job["jobId"] }}
-    - name: DLWS_NUM_PS
-      value: "{{ job["numps"] }}"
-    - name: DLWS_NUM_WORKER
-      value: "{{ job["numworker"] }}"
-    {% if job["gpuLimit"]|int < 1  %}
-    - name: NVIDIA_VISIBLE_DEVICES
-      value: ""
-    {% endif %}
-    {% if job["usefreeflow"] %}
-    - name: VNET_PREFIX
-      value: {{ job["pod_ip_range"] }}
-    - name: LD_PRELOAD
-      value: "/freeflow/libfsocket.so"
-    {% endif %}
-    - name: POD_NAME
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.name
-    - name: POD_IP
-      valueFrom:
-        fieldRef:
-          fieldPath: status.podIP
-    - name: DLWS_GID
-      value: "{{ job["gid"] }}"
-    - name: DLWS_UID
-      value: "{{ job["uid"] }}"
-    - name: DLWS_USER_NAME
-      value: "{{ job["user"] }}"
-    - name: DLWS_USER_EMAIL
-      value: "{{ job["user_email"] }}"
-    - name: DLWS_VC_NAME
-      value: {{ job["vcName"] }}
-    {% for env in job["envs"] %}
-    - name: {{ env.name }}
-      value: "{{ env.value }}"
-    {% endfor %}
-
-  imagePullSecrets:
-  - name: regcred
-
-  restartPolicy: Never
-  volumes:
-  - name: "init-user-script"
-    configMap:
-      name: "init-user-script"
-  - name: ssh-volume
-    emptyDir: {}
-  - name: id-rsa-volume
-    hostPath:
-      path: {{ job["homeFolderHostpath"] }}/.ssh/id_rsa
-  - name: id-rsa-pub-volume
-    hostPath:
-      path: {{ job["homeFolderHostpath"] }}/.ssh/id_rsa.pub
-  - name: authorized-keys-volume
-    hostPath:
-      path: {{ job["homeFolderHostpath"] }}/.ssh/authorized_keys
-  {% if not job["dnsPolicy"] %}
-  - name: resolv
-    hostPath:
-      path: /etc/resolv.conf
-  {% endif %}
-  {% for mp in job["mountpoints"] %}
-  {% if mp.enabled %}
-  - name: {{ mp.name }}
-    {% if mp.emptydir %}
-    emptyDir: {}
-    {% else %}
-    hostPath:
-      path: {{ mp.hostPath }}
-      {% if mp.type %}
-      type: {{ mp.type }}
-      {% endif %}
-    {% endif %}
-  {% endif %}
-  {% endfor %}
-  {% if job["usefreeflow"] %}
-  - name: freeflow
-    hostPath:
-      path: /freeflow
-  {% endif %}
-  - name: dshm
-    emptyDir:
-      medium: Memory
diff --git a/src/Jobs_Templete/DistJob.yaml.template b/src/Jobs_Templete/pod.yaml.template
similarity index 100%
rename from src/Jobs_Templete/DistJob.yaml.template
rename to src/Jobs_Templete/pod.yaml.template

From c3018ef4c2adac7044f6032eccaf410521a9b541 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 4 Jul 2019 15:00:57 +0000
Subject: [PATCH 048/194] fix pod["envs"] not properly generated

---
 src/ClusterManager/test_pod_template.py | 24 ++++++++++++++++++++++++
 src/Jobs_Templete/pod.yaml.template     |  4 ++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index 9e57276f7..651588013 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -46,6 +46,30 @@ def test_pod_template_without_custer_scheduler(self):
         # metadata.annotations["pod.alpha/DeviceInformation"] should be empty
         self.assertTrue(("annotations" not in data["metadata"]) or ("pod.alpha/DeviceInformation" not in data["metadata"]["annotations"]))
 
+    def test_generate_pod_with_envs(self):
+        enable_custom_scheduler = False
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        pod = {
+            "gpuLimit": 2,
+            "envs": [{"name": "my_env_name", "value": "my_env_value"}],
+        }
+        data = pod_template.generate_pod(pod)
+
+        self.assertIn({"name": "my_env_name", "value": "my_env_value"}, data["spec"]["containers"][0]["env"])
+
+    def test_generate_pod_with_labels(self):
+        enable_custom_scheduler = False
+        pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
+
+        pod = {
+            "gpuLimit": 2,
+            "labels": [{"name": "my_label_name", "value": "my_label_value"}],
+        }
+        data = pod_template.generate_pod(pod)
+
+        self.assertEqual("my_label_value", data["metadata"]["labels"]["my_label_name"])
+
     def test_pod_template_with_custom_scheduler(self):
         enable_custom_scheduler = True
         pod_template = PodTemplate(job.get_template(), enable_custom_scheduler)
diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index f78e2bced..6bf0bc11a 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -150,8 +150,8 @@ spec:
     - name: DLWS_VC_NAME
       value: {{ job["vcName"] }}
     {% for env in job["envs"] %}
-    - name: {{ env.name }}
-      value: "{{ env.value }}"
+    - name: {{env.name}}
+      value: "{{env.value}}"
     {% endfor %}
 
   imagePullSecrets:

From 5bdd37fdfcf035e3009a902df70fa08fd485f5cb Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 5 Jul 2019 02:22:52 +0000
Subject: [PATCH 049/194] add config.yaml for unittest

---
 src/utils/config.yaml | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 src/utils/config.yaml

diff --git a/src/utils/config.yaml b/src/utils/config.yaml
new file mode 100644
index 000000000..7f3e1964e
--- /dev/null
+++ b/src/utils/config.yaml
@@ -0,0 +1,36 @@
+# It's a dummy config, for unit test only!!!
+
+# database :
+#   hostname :
+#   username :
+#   password :
+#   database :
+# mysql :
+#   hostname : hao-dev-infra01.eastus.cloudapp.azure.com
+#   port : 3306
+#   username : root
+#   password : MSFT1234!
+# kubelet-path : /usr/local/bin/kubectl
+storage-mount-path : /dlwsdata
+# root-path : /DLWorkspace/src/
+root-path : ".." # for unittest only
+# nvidiaDriverPath : /opt/nvidia-driver/current
+# clusterId : b6815f6d-612c-4feb-a8f3-bc7ceaa43993
+# domain : eastus.cloudapp.azure.com
+# apiserver: https://hao-dev-infra01.eastus.cloudapp.azure.com:1443
+# certificate-authority: /etc/kubernetes/ssl/ca.pem
+# client-certificate: /etc/kubernetes/ssl/apiserver.pem
+# client-key: /etc/kubernetes/ssl/apiserver-key.pem
+# pod_ip_range: 10.2.0.0/16
+# per_user_gpu_limit: -1
+rest-api: http://faked.uri/
+# usefreeflow: False
+# mountdescription : {'hdfs': 'Hadoop file system (replicated distribute storage).', 'azurefileshare': 'Azure file storage', 'localHDD': 'Local HDD. ', 'glusterfs': 'GlusterFS (replicated distributed storage)', 'nfs': 'NFS (remote file share)', 'emptyDir': 'Kubernetes emptyDir (folder will be erased after job termination).', 'local': 'Local SSD. '}
+# mountpoints : {'rootshare': {'curphysicalmountpoint': '/mntdlws/nfs', 'mountpoints': '', 'server': '192.168.255.1', 'type': 'nfs', 'filesharename': '/data/share'}}
+# mounthomefolder : yes
+# deploymounts : []
+# default-storage-folders : ['jobfiles', 'storage', 'work', 'namenodeshare']
+# webportal_node: hao-dev-infra01.eastus.cloudapp.azure.com
+# datasource : MySQL
+# kube_custom_scheduler: False
+# WinbindServers: []

From 80af8df67ec95e4087c5a624ea6b4161d17e7200 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 5 Jul 2019 07:46:38 +0000
Subject: [PATCH 050/194] refine

---
 src/ClusterManager/job.py          | 13 +++++++------
 src/ClusterManager/pod_template.py |  2 +-
 src/ClusterManager/test_job.py     |  4 ++--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index 54fb07b47..ad18a53db 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -90,20 +90,21 @@ def add_mountpoints(self, mountpoint):
     def get_alias(self):
         return self.email.split("@")[0].strip()
 
-    def get_homefolder_hostpath(self):
-        return os.path.join(self.cluster["storage-mount-path"], "work/", self.get_alias())
+    def get_hostpath(self, path_relate_to_workpath):
+        """return os.path.join(self.cluster["storage-mount-path"], "work", path_relate_to_workpath)"""
+        return os.path.join(self.cluster["storage-mount-path"], "work", path_relate_to_workpath)
 
-    def get_local_job_path(self):
-        return os.path.join(self.cluster["storage-mount-path"], "work/", self.job_path)
+    def get_homefolder_hostpath(self):
+        return self.get_hostpath(self.get_alias())
 
     def job_path_mountpoint(self):
         assert(len(self.job_path) > 0)
-        job_host_path = os.path.join(self.cluster["storage-mount-path"], "work", self.job_path)
+        job_host_path = self.get_hostpath(self.job_path)
         return {"name": "job", "containerPath": "/job", "hostPath": job_host_path, "enabled": True}
 
     def work_path_mountpoint(self):
         assert(len(self.work_path) > 0)
-        work_host_path = os.path.join(self.cluster["storage-mount-path"], "work", self.work_path)
+        work_host_path = self.get_hostpath(self.work_path)
         return {"name": "work", "containerPath": "/work", "hostPath": work_host_path, "enabled": True}
 
     def data_path_mountpoint(self):
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index fcdde913c..21e22c8bd 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -106,7 +106,7 @@ def generate_pods(self, job):
         if "gpuType" in params:
             params["nodeSelector"]["gpuType"] = params["gpuType"]
 
-        local_job_path = job.get_local_job_path()
+        local_job_path = job.get_hostpath(job.job_path)
         params["LaunchCMD"] = PodTemplate.generate_launch_script(params["jobId"], local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
 
         pods = []
diff --git a/src/ClusterManager/test_job.py b/src/ClusterManager/test_job.py
index eff8c5c22..76fa8e299 100644
--- a/src/ClusterManager/test_job.py
+++ b/src/ClusterManager/test_job.py
@@ -130,10 +130,10 @@ def test_get_homefolder_hostpath(self):
         job = self.create_a_job()
         self.assertEqual("/dlwsdata/work/user", job.get_homefolder_hostpath())
 
-    def test_get_local_job_path(self):
+    def test_get_hostpath(self):
         job = self.create_a_job()
         self.assertEqual("user_alias/jobs/date/job_id", job.job_path)
-        self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.get_local_job_path())
+        self.assertEqual("/dlwsdata/work/user_alias/jobs/date/job_id", job.get_hostpath(job.job_path))
 
     def test_job_work_data_mountpoints(self):
         job = self.create_a_job()

From 5c6c0b4355d766ff3648f2e2de0af822484969ec Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 5 Jul 2019 07:58:40 +0000
Subject: [PATCH 051/194] mount /pod for Regular job

---
 src/ClusterManager/job.py          | 6 +++---
 src/ClusterManager/pod_template.py | 5 +++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/job.py b/src/ClusterManager/job.py
index ad18a53db..993f35b58 100644
--- a/src/ClusterManager/job.py
+++ b/src/ClusterManager/job.py
@@ -90,9 +90,9 @@ def add_mountpoints(self, mountpoint):
     def get_alias(self):
         return self.email.split("@")[0].strip()
 
-    def get_hostpath(self, path_relate_to_workpath):
-        """return os.path.join(self.cluster["storage-mount-path"], "work", path_relate_to_workpath)"""
-        return os.path.join(self.cluster["storage-mount-path"], "work", path_relate_to_workpath)
+    def get_hostpath(self, *path_relate_to_workpath):
+        """return os.path.join(self.cluster["storage-mount-path"], "work", *path_relate_to_workpath)"""
+        return os.path.join(self.cluster["storage-mount-path"], "work", *path_relate_to_workpath)
 
     def get_homefolder_hostpath(self):
         return self.get_hostpath(self.get_alias())
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 21e22c8bd..e45fb9b2a 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -133,6 +133,11 @@ def generate_pods(self, job):
             pod["numworker"] = 1
             pod["fragmentGpuJob"] = True
             pod["gpuLimit"] = pod["resourcegpu"]
+
+            # mount /pod
+            pod_path = job.get_hostpath(job.job_path, "master")
+            pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": pod_path, "enabled": True})
+
             k8s_pod = self.generate_pod(pod)
             k8s_pods.append(k8s_pod)
         return k8s_pods, None

From 72da3de9cc06471b0fcff1d828a02ed24c905bbc Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 5 Jul 2019 10:42:38 +0000
Subject: [PATCH 052/194] mount /pod for dist job

---
 src/ClusterManager/dist_pod_template.py | 80 +++++++++++++------------
 1 file changed, 41 insertions(+), 39 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 5847a95bc..637d5d6cf 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -103,11 +103,49 @@ def generate_pod(self, pod):
 
         dist_id = pod["distId"]
         job_id = pod["jobId"]
-        user_alias = pod["user"]
         job_path = pod["jobPath"]
+
+        pod["podName"] = "{}-{}".format(job_id, dist_id)
+
+        # TODO refine later
+        dist_job_path = os.path.join(job_path, dist_id)
+        for mp in pod["mountpoints"]:
+            if mp["name"] == "job":
+                mp["hostPath"] = mp["hostPath"] + "/" + dist_id
+
+        local_job_path = os.path.join(config["storage-mount-path"], "work/", dist_job_path)
+        if not os.path.exists(local_job_path):
+            mkdirsAsUser(local_job_path, pod["userId"])
+
+        random.seed(datetime.datetime.now())
+        if "hostNetwork" in pod and pod["hostNetwork"]:
+            pod["sshPort"] = random.randint(40000, 49999)
+        else:
+            pod["sshPort"] = int(random.random() * 1000 + 3000)
+
+        if (pod["distRole"] == "worker"):
+            pod["gpuLimit"] = pod["resourcegpu"]
+        else:
+            pod["gpuLimit"] = 0
+
+        if "envs" not in pod:
+            pod["envs"] = []
+        pod["envs"].append({"name": "DLWS_ROLE_NAME", "value": pod["distRole"]})
+        pod["envs"].append({"name": "DLWS_ROLE_IDX", "value": pod["distRoleIdx"]})
+
+        if "labels" not in pod:
+            pod["labels"] = []
+        pod["labels"].append({"name": "distRole", "value": pod["distRole"]})
+        pod["labels"].append({"name": "distRoleIdx", "value": pod["distRoleIdx"]})
+        pod["labels"].append({"name": "sshPort", "value": pod["sshPort"]})
+
+        # mount /pod
+        pod_path = os.path.join(config["storage-mount-path"], "work", job_path, dist_id)
+        pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": pod_path, "enabled": True})
+
+        user_alias = pod["user"]
         worker_num = pod["numpsworker"]
         cmd = pod["cmd"]
-
         pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cmd)
 
         pod_yaml = self.template.render(job=pod)
@@ -173,49 +211,13 @@ def generate_pods(self, job):
         for role in ["ps", "worker"]:
             for idx in range(nums[role]):
                 pod = copy.deepcopy(params)
-
-                pod["distId"] = "%s%d" % (role, idx)
                 pod["distRole"] = role
                 pod["distRoleIdx"] = idx
-                pod["podName"] = "{}-{}".format(pod["jobId"], pod["distId"])
-
-                # TODO refine later
-                dist_job_path = os.path.join(job.job_path, pod["distId"])
-                for mp in pod["mountpoints"]:
-                    if mp["name"] == "job":
-                        mp["hostPath"] = mp["hostPath"] + "/" + pod["distId"]
-
-                local_job_path = os.path.join(config["storage-mount-path"], "work/", dist_job_path)
-                if not os.path.exists(local_job_path):
-                    mkdirsAsUser(local_job_path, pod["userId"])
-
+                pod["distId"] = "%s%d" % (role, idx)
                 pods.append(pod)
 
         k8s_pods = []
         for pod in pods:
-
-            random.seed(datetime.datetime.now())
-            if "hostNetwork" in pod and pod["hostNetwork"]:
-                pod["sshPort"] = random.randint(40000, 49999)
-            else:
-                pod["sshPort"] = int(random.random() * 1000 + 3000)
-
-            if (pod["distRole"] == "worker"):
-                pod["gpuLimit"] = pod["resourcegpu"]
-            else:
-                pod["gpuLimit"] = 0
-
-            if "envs" not in pod:
-                pod["envs"] = []
-            pod["envs"].append({"name": "DLWS_ROLE_NAME", "value": pod["distRole"]})
-            pod["envs"].append({"name": "DLWS_ROLE_IDX", "value": pod["distRoleIdx"]})
-
-            if "labels" not in pod:
-                pod["labels"] = []
-            pod["labels"].append({"name": "distRole", "value": pod["distRole"]})
-            pod["labels"].append({"name": "distRoleIdx", "value": pod["distRoleIdx"]})
-            pod["labels"].append({"name": "sshPort", "value": pod["sshPort"]})
-
             k8s_pod = self.generate_pod(pod)
             k8s_pods.append(k8s_pod)
 

From e4bb0ef3d79f36ba9add5965509a4f251c8597fb Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 5 Jul 2019 11:27:14 +0000
Subject: [PATCH 053/194] mount /pod for each pod, put launcher.sh under it

---
 src/ClusterManager/dist_pod_template.py | 34 ++++++++++---------------
 src/ClusterManager/pod_template.py      |  8 +++---
 src/ClusterManager/test_pod_template.py |  2 +-
 3 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 637d5d6cf..2557bd7e3 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -20,7 +20,7 @@ def __init__(self, template, enable_custom_scheduler=False):
         self.enable_custom_scheduler = enable_custom_scheduler
 
     @staticmethod
-    def generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cmd):
+    def generate_launch_script(dist_id, job_id, user_id, user_alias, job_path, worker_num, cmd):
         # change ssh folder permission here because the setup permission
         #  script in launch_ps_job function may have race condition with init_user.sh script.
         # results in no such user error
@@ -81,21 +81,23 @@ def generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cm
 sudo mkdir -p /opt && sudo ln -s /job/hostfile /opt/hostfile  &>/dev/null;
 
 # TODO mark the worker as 'READY', better to change to '/pod/READY' later
-sudo touch /job/WORKER_READY
+sudo touch /pod/WORKER_READY
 
 sleep infinity
 """.format(user_alias)
 
-        local_job_path = os.path.join(config["storage-mount-path"], "work/", job_path, dist_id)
+        local_pod_path = os.path.join(config["storage-mount-path"], "work/", job_path, dist_id)
+        if not os.path.exists(local_pod_path):
+            mkdirsAsUser(local_pod_path, user_id)
         file_name = "launch-%s-%s.sh" % (job_id, dist_id)
-        launch_script_file = os.path.join(local_job_path, file_name)
+        launch_script_file = os.path.join(local_pod_path, file_name)
         with open(launch_script_file, 'w') as f:
             f.write(script)
         f.close()
 
-        launchScriptInContainer = "bash /job/launch-%s-%s.sh" % (job_id, dist_id)
+        launchScriptInContainer = "bash /pod/launch-%s-%s.sh" % (job_id, dist_id)
 
-        launchCMD = '["bash", "-c", "bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
+        launchCMD = '["bash", "-c", "bash /dlws/init_user.sh &>> /pod/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
         return launchCMD
 
     def generate_pod(self, pod):
@@ -107,16 +109,6 @@ def generate_pod(self, pod):
 
         pod["podName"] = "{}-{}".format(job_id, dist_id)
 
-        # TODO refine later
-        dist_job_path = os.path.join(job_path, dist_id)
-        for mp in pod["mountpoints"]:
-            if mp["name"] == "job":
-                mp["hostPath"] = mp["hostPath"] + "/" + dist_id
-
-        local_job_path = os.path.join(config["storage-mount-path"], "work/", dist_job_path)
-        if not os.path.exists(local_job_path):
-            mkdirsAsUser(local_job_path, pod["userId"])
-
         random.seed(datetime.datetime.now())
         if "hostNetwork" in pod and pod["hostNetwork"]:
             pod["sshPort"] = random.randint(40000, 49999)
@@ -139,14 +131,10 @@ def generate_pod(self, pod):
         pod["labels"].append({"name": "distRoleIdx", "value": pod["distRoleIdx"]})
         pod["labels"].append({"name": "sshPort", "value": pod["sshPort"]})
 
-        # mount /pod
-        pod_path = os.path.join(config["storage-mount-path"], "work", job_path, dist_id)
-        pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": pod_path, "enabled": True})
-
         user_alias = pod["user"]
         worker_num = pod["numpsworker"]
         cmd = pod["cmd"]
-        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, job_id, user_alias, job_path, worker_num, cmd)
+        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, job_id, pod["userId"], user_alias, job_path, worker_num, cmd)
 
         pod_yaml = self.template.render(job=pod)
         return yaml.full_load(pod_yaml)
@@ -214,6 +202,10 @@ def generate_pods(self, job):
                 pod["distRole"] = role
                 pod["distRoleIdx"] = idx
                 pod["distId"] = "%s%d" % (role, idx)
+                # mount /pod
+                local_pod_path = job.get_hostpath(job.job_path, pod["distId"])
+                pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": local_pod_path, "enabled": True})
+
                 pods.append(pod)
 
         k8s_pods = []
diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index e45fb9b2a..eac50bff9 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -25,9 +25,9 @@ def generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script):
             f.write("#!/bin/bash -x\n")
             f.write("mkdir /opt; \n")
             f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % gpu_num)
-            f.write("bash /dlws/init_user.sh &> /job/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
+            f.write("bash /dlws/init_user.sh &>> /pod/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
         os.system("sudo chown %s %s" % (user_id, launch_script_file))
-        luanch_cmd = "[\"bash\", \"/job/%s\"]" % file_name
+        luanch_cmd = "[\"bash\", \"/pod/%s\"]" % file_name
         return luanch_cmd
 
     def generate_pod(self, pod):
@@ -106,8 +106,8 @@ def generate_pods(self, job):
         if "gpuType" in params:
             params["nodeSelector"]["gpuType"] = params["gpuType"]
 
-        local_job_path = job.get_hostpath(job.job_path)
-        params["LaunchCMD"] = PodTemplate.generate_launch_script(params["jobId"], local_job_path, params["userId"], params["resourcegpu"], params["cmd"])
+        local_pod_path = job.get_hostpath(job.job_path, "master")
+        params["LaunchCMD"] = PodTemplate.generate_launch_script(params["jobId"], local_pod_path, params["userId"], params["resourcegpu"], params["cmd"])
 
         pods = []
         if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index 651588013..99df9df99 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -32,7 +32,7 @@ def test_generate_launch_script(self):
         script_file = PodTemplate.generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script)
 
         # return the container command
-        self.assertEqual('["bash", "/job/launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh"]', script_file)
+        self.assertEqual('["bash", "/pod/launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh"]', script_file)
 
     def test_pod_template_without_custer_scheduler(self):
         enable_custom_scheduler = False

From 15e1e9eb1d3908b019369be9f57cbc4b968afbb9 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 10 Jul 2019 07:56:31 +0000
Subject: [PATCH 054/194] add missing cluster prefix in alert manager

---
 src/ClusterBootstrap/services/monitor/alert-manager.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterBootstrap/services/monitor/alert-manager.yaml b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
index 8bc493e6d..a071618ca 100644
--- a/src/ClusterBootstrap/services/monitor/alert-manager.yaml
+++ b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
@@ -80,7 +80,7 @@ data:
       receiver: alert-email
       group_wait: 30s
       group_interval: 5m
-      group_by: [alertname]
+      group_by: [alertname, cluster]
       routes:
       - receiver: task_user
         repeat_interval: 4h

From e31e93e5d69d97740ba91819c4860c4ee9ede75a Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 11 Jul 2019 03:35:04 +0000
Subject: [PATCH 055/194] add cluster gpu statistic dashboard

---
 .../cluster-gpu-statistic-dashboard.json      | 239 ++++++++++++++++++
 1 file changed, 239 insertions(+)
 create mode 100644 src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json

diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json b/src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json
new file mode 100644
index 000000000..60dd045da
--- /dev/null
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json
@@ -0,0 +1,239 @@
+{
+  "dashboard": {
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "gnetId": null,
+    "graphTooltip": 0,
+    "hideControls": false,
+    "id": null,
+    "links": [],
+    "refresh": "30s",
+    "rows": [
+      {
+        "collapse": false,
+        "height": "250px",
+        "panels": [
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 1,
+            "legend": {
+              "avg": true,
+              "current": true,
+              "max": true,
+              "min": true,
+              "show": true,
+              "total": false,
+              "values": true
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "connected",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "(sum(k8s_node_gpu_total) - sum(k8s_node_gpu_available) - sum(k8s_node_gpu_reserved)) / sum(k8s_node_gpu_total) * 100",
+                "format": "time_series",
+                "instant": false,
+                "intervalFactor": 2,
+                "legendFormat": "Allocation Rate",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Cluster wide GPU allocation rate",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "percent",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": false
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      },
+      {
+        "collapse": false,
+        "height": 250,
+        "panels": [
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 2,
+            "legend": {
+              "avg": true,
+              "current": true,
+              "max": true,
+              "min": true,
+              "show": true,
+              "total": false,
+              "values": true
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "connected",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "avg(task_gpu_percent)",
+                "format": "time_series",
+                "intervalFactor": 2,
+                "legendFormat": "Avg Util",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Cluster wide avg util",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "percent",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": false
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      }
+    ],
+    "schemaVersion": 14,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now/w",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ]
+    },
+    "timezone": "browser",
+    "title": "Cluster GPU statistic",
+    "version": 0
+  }
+}

From a7341acb3379a38c8ed58281800a793a788f2849 Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Thu, 11 Jul 2019 21:19:20 -0700
Subject: [PATCH 056/194] fix issue of not fetching through restfulapi without
 login

---
 src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index eadc37f87..139b3e26a 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -210,6 +210,7 @@ await priorEntrys.ForEachAsync(userEntry =>
         [HttpGet("{op}")]
         public async Task<ActionResult> Get(string op)
         {
+            var tuple = await processRestfulAPICommon();
             if (!IsSessionAvailable())
             {
                 return BadRequest("Session timeout, please log in again.");
@@ -217,7 +218,6 @@ public async Task<ActionResult> Get(string op)
 
             var ret = "invalid API call!";
             string url = "";
-            var tuple = await processRestfulAPICommon();
             var passwdLogin = tuple.Item1;
             if (!String.IsNullOrEmpty(tuple.Item2))
                 return BadRequest(tuple.Item2);

From d9807168130b7830fcbc131b140d940d2969cd86 Mon Sep 17 00:00:00 2001
From: Deepak Bansal <deepakba@microsoft.com>
Date: Sun, 14 Jul 2019 06:50:46 -0700
Subject: [PATCH 057/194] consider used gpus while calculating reserved on
 unschedulable nodes

---
 src/ClusterManager/job_manager.py  | 4 ++--
 src/ClusterManager/node_manager.py | 5 +++--
 src/utils/JobRestAPIUtils.py       | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 616922800..cc53f44e4 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -861,7 +861,7 @@ def TakeJobActions(jobs):
     dataHandler.Close()
 
     globalTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
-    globalReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
+    globalReservedRes = ResourceInfo(clusterStatus["gpu_reserved"])
 
     localResInfo = ResourceInfo()
     globalResInfo = ResourceInfo.Difference(globalTotalRes, globalReservedRes)
@@ -869,7 +869,7 @@ def TakeJobActions(jobs):
     for vc in vcList:
         vcTotalRes = ResourceInfo(json.loads(vc["quota"]), vc["vcName"])
         clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"], vc["vcName"])
-        clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"], vc["vcName"])
+        clusterReservedRes = ResourceInfo(clusterStatus["gpu_reserved"], vc["vcName"])
         vcReservedRes = clusterReservedRes.GetFraction(vcTotalRes, clusterTotalRes)
         localResInfo.Add(ResourceInfo.Difference(vcTotalRes, vcReservedRes))
 
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index 326a6a2ba..35a4d8271 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -139,7 +139,7 @@ def get_cluster_status():
                     node_status["unschedulable"] = False
 
                 if "status" in node and "conditions" in node["status"]:
-                    for condi in node["status"]:
+                    for condi in node["status"]["conditions"]:
                         if "type" in condi and condi["type"] == "Ready" and "status" in condi and condi["status"] == "Unknown":
                             node_status["unschedulable"] = True
 
@@ -203,12 +203,13 @@ def get_cluster_status():
         for node_name, node_status in nodes_status.iteritems():
             if node_status["unschedulable"]:
                 gpu_unschedulable.Add(ResourceInfo(node_status["gpu_capacity"]))
+                gpu_reserved.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_used"])))
             else:
                 gpu_avaliable.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_allocatable"]), ResourceInfo(node_status["gpu_used"])))
                 gpu_schedulable.Add(ResourceInfo(node_status["gpu_capacity"]))
                 gpu_unschedulable.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_allocatable"])))
+                gpu_reserved.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_allocatable"])))
 
-            gpu_reserved.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_allocatable"])))
             gpu_used.Add(ResourceInfo(node_status["gpu_used"]))
             gpu_capacity.Add(ResourceInfo(node_status["gpu_capacity"]))
 
diff --git a/src/utils/JobRestAPIUtils.py b/src/utils/JobRestAPIUtils.py
index 3f20d58b0..c634a2847 100755
--- a/src/utils/JobRestAPIUtils.py
+++ b/src/utils/JobRestAPIUtils.py
@@ -498,7 +498,7 @@ def GetVC(userName, vcName):
 
     clusterStatus, dummy = DataManager.GetClusterStatus()
     clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
-    clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
+    clusterReservedRes = ResourceInfo(clusterStatus["gpu_reserved"])
 
     user_status = {}
 

From 35193a8e52e62f5c9e67c1884f2763f40f56ae29 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 15 Jul 2019 06:06:09 +0000
Subject: [PATCH 058/194] unify log usage

---
 src/ClusterManager/command_manager.py  |  13 +-
 src/ClusterManager/endpoint_manager.py |  39 ++---
 src/ClusterManager/job_manager.py      |  23 ++-
 src/ClusterManager/joblog_manager.py   |  11 +-
 src/ClusterManager/node_manager.py     |   7 +-
 src/ClusterManager/user_manager.py     |   2 +-
 src/RestAPI/dlwsrestapi.py             |  29 ++--
 src/utils/JobRestAPIUtils.py           |   8 +-
 src/utils/MyLogger.py                  |  52 -------
 src/utils/MySQLDataHandler.py          | 138 +++++++++---------
 src/utils/SQLDataHandler.py            |   4 +-
 src/utils/authorization.py             |   4 +-
 src/utils/cache.py                     | 191 +++++++++++++------------
 src/utils/k8sUtils.py                  |  16 ++-
 14 files changed, 246 insertions(+), 291 deletions(-)
 delete mode 100755 src/utils/MyLogger.py

diff --git a/src/ClusterManager/command_manager.py b/src/ClusterManager/command_manager.py
index 6038c86c3..3caad1db3 100755
--- a/src/ClusterManager/command_manager.py
+++ b/src/ClusterManager/command_manager.py
@@ -8,7 +8,6 @@
 import datetime
 import copy
 
-
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../storage"))
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../utils"))
 
@@ -32,8 +31,8 @@
 import random
 
 import logging
-import logging.config
 
+logger = logging.getLogger(__name__)
 
 def RunCommand(command):
     dataHandler = DataHandler()
@@ -50,13 +49,15 @@ def Run():
             pendingCommands = dataHandler.GetPendingCommands()
             for command in pendingCommands:
                 try:
-                    print "Processing command: %s" % (command["id"])
+                    logger.info("Processing command: %s", command["id"])
                     RunCommand(command)
                 except Exception as e:
-                    print e
+                    logger.exception("run command failed")
         except Exception as e:
-            print e
+            logger.exception("getting command failed")
         time.sleep(1)
 
 if __name__ == '__main__':
-    Run()
\ No newline at end of file
+    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
+            level=logging.INFO)
+    Run()
diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 272b3bf69..9628b4f65 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -11,9 +11,12 @@
 import traceback
 import random
 import re
+import logging
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
 
+logger = logging.getLogger(__name__)
+
 
 def is_ssh_server_ready(pod_name):
     bash_script = "sudo service ssh status"
@@ -76,14 +79,14 @@ def generate_node_port_service(job_id, pod_name, endpoint_id, name, target_port)
     targetPort: {4}
     port: {4}
 """.format(job_id, pod_name, endpoint_id, name, target_port)
-    print("endpointDescription: %s" % endpoint_description)
+    logger.info("endpointDescription: %s", endpoint_description)
     return endpoint_description
 
 
 def create_node_port(endpoint):
     endpoint_description = generate_node_port_service(endpoint["jobId"], endpoint["podName"], endpoint["id"], endpoint["name"], endpoint["podPort"])
     endpoint_description_path = os.path.join(config["storage-mount-path"], endpoint["endpointDescriptionPath"])
-    print("endpointDescriptionPath: %s" % endpoint_description_path)
+    logger.info("endpointDescriptionPath: %s", endpoint_description_path)
     with open(endpoint_description_path, 'w') as f:
         f.write(endpoint_description)
 
@@ -91,18 +94,18 @@ def create_node_port(endpoint):
     if result == "":
         raise Exception("Failed to create NodePort for ssh. JobId: %s " % endpoint["jobId"])
 
-    print("Submitted endpoint %s to k8s, returned with status %s" % (endpoint["jobId"], result))
+    logger.info("Submitted endpoint %s to k8s, returned with status %s", endpoint["jobId"], result)
 
 
 def setup_ssh_server(user_name, pod_name, host_network=False):
     '''Setup ssh server on pod and return the port'''
     # setup ssh server only is the ssh server is not up
     if not is_ssh_server_ready(pod_name):
-        print("Ssh server is not ready for pod: %s. Setup ..." % pod_name)
+        logger.info("Ssh server is not ready for pod: %s. Setup ...", pod_name)
         ssh_port = start_ssh_server(pod_name, user_name, host_network)
     else:
         ssh_port = query_ssh_port(pod_name)
-    print("Ssh server is ready for pod: %s. Ssh listen on %s" % (pod_name, ssh_port))
+    logger.info("Ssh server is ready for pod: %s. Ssh listen on %s", pod_name, ssh_port)
     return ssh_port
 
 
@@ -127,7 +130,7 @@ def setup_tensorboard(user_name, pod_name):
 
 def start_endpoint(endpoint):
     # pending, running, stopped
-    print("Starting endpoint: %s" % (endpoint))
+    logger.info("Starting endpoint: %s", endpoint)
 
     # podName
     pod_name = endpoint["podName"]
@@ -158,8 +161,8 @@ def is_user_ready(pod_name):
 
 def start_endpoints():
     try:
+        data_handler = DataHandler()
         try:
-            data_handler = DataHandler()
             pending_endpoints = data_handler.GetPendingEndpoints()
 
             for endpoint_id, endpoint in pending_endpoints.items():
@@ -174,7 +177,7 @@ def start_endpoints():
                 endpoint_description_dir = re.search("(.*/)[^/\.]+.yaml", job["jobDescriptionPath"]).group(1)
                 endpoint["endpointDescriptionPath"] = os.path.join(endpoint_description_dir, endpoint_id + ".yaml")
 
-                print("\n\n\n\n\n\n----------------Begin to start endpoint %s" % endpoint["id"])
+                logger.info("\n\n\n\n\n\n----------------Begin to start endpoint %s", endpoint["id"])
                 output = get_k8s_endpoint(endpoint["endpointDescriptionPath"])
                 if(output != ""):
                     endpoint_description = json.loads(output)
@@ -189,9 +192,11 @@ def start_endpoints():
                 endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
                 data_handler.UpdateEndpoint(endpoint)
         except Exception as e:
-            traceback.print_exc()
+            logger.exception("start endpoint failed")
+        finally:
+            data_handler.Close()
     except Exception as e:
-        traceback.print_exc()
+        logger.exception("close data handler failed")
 
 
 def cleanup_endpoints():
@@ -200,33 +205,33 @@ def cleanup_endpoints():
         try:
             dead_endpoints = data_handler.GetDeadEndpoints()
             for endpoint_id, dead_endpoint in dead_endpoints.items():
-                print("\n\n\n\n\n\n----------------Begin to cleanup endpoint %s" % endpoint_id)
+                logger.info("\n\n\n\n\n\n----------------Begin to cleanup endpoint %s", endpoint_id)
                 endpoint_description_path = os.path.join(config["storage-mount-path"], dead_endpoint["endpointDescriptionPath"])
                 still_running = get_k8s_endpoint(endpoint_description_path)
                 # empty mean not existing
                 if still_running == "":
-                    print("Endpoint already gone %s" % endpoint_id)
+                    logger.info("Endpoint already gone %s", endpoint_id)
                     status = "stopped"
                 else:
                     output = k8sUtils.kubectl_delete(endpoint_description_path)
                     # 0 for success
                     if output == 0:
                         status = "stopped"
-                        print("Succeed cleanup endpoint %s" % endpoint_id)
+                        logger.info("Succeed cleanup endpoint %s", endpoint_id)
                     else:
                         # TODO will need to clean it up eventually
                         status = "unknown"
-                        print("Clean dead endpoint %s failed, endpoints: %s" % (endpoint_id, dead_endpoint))
+                        logger.info("Clean dead endpoint %s failed, endpoints: %s", endpoint_id, dead_endpoint)
 
                 dead_endpoint["status"] = status
                 dead_endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
                 data_handler.UpdateEndpoint(dead_endpoint)
         except Exception as e:
-            traceback.print_exc()
+            logger.exception("cleanup endpoint failed")
         finally:
             data_handler.Close()
     except Exception as e:
-        traceback.print_exc()
+        logger.exception("close data handler failed")
 
 
 def Run():
@@ -241,4 +246,6 @@ def Run():
 
 
 if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
+            level=logging.INFO)
     Run()
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 8c07ae989..b4b2dc955 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -42,9 +42,6 @@
 from job_deployer import JobDeployer
 
 
-def printlog(msg):
-    print("%s - %s" % (datetime.datetime.utcnow().strftime("%x %X"),msg))
-
 
 def SubmitJob(job):
     ret = {}
@@ -200,7 +197,7 @@ def UpdateJobStatus(job):
                 try:
                     launch_ps_dist_job(jobParams)
                 except Exception as e:
-                    print(e)
+                    logging.exception("launch ps distributed job failed")
             return
 
     jobPath,workPath,dataPath = GetStoragePath(jobParams["jobPath"],jobParams["workPath"],jobParams["dataPath"])
@@ -227,7 +224,7 @@ def UpdateJobStatus(job):
             dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","running")
 
     elif result.strip() == "Failed":
-        printlog("Job %s fails, cleaning..." % job["jobId"])
+        logging.warning("Job %s fails, cleaning...", job["jobId"])
         joblog_manager.extract_job_log(job["jobId"],logPath,jobParams["userId"])
         dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","failed")
         dataHandler.UpdateJobTextField(job["jobId"],"errorMsg",detail)
@@ -241,16 +238,16 @@ def UpdateJobStatus(job):
             del UnusualJobs[job["jobId"]]
             retries = dataHandler.AddandGetJobRetries(job["jobId"])
             if retries >= 5:
-                printlog("Job %s fails for more than 5 times, abort" % job["jobId"])
+                logging.warning("Job %s fails for more than 5 times, abort", job["jobId"])
                 dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","error")
                 dataHandler.UpdateJobTextField(job["jobId"],"errorMsg","cannot launch the job.")
                 if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
                     k8sUtils.kubectl_delete(jobDescriptionPath)
             else:
-                printlog("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d" % (job["jobId"] , retries))
+                logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"] , retries)
                 SubmitJob(job)
     elif result.strip() == "PendingHostPort":
-        printlog("Cannot find host ports for job :%s, re-launch the job with different host ports " % (job["jobId"]))
+        logging.warning("Cannot find host ports for job :%s, re-launch the job with different host ports ", job["jobId"])
 
         SubmitJob(job)
 
@@ -349,7 +346,7 @@ def launch_ps_dist_job(jobParams):
     for [idx, pod] in enumerate(pods["items"]):
         pod_name = pod["metadata"]["name"]
         bash_script = "cat > /home/" + user_name + "/.ssh/config <<EOF " + sshconfigstr + "\nEOF"
-        print("override ssh client config: %s" % bash_script)
+        logging.info("override ssh client config: %s", bash_script)
         k8sUtils.kubectl_exec("exec %s -- bash -c \'%s\' ; chown -R %s /home/%s/.ssh/config" % (pod_name, bash_script,user_name,user_name))
 
         # fix ~/.ssh/ folder permission
@@ -490,7 +487,7 @@ def Run():
             config["racks"] = k8sUtils.get_node_labels("rack")
             config["skus"] = k8sUtils.get_node_labels("sku")
         except Exception as e:
-            print(e)
+            logging.exception("get node labels failed")
 
         try:
             dataHandler = DataHandler()
@@ -514,15 +511,17 @@ def Run():
                     except Exception as e:
                         logging.info(e)
             except Exception as e:
-                print(str(e))
+                logging.exception("process pending job failed")
             finally:
                 dataHandler.Close()
         except Exception as e:
-            print(str(e))
+            logging.exception("close data handler failed")
 
         time.sleep(1)
 
 
 if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
+            level=logging.INFO)
     Run()
     #print k8sUtils.get_pod_events("d493d41c-45ea-4e85-8ca4-01c3533cd727")
diff --git a/src/ClusterManager/joblog_manager.py b/src/ClusterManager/joblog_manager.py
index b10630c1e..aacf15138 100755
--- a/src/ClusterManager/joblog_manager.py
+++ b/src/ClusterManager/joblog_manager.py
@@ -23,8 +23,6 @@
 
 from multiprocessing import Process, Manager
 
-
-
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../storage"))
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../utils"))
 
@@ -34,6 +32,7 @@
 from config import config, GetStoragePath
 from DataHandler import DataHandler
 
+logger = logging.getLogger(__name__)
 
 def create_log( logdir = '/var/log/dlworkspace' ):
     if not os.path.exists( logdir ):
@@ -109,7 +108,7 @@ def extract_job_log(jobId,logPath,userId):
                     f.close()
                     os.system("chown -R %s %s" % (userId, containerLogPath))
                 except Exception as e:
-                    print e
+                    logger.exception("write container log failed")
 
 
         if len(trimlogstr.strip()) > 0:
@@ -156,8 +155,10 @@ def Run():
         try:
             update_job_logs()
         except Exception as e:
-            print e
+            logger.exception("update job logs failed")
         time.sleep(1)
 
 if __name__ == '__main__':
-    Run()
\ No newline at end of file
+    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
+            level=logging.INFO)
+    Run()
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index 326a6a2ba..4ed8f81ca 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -224,7 +224,7 @@ def get_cluster_status():
         cluster_status["node_status"] = [node_status for node_name, node_status in nodes_status.iteritems()] 
 
     except Exception as e:
-        print(e)
+        logging.exception("get cluster status")
 
     dataHandler = DataHandler()
     cluster_status["AvaliableJobNum"] = dataHandler.GetActiveJobsCount()
@@ -248,9 +248,10 @@ def Run():
         try:
             get_cluster_status()
         except Exception as e:
-            print e
-            logging.info(str(e))
+            logging.exception("get cluster status failed")
         time.sleep(30)
 
 if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
+            level=logging.INFO)
     Run()
diff --git a/src/ClusterManager/user_manager.py b/src/ClusterManager/user_manager.py
index 7b3418ff8..5bcd63140 100755
--- a/src/ClusterManager/user_manager.py
+++ b/src/ClusterManager/user_manager.py
@@ -86,7 +86,7 @@ def Run():
         try:
             set_user_directory()
         except Exception as e:
-            print e
+            logging.exception("set user directory failed")
         time.sleep(1)
 
 
diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index e94653dae..df34e6600 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -33,13 +33,12 @@
     logging_config = yaml.load(f)
     dictConfig(logging_config)
 logger = logging.getLogger('restfulapi')
-global_vars["logger"] = logger
 
 app = Flask(__name__)
 api = Api(app)
 verbose = True
 logger.info( "------------------- Restful API started ------------------------------------- ")
-logger.info("%s" % config )
+logger.info("%s", config)
 
 if "initAdminAccess" not in global_vars or not global_vars["initAdminAccess"]:
     logger.info("===========Init Admin Access===============")
@@ -255,9 +254,9 @@ def get(self):
                                     if oneshare==alias:
                                         addcmd += "chown %s:%s %s ; " % ( params["userId"], "500000513", containerPath )
             if verbose and len(params["mountpoints"]) > 0:
-                logger.info("Mount path for job %s" % params )
+                logger.info("Mount path for job %s", params )
                 for mounts in params["mountpoints"]:
-                    logger.info( "Share %s, mount %s at %s" % (mounts["name"], mounts["hostPath"], mounts["containerPath"]) )
+                    logger.info( "Share %s, mount %s at %s", mounts["name"], mounts["hostPath"], mounts["containerPath"])
             if len(addcmd) > 0:
                 params["cmd"] = addcmd + params["cmd"]
             output = JobRestAPIUtils.SubmitJob(json.dumps(params))
@@ -285,8 +284,8 @@ class PostJob(Resource):
     def post(self):
         params = request.get_json(force=True)
         monitor = yaml.safe_dump(params, default_flow_style=False)
-        logger.info("Post Job" )
-        logger.info(monitor )
+        logger.info("Post Job")
+        logger.info(monitor)
         ret = {}
         if True:
             output = JobRestAPIUtils.SubmitJob(json.dumps(params))
@@ -298,7 +297,7 @@ def post(self):
                     ret["error"] = "Cannot create job!" + output["error"]
                 else:
                     ret["error"] = "Cannot create job!"
-            logger.info("Submit job through restapi, output is %s, ret is %s" %(output, ret) )
+            logger.info("Submit job through restapi, output is %s, ret is %s", output, ret)
         resp = jsonify(ret)
         resp.headers["Access-Control-Allow-Origin"] = "*"
         resp.headers["dataType"] = "json"
@@ -1095,9 +1094,9 @@ def endpoint_exist(endpoint_id):
                 endpoint_id = "e-" + pod_name + "-ssh"
 
                 if endpoint_exist(endpoint_id=endpoint_id):
-                    print("Endpoint {} exists. Skip.".format(endpoint_id))
+                    logger.info("Endpoint %s exists. Skip.", endpoint_id)
                     continue
-                print("Endpoint {} does not exist. Add.".format(endpoint_id))
+                logger.info("Endpoint %s does not exist. Add.", endpoint_id)
 
                 endpoint = {
                     "id": endpoint_id,
@@ -1123,7 +1122,7 @@ def endpoint_exist(endpoint_id):
             endpoint_id = "e-" + job_id + "-ipython"
 
             if not endpoint_exist(endpoint_id=endpoint_id):
-                print("Endpoint {} does not exist. Add.".format(endpoint_id))
+                logger.info("Endpoint %s does not exist. Add.", endpoint_id)
                 endpoint = {
                     "id": endpoint_id,
                     "jobId": job_id,
@@ -1135,7 +1134,7 @@ def endpoint_exist(endpoint_id):
                 }
                 endpoints[endpoint_id] = endpoint
             else:
-                print("Endpoint {} exists. Skip.".format(endpoint_id))
+                logger.info("Endpoint {} exists. Skip.", endpoint_id)
 
         # Only open tensorboard on the master
         if 'tensorboard' in requested_endpoints:
@@ -1150,7 +1149,7 @@ def endpoint_exist(endpoint_id):
             endpoint_id = "e-" + job_id + "-tensorboard"
 
             if not endpoint_exist(endpoint_id=endpoint_id):
-                print("Endpoint {} does not exist. Add.".format(endpoint_id))
+                logger.info("Endpoint %s does not exist. Add.", endpoint_id)
                 endpoint = {
                     "id": endpoint_id,
                     "jobId": job_id,
@@ -1162,7 +1161,7 @@ def endpoint_exist(endpoint_id):
                 }
                 endpoints[endpoint_id] = endpoint
             else:
-                print("Endpoint {} exists. Skip.".format(endpoint_id))
+                logger.info("Endpoint %s exists. Skip.", endpoint_id)
 
         # interactive port
         for interactive_port in interactive_ports:
@@ -1176,7 +1175,7 @@ def endpoint_exist(endpoint_id):
 
             endpoint_id = "e-" + job_id + "-" + interactive_port["name"]
             if not endpoint_exist(endpoint_id=endpoint_id):
-                print("Endpoint {} does not exist. Add.".format(endpoint_id))
+                logger.info("Endpoint %s does not exist. Add.", endpoint_id)
                 endpoint = {
                     "id": endpoint_id,
                     "jobId": job_id,
@@ -1189,7 +1188,7 @@ def endpoint_exist(endpoint_id):
                 }
                 endpoints[endpoint_id] = endpoint
             else:
-                print("Endpoint {} exists. Skip.".format(endpoint_id))
+                logger.info("Endpoint %s exists. Skip.", endpoint_id)
 
         data_handler = DataHandler()
         for [_, endpoint] in endpoints.items():
diff --git a/src/utils/JobRestAPIUtils.py b/src/utils/JobRestAPIUtils.py
index 3f20d58b0..14b84ad37 100755
--- a/src/utils/JobRestAPIUtils.py
+++ b/src/utils/JobRestAPIUtils.py
@@ -17,7 +17,6 @@
 import re
 
 from config import global_vars
-from MyLogger import MyLogger
 from authorization import ResourceType, Permission, AuthorizationManager, IdentityManager
 import authorization
 from cache import CacheManager
@@ -25,8 +24,9 @@
 from ResourceInfo import ResourceInfo
 
 import copy
+import logging
 
-logger = MyLogger()
+logger = logging.getLogger(__name__)
 
 def LoadJobParams(jobParamsJsonStr):
     return json.loads(jobParamsJsonStr)
@@ -237,8 +237,8 @@ def GetJobList(userName, vcName, jobOwner, num=None):
         dataHandler.Close()
         return jobs
     except Exception as e:
-        logger.error('Exception: '+ str(e))
-        logger.warn("Fail to get job list for user %s, return empty list" % userName)
+        logger.error('Exception: %s', str(e))
+        logger.warn("Fail to get job list for user %s, return empty list", userName)
         return []
 
 
diff --git a/src/utils/MyLogger.py b/src/utils/MyLogger.py
deleted file mode 100755
index ebcd48226..000000000
--- a/src/utils/MyLogger.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from config import global_vars
-import logging
-from logging.config import dictConfig
-import thread
-
-
-class MyLogger:
-
-    def init(self):
-        if self.logger is None and "logger" in global_vars and global_vars["logger"] is not None:
-            self.logger = global_vars["logger"]
-
-    def __init__(self):
-        self.logger = None
-        self.init()
-
-    def info(self, msg):
-        self.init()
-        txt = str(thread.get_ident()) + " : "  + msg 
-        #print txt
-        if self.logger is not None:
-            self.logger.info(txt)
-
-    def error(self, msg):
-        self.init()
-        txt = str(thread.get_ident()) + " : "  + msg 
-        #print msg
-
-        if self.logger is not None:
-            self.logger.error(txt)
-
-    def warn(self, msg):
-        self.init()
-        txt = str(thread.get_ident()) + " : "  + msg 
-        #print msg
-
-        if self.logger is not None:
-            self.logger.warn(txt)
-
-    def debug(self, msg):
-        self.init()
-        print msg
-
-        if self.logger is not None:
-            self.logger.debug(msg)
-
-    def exception(self, msg):
-        self.init()
-        print msg
-
-        if self.logger is not None:
-            self.logger.exception(msg)
diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index 03de3baa3..380138c79 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -3,6 +3,7 @@
 import json
 import base64
 import os
+import logging
 
 import timeit
 
@@ -10,9 +11,8 @@
 
 from config import config
 from config import global_vars
-from MyLogger import MyLogger
 
-logger = MyLogger()
+logger = logging.getLogger(__name__)
 
 
 class DataHandler:
@@ -41,7 +41,7 @@ def __init__(self):
 
 
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler initialization, time elapsed %f s" % elapsed)
+        logger.info("DataHandler initialization, time elapsed %f s", elapsed)
 
 
 
@@ -228,10 +228,10 @@ def AddStorage(self, vcName, url, storageType, metadata, defaultMountPath):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: AddStorage to DB: url : %s, vcName: %s , time elapsed %f s" % (url, vcName, elapsed))
+            logger.info("DataHandler: AddStorage to DB: url : %s, vcName: %s , time elapsed %f s", url, vcName, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -244,10 +244,10 @@ def DeleteStorage(self, vcName, url):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteStorage: url:%s, vcName:%s, time elapsed %f s" % (url, vcName, elapsed))
+            logger.info("DataHandler: DeleteStorage: url:%s, vcName:%s, time elapsed %f s", url, vcName, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -267,12 +267,12 @@ def ListStorages(self, vcName):
                 record["defaultMountPath"] = defaultMountPath
                 ret.append(record)
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             pass
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: ListStorages time elapsed %f s" % (elapsed))
+        logger.info("DataHandler: ListStorages time elapsed %f s", elapsed)
         return ret
 
 
@@ -285,10 +285,10 @@ def UpdateStorage(self, vcName, url, storageType, metadata, defaultMountPath):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateStorage: vcName: %s, url: %s, time elapsed %f s" % (vcName, url, elapsed))
+            logger.info("DataHandler: UpdateStorage: vcName: %s, url: %s, time elapsed %f s", vcName, url, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -301,10 +301,10 @@ def AddVC(self, vcName, quota, metadata):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: AddVC to DB: vcName: %s , time elapsed %f s" % (vcName, elapsed))
+            logger.info("DataHandler: AddVC to DB: vcName: %s , time elapsed %f s", vcName, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -322,12 +322,12 @@ def ListVCs(self):
                 record["metadata"] = metadata
                 ret.append(record)
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             pass
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: ListVCs time elapsed %f s" % (elapsed))
+        logger.info("DataHandler: ListVCs time elapsed %f s", elapsed)
         return ret
 
 
@@ -340,10 +340,10 @@ def DeleteVC(self, vcName):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteVC: vcName: %s , time elapsed %f s" % (vcName, elapsed))
+            logger.info("DataHandler: DeleteVC: vcName: %s , time elapsed %f s", vcName, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -356,10 +356,10 @@ def UpdateVC(self, vcName, quota, metadata):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateVC: vcName: %s , time elapsed %f s" % (vcName, elapsed))
+            logger.info("DataHandler: UpdateVC: vcName: %s , time elapsed %f s", vcName, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -378,12 +378,12 @@ def GetIdentityInfo(self, identityName):
                 record["groups"] = json.loads(groups)
                 ret.append(record)
         except Exception as e:
-            logger.error('GetIdentityInfo Exception: '+ str(e))
+            logger.error('GetIdentityInfo Exception: %s', str(e))
             pass
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetIdentityInfo time elapsed %f s" % (elapsed))
+        logger.info("DataHandler: GetIdentityInfo time elapsed %f s", elapsed)
         return ret
 
 
@@ -405,10 +405,10 @@ def UpdateIdentityInfo(self, identityName, uid, gid, groups):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateIdentityInfo %s to database , time elapsed %f s" % (identityName, elapsed))
+            logger.info("DataHandler: UpdateIdentityInfo %s to database , time elapsed %f s", identityName, elapsed)
             return True
         except Exception as e:
-            logger.error('UpdateIdentityInfo Exception: '+ str(e))
+            logger.error('UpdateIdentityInfo Exception: %s', str(e))
             return False
 
 
@@ -423,7 +423,7 @@ def GetAceCount(self, identityName, resource):
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetAceCount time elapsed %f s" % ( elapsed))
+        logger.info("DataHandler: GetAceCount time elapsed %f s", elapsed)
         return ret
 
 
@@ -444,10 +444,10 @@ def UpdateAce(self, identityName, identityId, resource, permissions, isDeny):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateAce %s - %s to database , time elapsed %f s" % (identityName, resource, elapsed))
+            logger.info("DataHandler: UpdateAce %s - %s to database , time elapsed %f s", identityName, resource, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -461,10 +461,10 @@ def UpdateAclIdentityId(self, identityName, identityId):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateAclIdentityId %s - %s to database , time elapsed %f s" % (identityName, identityId, elapsed))
+            logger.info("DataHandler: UpdateAclIdentityId %s - %s to database , time elapsed %f s", identityName, identityId, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -479,10 +479,10 @@ def DeleteResourceAcl(self, resource):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteResourceAcl %s, time elapsed %f s" % (resource, elapsed))
+            logger.info("DataHandler: DeleteResourceAcl %s, time elapsed %f s", resource, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -497,10 +497,10 @@ def DeleteAce(self, identityName, resource):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteAce %s : %s time elapsed %f s" % (resource, identityName, elapsed))
+            logger.info("DataHandler: DeleteAce %s : %s time elapsed %f s", resource, identityName, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
 
@@ -520,12 +520,12 @@ def GetAcl(self):
                 record["isDeny"] = isDeny
                 ret.append(record)
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             pass
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetAcl time elapsed %f s" % ( elapsed))
+        logger.info("DataHandler: GetAcl time elapsed %f s", elapsed)
         return ret
 
 
@@ -545,12 +545,11 @@ def GetResourceAcl(self, resource):
                 record["isDeny"] = isDeny
                 ret.append(record)
         except Exception as e:
-            logger.error('Exception: '+ str(e))
-            pass
+            logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetResourceAcl time elapsed %f s" % ( elapsed))
+        logger.info("DataHandler: GetResourceAcl time elapsed %f s", elapsed)
         return ret
 
 
@@ -564,11 +563,11 @@ def AddJob(self, jobParams):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: added job %s to database, time elapsed %f s" % (jobParams["jobId"], elapsed))
+            logger.info("DataHandler: added job %s to database, time elapsed %f s", jobParams["jobId"], elapsed)
             return True
         except Exception as e:
-           logger.error('Exception: '+ str(e))
-           return False
+            logger.error('Exception: %s', str(e))
+            return False
 
 
     def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or")):
@@ -598,7 +597,7 @@ def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or"
             start_time2 = timeit.default_timer()
             data = cursor.fetchall()
             elapsed2 = timeit.default_timer() - start_time2
-            logger.info ("(fetchall time: %f)" % (elapsed2))
+            logger.info("(fetchall time: %f)", elapsed2)
             for (jobId,jobName,userName, vcName, jobStatus,jobStatusDetail, jobType, jobDescriptionPath, jobDescription, jobTime, endpoints, jobParams,errorMsg, jobMeta) in data:
                 record = {}
                 record["jobId"] = jobId
@@ -617,12 +616,11 @@ def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or"
                 record["jobMeta"] = jobMeta
                 ret.append(record)
         except Exception as e:
-            logger.error('Exception: '+ str(e))
-            pass
+            logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get job list for user %s , time elapsed %f s (SQL query time: %f)" % (userName, elapsed, elapsed1))
+        logger.info("DataHandler: get job list for user %s , time elapsed %f s (SQL query time: %f)", userName, elapsed, elapsed1)
         return ret
 
     def GetJob(self, **kwargs):
@@ -641,7 +639,7 @@ def GetJob(self, **kwargs):
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get job details with query %s=%s , time elapsed %f s" % (key, expected, elapsed))
+        logger.info("DataHandler: get job details with query %s=%s , time elapsed %f s", key, expected, elapsed)
         return ret
 
     def AddCommand(self, jobId, command):
@@ -653,10 +651,10 @@ def AddCommand(self, jobId, command):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: add command to database, jobId: %s , time elapsed %f s" % (jobId, elapsed))
+            logger.info("DataHandler: add command to database, jobId: %s , time elapsed %f s", jobId, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
     def GetPendingCommands(self):
@@ -674,7 +672,7 @@ def GetPendingCommands(self):
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get pending command , time elapsed %f s" % (elapsed))
+        logger.info("DataHandler: get pending command , time elapsed %f s", elapsed)
         return ret
 
     def FinishCommand(self, commandId):
@@ -686,10 +684,10 @@ def FinishCommand(self, commandId):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: set command %s as finished , time elapsed %f s" % (commandId, elapsed))
+            logger.info("DataHandler: set command %s as finished , time elapsed %f s", commandId, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
     def GetCommands(self, jobId):
@@ -708,7 +706,7 @@ def GetCommands(self, jobId):
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get command list for job %s , time elapsed %f s" % (jobId, elapsed))
+        logger.info("DataHandler: get command list for job %s , time elapsed %f s", jobId, elapsed)
         return ret
 
     def load_json(self, raw_str):
@@ -731,7 +729,7 @@ def GetPendingEndpoints(self):
             pendingEndpoints = {k: v for d in endpoints for k, v in d.items() if v["status"] == "pending"}
 
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: get pending endpoints %d, time elapsed %f s" % (len(pendingEndpoints), elapsed))
+            logger.info("DataHandler: get pending endpoints %d, time elapsed %f s", len(pendingEndpoints), elapsed)
             return pendingEndpoints
         except Exception as e:
             logger.exception("Query pending endpoints failed!")
@@ -751,11 +749,9 @@ def GetDeadEndpoints(self):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: get dead endpoints %d , time elapsed %f s" % (len(dead_endpoints), elapsed))
+            logger.info("DataHandler: get dead endpoints %d , time elapsed %f s", len(dead_endpoints), elapsed)
             return dead_endpoints
         except Exception as e:
-            import traceback
-            traceback.print_exc()
             logger.exception("Query dead endpoints failed!")
             return {}
 
@@ -775,7 +771,7 @@ def UpdateEndpoint(self, endpoint):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: update endpoints to database, endpointId: %s , time elapsed %f s" % (endpoint["id"], elapsed))
+            logger.info("DataHandler: update endpoints to database, endpointId: %s , time elapsed %f s", endpoint["id"], elapsed)
             return True
         except Exception as e:
             logger.exception("Update endpoints failed!")
@@ -806,7 +802,7 @@ def GetPendingJobs(self):
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get pending jobs %d, time elapsed %f s" % (len(ret), elapsed))
+        logger.info("DataHandler: get pending jobs %d, time elapsed %f s", len(ret), elapsed)
         return ret
 
 
@@ -820,10 +816,10 @@ def SetJobError(self, jobId, errorMsg):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: set job %s error status in database, time elapsed %f s" % (jobId, elapsed))
+            logger.info("DataHandler: set job %s error status in database, time elapsed %f s", jobId, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
     def UpdateJobTextField(self, jobId, field, value):
@@ -835,10 +831,10 @@ def UpdateJobTextField(self, jobId, field, value):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: update job %s, field %s to %s, time elapsed %f s" % (jobId, field, value, elapsed))
+            logger.info("DataHandler: update job %s, field %s to %s, time elapsed %f s", jobId, field, value, elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
     def GetJobTextField(self, jobId, field):
@@ -851,12 +847,12 @@ def GetJobTextField(self, jobId, field):
             for (jobId, value) in cursor:
                 ret = value
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             pass
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get filed %s of job %s , time elapsed %f s" % (field, jobId, elapsed))
+        logger.info("DataHandler: get filed %s of job %s , time elapsed %f s", field, jobId, elapsed)
         return ret
 
     def AddandGetJobRetries(self, jobId):
@@ -877,7 +873,7 @@ def AddandGetJobRetries(self, jobId):
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get and update retries for job %s , time elapsed %f s" % (jobId, elapsed))
+        logger.info("DataHandler: get and update retries for job %s , time elapsed %f s", jobId, elapsed)
         return ret
 
     def UpdateClusterStatus(self, clusterStatus):
@@ -891,10 +887,10 @@ def UpdateClusterStatus(self, clusterStatus):
             self.conn.commit()
             cursor.close()
             elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: update cluster status, time elapsed %f s" % (elapsed))
+            logger.info("DataHandler: update cluster status, time elapsed %f s", elapsed)
             return True
         except Exception as e:
-            logger.error('Exception: '+ str(e))
+            logger.error('Exception: %s', str(e))
             return False
 
     def GetClusterStatus(self):
@@ -909,12 +905,11 @@ def GetClusterStatus(self):
                 ret = json.loads(base64.b64decode(value))
                 time = t
         except Exception as e:
-            logger.error('Exception: '+ str(e))
-            pass
+            logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get cluster status , time elapsed %f s" % (elapsed))
+        logger.info("DataHandler: get cluster status , time elapsed %f s", elapsed)
         return ret, time
 
 
@@ -928,12 +923,11 @@ def GetUsers(self):
             for (identityName,uid) in cursor:
                 ret.append((identityName,uid))
         except Exception as e:
-            logger.error('Exception: '+ str(e))
-            pass
+            logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
         elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get users, time elapsed %f s" % (elapsed))
+        logger.info("DataHandler: get users, time elapsed %f s", elapsed)
         return ret
 
     def GetActiveJobsCount(self):
diff --git a/src/utils/SQLDataHandler.py b/src/utils/SQLDataHandler.py
index ca7e31581..d60c806d2 100755
--- a/src/utils/SQLDataHandler.py
+++ b/src/utils/SQLDataHandler.py
@@ -3,6 +3,7 @@
 import json
 import base64
 import os
+import logging
 
 import timeit
 
@@ -11,9 +12,8 @@
 
 from config import config
 from config import global_vars
-from MyLogger import MyLogger
 
-logger = MyLogger()
+logger = logging.getLogger(__name__)
 
 ### set to a larger number if flask is running on multithreading
 sql_max_connect_num = 35
diff --git a/src/utils/authorization.py b/src/utils/authorization.py
index 112f45c4e..4145102fa 100755
--- a/src/utils/authorization.py
+++ b/src/utils/authorization.py
@@ -1,5 +1,5 @@
 from DataHandler import DataHandler, DataManager
-from MyLogger import MyLogger
+import logging
 import json
 import requests
 import random
@@ -7,7 +7,7 @@
 import timeit
 from cache import fcache
 
-logger = MyLogger()
+logger = logging.getLogger(__name__)
 
 def enum(*sequential, **named):
     enums = dict(zip(sequential, range(len(sequential))), **named)
diff --git a/src/utils/cache.py b/src/utils/cache.py
index d3f51b538..a3a174447 100644
--- a/src/utils/cache.py
+++ b/src/utils/cache.py
@@ -1,94 +1,97 @@
-from functools import wraps
-import threading
-from datetime import datetime
-from datetime import timedelta
-import time
-import Queue
-import copy
-
-# decorator (with different TTL for each function)
-# No removal of entries (designed for small number of entries to be always kept in memory)
-# option to invalidate specific entries
-# non-blocking (updates done by background thread; stale data returned while fetching in progress) : avoids thundering herd for data source
-def fcache(TTLInSec=30):
-    def fcache_decorator(func):
-        @wraps(func)
-        def wrapped_function(*args, **kwargs):
-            val = CacheManager.GetValue(func, TTLInSec, args)
-            if (None == val):               
-                return func(*args)
-            return copy.deepcopy(val[0])           
-        return wrapped_function
-    return fcache_decorator
-
-
-
-class CacheManager(object):
-    data = {}
-    taskQueue = Queue.Queue()
-    pendingTasks = set()
-
-    @staticmethod
-    def Invalidate(funcName, *args):
-        key = CacheManager._GetKey(funcName, args)
-        if key in CacheManager.data:
-            val = CacheManager.data[key]
-            CacheManager.data[key] = [val[0], datetime.now()]
-            print("Cache invalidated " + key)
-
-    @staticmethod
-    def GetValue(func, ttl, args):
-        val = None
-        key = CacheManager._GetKey(func.__name__, args)
-        needUpdate = False
-        if key not in CacheManager.data:
-            print("Cache miss " + key)
-            needUpdate = True
-        else:           
-            val = CacheManager.data[key]
-            print("Cache hit " + key + " " + str(val[1]) + " " + str(len(CacheManager.data)) + " " + str(CacheManager.taskQueue.qsize()))
-            if CacheManager._Invalid(val):
-                needUpdate = True
-
-        if needUpdate and key not in CacheManager.pendingTasks:
-            CacheManager.taskQueue.put((func, ttl, args))
-            CacheManager.pendingTasks.add(key)
-        
-        return val
-
-    
-    @staticmethod
-    def _GetKey(funcName, args):
-        key = funcName
-        for arg in args:
-            key += "__"
-            key += str(arg)
-        return key
-
-    @staticmethod
-    def _Invalid(value):
-        if value[1] < datetime.now():
-            return True
-        return False
-
-    @staticmethod
-    def _WorkerThreadFunc():
-        while (True):
-            try:
-                while not CacheManager.taskQueue.empty():
-                    task = CacheManager.taskQueue.get()
-                    key = CacheManager._GetKey(task[0].__name__, task[2])
-                    if key in CacheManager.pendingTasks:
-                        if key not in CacheManager.data or CacheManager._Invalid(CacheManager.data[key]):
-                            result = task[0](*(task[2]))
-                            CacheManager.data[key] = [result, datetime.now() + timedelta(seconds=int(task[1]))]
-                            print("Cache inserted " + key)                    
-                        CacheManager.pendingTasks.remove(key)
-                time.sleep(0.001)
-            except Exception as e:
-                print('cache exception: '+ str(e))
-
-workerThread = threading.Thread(target=CacheManager._WorkerThreadFunc, args=())
-workerThread.daemon = True
-workerThread.start()
-
+from functools import wraps
+import threading
+from datetime import datetime
+from datetime import timedelta
+import time
+import Queue
+import copy
+import logging
+
+logger = logging.getLogger(__name__)
+
+# decorator (with different TTL for each function)
+# No removal of entries (designed for small number of entries to be always kept in memory)
+# option to invalidate specific entries
+# non-blocking (updates done by background thread; stale data returned while fetching in progress) : avoids thundering herd for data source
+def fcache(TTLInSec=30):
+    def fcache_decorator(func):
+        @wraps(func)
+        def wrapped_function(*args, **kwargs):
+            val = CacheManager.GetValue(func, TTLInSec, args)
+            if (None == val):
+                return func(*args)
+            return copy.deepcopy(val[0])
+        return wrapped_function
+    return fcache_decorator
+
+
+
+class CacheManager(object):
+    data = {}
+    taskQueue = Queue.Queue()
+    pendingTasks = set()
+
+    @staticmethod
+    def Invalidate(funcName, *args):
+        key = CacheManager._GetKey(funcName, args)
+        if key in CacheManager.data:
+            val = CacheManager.data[key]
+            CacheManager.data[key] = [val[0], datetime.now()]
+            logger.info("Cache invalidated %s", key)
+
+    @staticmethod
+    def GetValue(func, ttl, args):
+        val = None
+        key = CacheManager._GetKey(func.__name__, args)
+        needUpdate = False
+        if key not in CacheManager.data:
+            logger.info("Cache miss %s", key)
+            needUpdate = True
+        else:
+            val = CacheManager.data[key]
+            logger.info("Cache hit %s %s %s %s", key, str(val[1]), str(len(CacheManager.data)), str(CacheManager.taskQueue.qsize()))
+            if CacheManager._Invalid(val):
+                needUpdate = True
+
+        if needUpdate and key not in CacheManager.pendingTasks:
+            CacheManager.taskQueue.put((func, ttl, args))
+            CacheManager.pendingTasks.add(key)
+
+        return val
+
+
+    @staticmethod
+    def _GetKey(funcName, args):
+        key = funcName
+        for arg in args:
+            key += "__"
+            key += str(arg)
+        return key
+
+    @staticmethod
+    def _Invalid(value):
+        if value[1] < datetime.now():
+            return True
+        return False
+
+    @staticmethod
+    def _WorkerThreadFunc():
+        while (True):
+            try:
+                while not CacheManager.taskQueue.empty():
+                    task = CacheManager.taskQueue.get()
+                    key = CacheManager._GetKey(task[0].__name__, task[2])
+                    if key in CacheManager.pendingTasks:
+                        if key not in CacheManager.data or CacheManager._Invalid(CacheManager.data[key]):
+                            result = task[0](*(task[2]))
+                            CacheManager.data[key] = [result, datetime.now() + timedelta(seconds=int(task[1]))]
+                            logger.info("Cache inserted %s", key)
+                        CacheManager.pendingTasks.remove(key)
+                time.sleep(0.001)
+            except Exception as e:
+                logger.warning('cache exception: %s', str(e))
+
+workerThread = threading.Thread(target=CacheManager._WorkerThreadFunc, args=())
+workerThread.daemon = True
+workerThread.start()
+
diff --git a/src/utils/k8sUtils.py b/src/utils/k8sUtils.py
index 7f6943d65..51a205174 100755
--- a/src/utils/k8sUtils.py
+++ b/src/utils/k8sUtils.py
@@ -9,6 +9,7 @@
 from tzlocal import get_localzone
 import pytz
 
+import logging
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../storage"))
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../utils"))
@@ -28,6 +29,7 @@
 import pycurl
 from StringIO import StringIO
 
+logger = logging.getLogger(__name__)
 
 def curl_get(url):
     curl = pycurl.Curl()
@@ -53,7 +55,7 @@ def kubectl_create(jobfile, EXEC=True):
         try:
             output = subprocess32.check_output(["bash", "-c", config["kubelet-path"] + " create -f " + jobfile])
         except Exception as e:
-            print e
+            logger.exception("kubectl create")
             output = ""
     else:
         output = "Job " + jobfile + " is not submitted to kubernetes cluster"
@@ -64,10 +66,10 @@ def kubectl_delete(jobfile, EXEC=True):
     if EXEC:
         try:
             cmd = "bash -c '" + config["kubelet-path"] + " delete -f " + jobfile + "'"
-            print cmd
+            logger.info("executing %s", cmd)
             output = os.system(cmd)
         except Exception as e:
-            print e
+            logger.exception("kubectl delete")
             output = -1
     else:
         output = -1
@@ -83,7 +85,7 @@ def kubectl_exec(params, timeout=None):
         # TODO set the timeout
         output = subprocess32.check_output(["bash", "-c", config["kubelet-path"] + " " + params], timeout=timeout)
     except Exception as e:
-        print "EXCEPTION: " + str(e)
+        logger.exception("kubectl exec")
         output = ""
     return output
 
@@ -166,7 +168,7 @@ def GetPod(selector):
         output = kubectl_exec(" get pod -o yaml --show-all -l " + selector)
         podInfo = yaml.load(output)
     except Exception as e:
-        print e
+        logger.exception("kubectl get pod")
         podInfo = None
     return podInfo
 
@@ -357,8 +359,8 @@ def GetJobStatus(jobId):
 
 def all_pod_ready(job_id):
     pods = GetPod("run=" + job_id)
-    print("\n\n\n--------------------------------------------\n\n")
-    print("=======%s" % pods)
+    logger.info("\n\n\n--------------------------------------------\n\n")
+    logger.info("=======%s", pods)
     if pods is None:
         return False
     if "items" in pods:

From 91e17e6ca4ea73400a660922dae43ce2c4d48d68 Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Tue, 16 Jul 2019 16:36:35 -0700
Subject: [PATCH 059/194] Disable  distrubed job when using low priority
 cluster job in job type when using low priority cluster (#418)

---
 .../WebPortal/Views/Home/JobSubmission.cshtml | 29 ++++++++++++++++---
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 788331247..4e0a37309 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -256,6 +256,17 @@
                         gpu_available: gpu_available[key] < 0 ? 0 : gpu_available[key],
                         quota: quota[key]
                     };
+                    $scope.isLowPriority = value.low_priority === true ? true : false;
+                    if ($scope.isLowPriority) {
+                        var filteredJobList = [];
+                        $scope.joblist.forEach(function(job) {
+                            if (JSON.parse(job.Json).jobtrainingtype !== "PSDistJob") {
+                                filteredJobList.push(job);
+                            }
+                        });
+                        $scope.joblist = filteredJobList;
+                    }
+
                     $scope.gpus[key] = gpu;
                 });
                 $scope.checkExtras();
@@ -606,6 +617,9 @@
             if (!$scope.current.jobName) return false;
             if (!$scope.current.image) return false;
             if (!$scope.current.cmd || !String($scope.current.cmd).trim()) return false;
+            if ($scope.isLowPriority && $scope.current.jobtrainingtype == 'PSDistJob') {
+                return false;
+            }
             if (
                 $scope.current.jobtrainingtype === 'RegularJob' &&
                 typeof $scope.current.resourcegpu !== 'number'
@@ -681,10 +695,17 @@
                             </md-select>
                         </md-input-container>
                         <md-input-container flex="50">
-                            <label>Job Type</label>
-                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 ? false : true" required md-no-asterisk>
-                                <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob">Distributed Job</md-option>
+                            <label style="pointer-events:auto">Job Type  
+                                <i ng-hide="!isLowPriority"
+                                   class="layui-icon layui-icon-tips" 
+                                   data-toggle="tooltip" 
+                                   title="Low priority cluster does not support distributed jobs" 
+                                   data-placement="right"
+                                 ></i>
+                            </label>
+                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority ? false : true"  required md-no-asterisk>
+                                <md-option value="RegularJob" ng-selected="isLowPriority">Regular Job</md-option>
+                                <md-option value="PSDistJob" ng-disabled="isLowPriority ? true : false" ng-hide="isLowPriority ? true : false">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From e2a3c1a7b22fa276278027736a90d9fe5e5db72b Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Tue, 16 Jul 2019 23:17:51 -0700
Subject: [PATCH 060/194] Fix issue of changing job template based on low
 priority

---
 .../WebPortal/Views/Home/JobSubmission.cshtml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 4e0a37309..12b937411 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -193,7 +193,7 @@
         $scope.lastTemplateValue = -1;
         $scope.adancedOption = false;
 
-        $scope.$watch('cluster', function (cluster) {
+        $scope.$watch('cluster', function (cluster,oldValue) {
             $http.get('/api/dlws/GetMountPoints', { params: { cluster: cluster } }).then(function (response) {
                 var mpstring = response.data.mountpoints;
                 var mpdescription = response.data.mountdescription;
@@ -272,13 +272,22 @@
                 $scope.checkExtras();
             })
             $scope.extras.gpuType = null;
+            if (cluster !== oldValue) {
+                $scope.current.jobName = "";
+                $scope.current.resourcegpu = 0;
+                $scope.current.image = "";
+                $scope.current.cmd = "";
+                $scope.current.jobtrainingtype = "RegularJob";
+
+            }
         });
 
-        $scope.$watch('current.jobtrainingtype', function (value) {
+        $scope.$watch('current.jobtrainingtype', function (value, oldValue) {
             if (value === 'PSDistJob') {
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
             } else {
+                $scope.changed = true;
                 delete $scope.current.numps;
             }
         })
@@ -703,9 +712,9 @@
                                    data-placement="right"
                                  ></i>
                             </label>
-                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority ? false : true"  required md-no-asterisk>
-                                <md-option value="RegularJob" ng-selected="isLowPriority">Regular Job</md-option>
-                                <md-option value="PSDistJob" ng-disabled="isLowPriority ? true : false" ng-hide="isLowPriority ? true : false">Distributed Job</md-option>
+                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
+                                <md-option value="RegularJob" selected>Regular Job</md-option>
+                                <md-option value="PSDistJob">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 6548dbf032de0adc4d015de3fd73c5d9afc24bb2 Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Tue, 16 Jul 2019 23:19:28 -0700
Subject: [PATCH 061/194] Fix issue of changing job template based on low
 priority

---
 src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 12b937411..826343310 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -287,7 +287,6 @@
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
             } else {
-                $scope.changed = true;
                 delete $scope.current.numps;
             }
         })

From 531f7ba6096666173411ccbd9dfa138cf82b010f Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 9 Jul 2019 06:23:09 +0000
Subject: [PATCH 062/194] implement notify

---
 .../services/monitor/alert-manager.yaml       |  23 ++-
 .../monitor/alert-templates/job_state.tmpl    |  71 ++++++++
 src/utils/notify.py                           | 166 ++++++++++++++++++
 3 files changed, 258 insertions(+), 2 deletions(-)
 create mode 100644 src/ClusterBootstrap/services/monitor/alert-templates/job_state.tmpl
 create mode 100644 src/utils/notify.py

diff --git a/src/ClusterBootstrap/services/monitor/alert-manager.yaml b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
index 8bc493e6d..819756e14 100644
--- a/src/ClusterBootstrap/services/monitor/alert-manager.yaml
+++ b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
@@ -82,12 +82,17 @@ data:
       group_interval: 5m
       group_by: [alertname]
       routes:
-      - receiver: task_user
+      - receiver: idle_gpu_receiver
         repeat_interval: 4h
         group_by: [alertname, user_email, cluster]
         match_re:
           type: user_alert
           alertname: "zero-gpu-usage"
+      - receiver: job_state_change_receiver
+        group_by: [alertname, user_email, cluster, subject]
+        match_re:
+          type: user_alert
+          alertname: "job-state-changed"
     receivers:
     - name: "alert-email"
       email_configs:
@@ -95,7 +100,7 @@ data:
           html: '{{ "{{" }} template "email.html" . {{ "}}" }}'
           headers:
             subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
-    - name: "task_user"
+    - name: "idle_gpu_receiver"
       email_configs:
         {% if cnf["alert-manager"]["alert_users"] %}
         - to: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }},{{ alert_info["receiver"] }}'
@@ -109,4 +114,18 @@ data:
             CC: '{{ alert_info["receiver"] }}'
             {% endif %}
             subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
+    - name: "job_state_change_receiver"
+      email_configs:
+        {% if cnf["alert-manager"]["alert_users"] %}
+        - to: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }},{{ alert_info["receiver"] }}'
+        {% else %}
+        - to: '{{ alert_info["receiver"] }}'
+        {% endif %}
+          html: '{{ "{{" }} template "job_state.html" . {{ "}}" }}'
+          headers:
+            {% if cnf["alert-manager"]["alert_users"] %}
+            To: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }}'
+            CC: '{{ alert_info["receiver"] }}'
+            {% endif %}
+            subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
 {% endif %}
diff --git a/src/ClusterBootstrap/services/monitor/alert-templates/job_state.tmpl b/src/ClusterBootstrap/services/monitor/alert-templates/job_state.tmpl
new file mode 100644
index 000000000..2da286cc1
--- /dev/null
+++ b/src/ClusterBootstrap/services/monitor/alert-templates/job_state.tmpl
@@ -0,0 +1,71 @@
+{{ define "job_state.html" }}
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--
+Style and HTML derived from https://github.com/mailgun/transactional-email-templates
+
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Mailgun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+<head>
+<meta name="viewport" content="width=device-width"/>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+<title>{{ template "__subject" . }}</title>
+
+</head>
+
+<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">
+
+<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
+  <tr>
+    <td valign="top"></td>
+    <td width="600" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; width: 100% !important; margin: 0 auto; padding: 0;" valign="top">
+      <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
+        <table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
+          <tr>
+            <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
+              <table width="100%" cellpadding="0" cellspacing="0">
+                {{ range .Alerts.Firing }}
+                <tr>
+                  <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
+Your job
+<a href="http://dltshub.redmond.corp.microsoft.com/Home/JobDetail/?cluster={{.Labels.cluster}}&jobId={{.Labels.job_name}}">
+<strong>{{.Labels.job_name}}</strong>
+</a> from cluster '{{.Labels.cluster}}' has changed to state of {{.Labels.job_state}}.
+                  </td>
+                </tr>
+                {{ end }}
+              </table>
+            </td>
+          </tr>
+        </table>
+
+        </div>
+    </td>
+    <td valign="top"></td>
+  </tr>
+</table>
+
+</body>
+</html>
+{{ end }}
diff --git a/src/utils/notify.py b/src/utils/notify.py
new file mode 100644
index 000000000..67278ae98
--- /dev/null
+++ b/src/utils/notify.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+
+import threading
+import time
+import logging
+import urlparse
+import json
+import smtplib
+
+from Queue import Queue
+from Queue import Empty
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+class NotifyMsg(object):
+    def __init__(self, email, alert_name):
+        self.email = email
+        self.alert_name = alert_name
+
+    def labels(self):
+        raise NotImplementedError()
+
+    def subject(self):
+        raise NotImplementedError()
+
+    def body(self):
+        return self.subject()
+
+
+class JobStateChangedMsg(NotifyMsg):
+    def __init__(self, email, alert_name, job_name, job_state):
+        super(JobStateChangedMsg, self).__init__(email, alert_name)
+        self.job_name = job_name
+        self.job_state = job_state
+
+    def labels(self):
+        return {"job_name": self.job_name, "job_state": self.job_state}
+
+    def subject(self):
+        return "Your job %s has changed to state of %s" % (self.job_name, self.job_state)
+
+
+def new_job_state_change_message(email, job_name, state):
+    return JobStateChangedMsg(email, "job-state-changed", job_name, state)
+
+
+class Notifier(object):
+    def __init__(self, config):
+        self.queue = Queue()
+        self.running = False
+        self.thread = None
+
+        self.cluster = None
+        self.alert_manager_url = None
+        self.smtp_url = self.smtp_from = self.smtp_auth_name = self.smtp_auth_pass = None
+
+        if "notifier" in config:
+            notifier_config = config["notifier"]
+
+            self.cluster = notifier_config.get("cluster")
+            self.smtp_url = notifier_config.get("smtp-url")
+            self.smtp_from = notifier_config.get("smtp-from")
+            self.smtp_auth_name = notifier_config.get("smtp-auth-username")
+            self.smtp_auth_pass = notifier_config.get("smtp-auth-password")
+
+            alert_manager_url = notifier_config.get("alert-manager-url")
+            if alert_manager_url is not None and len(alert_manager_url) > 0:
+                if alert_manager_url[-1] == "/":
+                    self.alert_manager_url = alert_manager_url + "api/v1/alerts"
+                else:
+                    self.alert_manager_url = alert_manager_url + "/api/v1/alerts"
+
+        if self.cluster is None or \
+                self.alert_manager_url is None and (
+                        self.smtp_url is None or \
+                        self.smtp_from is None or \
+                        self.smtp_auth_name is None or \
+                        self.smtp_auth_pass is None):
+            logger.warning("Notifier not configured")
+
+    def start(self):
+        if not self.running:
+            self.running = True
+            self.thread = threading.Thread(target=self.process, name="notifier")
+            self.thread.start()
+
+    def stop(self):
+        if self.running:
+            self.running = False
+            self.thread.join()
+            self.thread = None
+
+    def notify(self, msg):
+        self.queue.put(msg)
+
+    def process(self):
+        while self.running:
+            try:
+                msg = self.queue.get(block=True, timeout=1) # 1s timeout
+            except Empty:
+                continue
+
+            retry_count = 0
+            sent = False
+
+            while retry_count < 3:
+                if self.send(msg):
+                    sent = True
+                    break
+                time.sleep(0.2)
+                retry_count += 1
+
+            if not sent:
+                logger.error("failed to send out, discard msg: %s", msg)
+
+    def send(self, msg):
+        subject = msg.subject()
+
+        try:
+            if self.alert_manager_url is not None:
+                labels = msg.labels()
+                labels.update({
+                    "alertname": msg.alert_name,
+                    "type": "user_alert",
+                    "cluster": self.cluster,
+                    "user_email": msg.email,
+                    "subject": subject,
+                    })
+
+                resp = requests.post(self.alert_manager_url, timeout=5,
+                        data=json.dumps([{"labels": labels}]))
+                resp.raise_for_status()
+                return True
+            elif self.smtp_url is not None and \
+                    self.smtp_from is not None and \
+                    self.smtp_auth_name is not None and \
+                    self.smtp_auth_pass is not None:
+                smtp_send_email(self.smtp_url, self.smtp_from,
+                        self.smtp_auth_name, self.smtp_auth_pass,
+                        msg.email, subject, msg.body())
+                return True
+            else:
+                # not configured, discard message
+                return True
+        except Exception as e:
+            logger.exception("sending email failed")
+            return False
+
+
+def smtp_send_email(smtp_url, smtp_from, smtp_auth_name, smtp_auth_pass, to, subject, body):
+    msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s" % (smtp_from, to, subject, body)
+    conn = smtplib.SMTP(smtp_url)
+    conn.starttls()
+    conn.login(smtp_auth_name, smtp_auth_pass)
+    conn.sendmail(smtp_from, to, msg)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
+    notifier = Notifier({"notifier": {"cluster": "local", "alert-manager-url": "http://localhost:9093/alert-manager"}})
+    notifier.start()
+
+    notifier.notify(new_job_state_change_message("dixu@microsoft.com", "job-id", "stopped"))
+    notifier.stop()

From 399cc3a8644311f24bb66325436779e764044e71 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 9 Jul 2019 06:52:06 +0000
Subject: [PATCH 063/194] call notifier in jobmanager

---
 src/ClusterManager/job_manager.py | 11 +++++++++--
 src/utils/notify.py               |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index b4b2dc955..6469cecbd 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -18,6 +18,7 @@
 import k8sUtils
 import joblog_manager
 from osUtils import mkdirsAsUser
+import notify
 
 import yaml
 from jinja2 import Environment, FileSystemLoader, Template
@@ -179,7 +180,7 @@ def AutoApproveJob(job):
 
 UnusualJobs = {}
 
-def UpdateJobStatus(job):
+def UpdateJobStatus(job, notifier):
     dataHandler = DataHandler()
     jobParams = json.loads(base64.b64decode(job["jobParams"]))
 
@@ -225,6 +226,10 @@ def UpdateJobStatus(job):
 
     elif result.strip() == "Failed":
         logging.warning("Job %s fails, cleaning...", job["jobId"])
+
+        notifier.notify(notify.new_job_state_change_message(
+            job["userName"], job["jobId"], result.strip()))
+
         joblog_manager.extract_job_log(job["jobId"],logPath,jobParams["userId"])
         dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","failed")
         dataHandler.UpdateJobTextField(job["jobId"],"errorMsg",detail)
@@ -480,6 +485,8 @@ def TakeJobActions(jobs):
 
 
 def Run():
+    notifier = Notifier(config.get("job-manager"))
+    notifier.start()
 
     while True:
 
@@ -505,7 +512,7 @@ def Run():
                         elif job["jobStatus"] == "pausing":
                             KillJob(job, "paused")
                         elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
-                            UpdateJobStatus(job)
+                            UpdateJobStatus(job, notifier)
                         elif job["jobStatus"] == "unapproved":
                             AutoApproveJob(job)
                     except Exception as e:
diff --git a/src/utils/notify.py b/src/utils/notify.py
index 67278ae98..584f99c7f 100644
--- a/src/utils/notify.py
+++ b/src/utils/notify.py
@@ -56,7 +56,7 @@ def __init__(self, config):
         self.alert_manager_url = None
         self.smtp_url = self.smtp_from = self.smtp_auth_name = self.smtp_auth_pass = None
 
-        if "notifier" in config:
+        if config is not None and "notifier" in config:
             notifier_config = config["notifier"]
 
             self.cluster = notifier_config.get("cluster")

From ef61d4047d34edcbd7104df948236b1258d7a1c6 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 17 Jul 2019 06:50:57 +0000
Subject: [PATCH 064/194] fix according to review

---
 src/ClusterManager/job_manager.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 6469cecbd..e89267355 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -180,7 +180,7 @@ def AutoApproveJob(job):
 
 UnusualJobs = {}
 
-def UpdateJobStatus(job, notifier):
+def UpdateJobStatus(job, notifier=None):
     dataHandler = DataHandler()
     jobParams = json.loads(base64.b64decode(job["jobParams"]))
 
@@ -227,8 +227,9 @@ def UpdateJobStatus(job, notifier):
     elif result.strip() == "Failed":
         logging.warning("Job %s fails, cleaning...", job["jobId"])
 
-        notifier.notify(notify.new_job_state_change_message(
-            job["userName"], job["jobId"], result.strip()))
+        if notifier is not None:
+            notifier.notify(notify.new_job_state_change_message(
+                job["userName"], job["jobId"], result.strip()))
 
         joblog_manager.extract_job_log(job["jobId"],logPath,jobParams["userId"])
         dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","failed")

From cb11da1cb157fc7ea2e6dc320c753a034a3e2893 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 9 Jul 2019 09:42:12 +0000
Subject: [PATCH 065/194] refine kill job

---
 src/ClusterManager/job_deployer.py      | 80 ++++++++++++++++++++++---
 src/ClusterManager/job_manager.py       | 32 +++++-----
 src/ClusterManager/test_job_deployer.py | 41 +++++++++++++
 3 files changed, 128 insertions(+), 25 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index b9f502754..62272ee67 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -2,6 +2,7 @@
 import os
 import logging
 from kubernetes import client, config
+from kubernetes.client.rest import ApiException
 
 
 class JobDeployer:
@@ -29,27 +30,88 @@ def delete_pod(self, name, dry_run=None):
             pretty=self.pretty,
             body=client.V1DeleteOptions(),
             dry_run=dry_run,
-            #  grace_period_seconds=grace_period_seconds,
-            #  orphan_dependents=orphan_dependents,
-            #  propagation_policy=propagation_policy,
         )
-        pass
+        return api_response
+
+    def create_service(self, body, dry_run=None):
+        api_response = self.v1.create_namespaced_service(
+            namespace=self.namespace,
+            body=body,
+            pretty=self.pretty,
+            dry_run=dry_run,
+        )
+        return api_response
+
+    def delete_service(self, name, dry_run=None):
+        api_response = self.v1.delete_namespaced_service(
+            name=name,
+            namespace=self.namespace,
+            pretty=self.pretty,
+            body=client.V1DeleteOptions(),
+            dry_run=dry_run,
+        )
+        return api_response
 
     def cleanup_pods(self, pods):
+        errors = []
         for pod in pods:
             try:
                 pod_name = pod["metadata"]["name"]
                 self.delete_pod(pod_name)
             except Exception as e:
-                logging.warning("Delete pod failed: %s!" % pod_name, exc_info=True)
+                message = "Delete pod failed: {}".format(pod_name)
+                logging.warning(message, exc_info=True)
+                errors.append({"message": message, "exception": e})
+        return errors
+
+    def cleanup_services(self, services):
+        errors = []
+        for service in services:
+            try:
+                service_name = service["metadata"]["name"]
+                self.delete_service(service_name)
+            except ApiException as e:
+                message = "Delete service failed: {}".format(service_name)
+                logging.warning(message, exc_info=True)
+                errors.append({"message": message, "exception": e})
+        return errors
 
     def create_pods(self, pods):
         # TODO instead of delete, we could check update existiong ones. During refactoring, keeping the old way.
         self.cleanup_pods(pods)
         created = []
         for pod in pods:
-            self.create_pod(pod)
-            pod_name = pod["metadata"]["name"]
-            created.append(pod_name)
-            logging.info("Create pod succeed: %s" % pod_name)
+            created_pod = self.create_pod(pod)
+            created.append(created_pod)
+            logging.info("Create pod succeed: %s" % created_pod.metadata.name)
         return created
+
+    def get_pods_by_label(self, label_selector):
+        api_response = self.v1.list_namespaced_pod(
+            namespace=self.namespace,
+            pretty=self.pretty,
+            label_selector=label_selector,
+        )
+        return api_response.items
+
+    def get_services_by_label(self, label_selector):
+        api_response = self.v1.list_namespaced_service(
+            namespace=self.namespace,
+            pretty=self.pretty,
+            label_selector=label_selector,
+        )
+        return api_response.items
+
+    def delete_job(self, job_id):
+        label_selector = "run={}".format(job_id)
+
+        # query pods then delete
+        pods = self.get_pod_by_label(label_selector)
+        pod_errors = self.cleanup_pods(pods)
+
+        # query services then delete
+        services = self.get_service_by_label(label_selector)
+        service_errors = self.cleanup_services(services)
+
+        errors = pod_errors + service_errors
+        return errors
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index b4b2dc955..c7d00273c 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -86,7 +86,8 @@ def SubmitJob(job):
 
         job_deployer = JobDeployer()
         try:
-            ret["output"] = job_deployer.create_pods(pods)
+            pods = job_deployer.create_pods(pods)
+            ret["output"] = "Created pods: {}".format([pod.metedata.name for pod in pods])
         except Exception as e:
             ret["output"] = "Error: %s" % e.message
             logging.error(e, exc_info=True)
@@ -121,22 +122,21 @@ def SubmitJob(job):
 def KillJob(job, desiredState="killed"):
     dataHandler = DataHandler()
     result, detail = k8sUtils.GetJobStatus(job["jobId"])
-    dataHandler.UpdateJobTextField(job["jobId"],"jobStatusDetail",base64.b64encode(json.dumps(detail)))
-    logging.info("Killing job %s, with status %s, %s" %(job["jobId"], result,detail))
-    if "jobDescriptionPath" in job and job["jobDescriptionPath"] is not None:
-        jobDescriptionPath = os.path.join(config["storage-mount-path"], job["jobDescriptionPath"])
-        if os.path.isfile(jobDescriptionPath):
-            if k8sUtils.kubectl_delete(jobDescriptionPath) == 0:
-                dataHandler.UpdateJobTextField(job["jobId"],"jobStatus", desiredState)
-                return True
-            else:
-                dataHandler.UpdateJobTextField(job["jobId"],"errorMsg","Cannot delete job from Kubernetes Cluster!")
-    else:
-        dataHandler.UpdateJobTextField(job["jobId"],"errorMsg","Cannot find job description file!")
+    dataHandler.UpdateJobTextField(job["jobId"], "jobStatusDetail", base64.b64encode(json.dumps(detail)))
+    logging.info("Killing job %s, with status %s, %s" % (job["jobId"], result, detail))
 
-    dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","error")
-    dataHandler.Close()
-    return False
+    job_deployer = JobDeployer()
+    errors = job_deployer.delete_job(job["jobId"])
+
+    if len(errors) == 0:
+        dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", desiredState)
+        dataHandler.Close()
+        return True
+    else:
+        dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
+        dataHandler.Close()
+        logger.error("Kill job failed with errors: {}".format(errors))
+        return False
 
 
 # TODO remove it latter
diff --git a/src/ClusterManager/test_job_deployer.py b/src/ClusterManager/test_job_deployer.py
index d466e596b..295af0461 100644
--- a/src/ClusterManager/test_job_deployer.py
+++ b/src/ClusterManager/test_job_deployer.py
@@ -47,3 +47,44 @@ def test_cleanup_pods(self):
         ]
 
         job_deployer.cleanup_pods(pods)
+
+    def test_get_pod_by_label(self):
+        job_deployer = self.create_job_deployer()
+        label_selector = "run=some_job_id"
+
+        pods = job_deployer.get_pods_by_label(label_selector)
+
+        self.assertEqual(0, len(pods))
+
+    def test_get_services_by_label(self):
+        job_deployer = self.create_job_deployer()
+        label_selector = "run=some_job_id"
+
+        services = job_deployer.get_services_by_label(label_selector)
+
+        self.assertEqual(0, len(services))
+
+    def test_create_endpoint(self):
+        job_deployer = self.create_job_deployer()
+        raw_yaml = """
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-service
+spec:
+  selector:
+    app: MyApp
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 9376
+    """
+        body = yaml.full_load(raw_yaml)
+
+        # with self.assertRaises(ApiException):
+        job_deployer.create_service(body)
+
+    def test_delete_service(self):
+        job_deployer = self.create_job_deployer()
+
+        job_deployer.delete_service("test-service")

From 80318d9d787542edeb8c747cac32957462e40d08 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 10 Jul 2019 08:00:49 +0000
Subject: [PATCH 066/194] update job.lastUpdated

---
 src/ClusterManager/job_manager.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index c7d00273c..f77eff3ee 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -130,10 +130,12 @@ def KillJob(job, desiredState="killed"):
 
     if len(errors) == 0:
         dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", desiredState)
+        dataHandler.UpdateJobTextField(job["jobId"], "lastUpdated", datetime.datetime.now().isoformat())
         dataHandler.Close()
         return True
     else:
         dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
+        dataHandler.UpdateJobTextField(job["jobId"], "lastUpdated", datetime.datetime.now().isoformat())
         dataHandler.Close()
         logger.error("Kill job failed with errors: {}".format(errors))
         return False

From 4f7f3e480c528d4bdfd14c3e131a78026fd2ab50 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 10 Jul 2019 09:32:28 +0000
Subject: [PATCH 067/194] add func pod_exec

---
 src/ClusterManager/job_deployer.py      | 29 +++++++++++++++++++++++++
 src/ClusterManager/test_job_deployer.py | 28 ++++++++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 62272ee67..2d62abe88 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -3,6 +3,8 @@
 import logging
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
+from kubernetes.stream import stream
+from kubernetes.stream.ws_client import ERROR_CHANNEL, STDERR_CHANNEL, STDOUT_CHANNEL
 
 
 class JobDeployer:
@@ -115,3 +117,30 @@ def delete_job(self, job_id):
 
         errors = pod_errors + service_errors
         return errors
+
+    def pod_exec(self, pod_name, exec_command):
+        logging.info("Exec on pod {}: {}".format(pod_name, exec_command))
+        client = stream(
+            self.v1.connect_get_namespaced_pod_exec,
+            name=pod_name,
+            namespace=self.namespace,
+            command=exec_command,
+            stderr=True,
+            stdin=False,
+            stdout=True,
+            tty=False,
+            _preload_content=False,
+        )
+
+        client.run_forever(timeout=60)
+        err = yaml.full_load(client.read_channel(ERROR_CHANNEL))
+        if err["status"] == "Success":
+            status_code = 0
+        else:
+            logging.warning("Exec on pod {} failed: {}".format(pod_name, err))
+            status_code = int(err["details"]["causes"][0]["message"])
+
+        output = client.read_channel(STDOUT_CHANNEL) + client.read_channel(STDERR_CHANNEL)
+
+        logging.info("Exec on pod {}, status: {}, output: {}".format(pod_name, status_code, output))
+        return [status_code, output]
diff --git a/src/ClusterManager/test_job_deployer.py b/src/ClusterManager/test_job_deployer.py
index 295af0461..c599be348 100644
--- a/src/ClusterManager/test_job_deployer.py
+++ b/src/ClusterManager/test_job_deployer.py
@@ -5,6 +5,15 @@
 
 from job_deployer import JobDeployer
 
+import logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s",
+    handlers=[
+        logging.StreamHandler()
+    ]
+)
+
 
 class TestJobDeployer(unittest.TestCase):
 
@@ -88,3 +97,22 @@ def test_delete_service(self):
         job_deployer = self.create_job_deployer()
 
         job_deployer.delete_service("test-service")
+
+    def test_pod_exec(self):
+        job_deployer = self.create_job_deployer()
+        exec_command = [
+            '/bin/sh',
+            '-c',
+            'echo This message goes to stderr >&2 && echo This message goes to stdout'
+        ]
+
+        status_code, ouput = job_deployer.pod_exec("test-pod", exec_command)
+        self.assertEqual(0, status_code)
+
+        bad_command = [
+            '/bin/sh',
+            '-c',
+            'echo This message goes to stderr >&2 && xecho This message goes to stdout; exit 8'
+        ]
+        status_code, ouput = job_deployer.pod_exec("test-pod", bad_command)
+        self.assertEqual(8, status_code)

From ffe8f7301e2930e98d95c3c98c33dbf5c4922ddd Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 11 Jul 2019 12:12:38 +0000
Subject: [PATCH 068/194] add timeout

---
 src/ClusterManager/job_deployer.py      | 10 ++++++----
 src/ClusterManager/test_job_deployer.py | 10 +++++++++-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 2d62abe88..772cdab1d 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -118,7 +118,8 @@ def delete_job(self, job_id):
         errors = pod_errors + service_errors
         return errors
 
-    def pod_exec(self, pod_name, exec_command):
+    def pod_exec(self, pod_name, exec_command, timeout=60):
+        """work as the command (with timeout): kubectl exec 'pod_name' 'exec_command'"""
         logging.info("Exec on pod {}: {}".format(pod_name, exec_command))
         client = stream(
             self.v1.connect_get_namespaced_pod_exec,
@@ -131,16 +132,17 @@ def pod_exec(self, pod_name, exec_command):
             tty=False,
             _preload_content=False,
         )
+        client.run_forever(timeout=timeout)
 
-        client.run_forever(timeout=60)
         err = yaml.full_load(client.read_channel(ERROR_CHANNEL))
+        if err is None:
+            return [-1, "Timeout"]
+
         if err["status"] == "Success":
             status_code = 0
         else:
             logging.warning("Exec on pod {} failed: {}".format(pod_name, err))
             status_code = int(err["details"]["causes"][0]["message"])
-
         output = client.read_channel(STDOUT_CHANNEL) + client.read_channel(STDERR_CHANNEL)
-
         logging.info("Exec on pod {}, status: {}, output: {}".format(pod_name, status_code, output))
         return [status_code, output]
diff --git a/src/ClusterManager/test_job_deployer.py b/src/ClusterManager/test_job_deployer.py
index c599be348..091a268fc 100644
--- a/src/ClusterManager/test_job_deployer.py
+++ b/src/ClusterManager/test_job_deployer.py
@@ -112,7 +112,15 @@ def test_pod_exec(self):
         bad_command = [
             '/bin/sh',
             '-c',
-            'echo This message goes to stderr >&2 && xecho This message goes to stdout; exit 8'
+            'echo This message goes to stderr >&2 && xecho This message goes to stdout; sleep 3; exit 8'
         ]
         status_code, ouput = job_deployer.pod_exec("test-pod", bad_command)
         self.assertEqual(8, status_code)
+
+        bad_command = [
+            '/bin/sh',
+            '-c',
+            'echo This message goes to stderr >&2 && xecho This message goes to stdout; sleep 3; exit 8'
+        ]
+        status_code, ouput = job_deployer.pod_exec("test-pod", bad_command, 1)
+        self.assertEqual(-1, status_code)

From 786337c1fdd27bf35336aac6e60cfc08e3095447 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 12 Jul 2019 08:51:22 +0000
Subject: [PATCH 069/194] fix typo

---
 src/ClusterManager/job_deployer.py | 8 ++++----
 src/ClusterManager/job_manager.py  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 772cdab1d..34ec95dde 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -58,7 +58,7 @@ def cleanup_pods(self, pods):
         errors = []
         for pod in pods:
             try:
-                pod_name = pod["metadata"]["name"]
+                pod_name = pod.metadata.name
                 self.delete_pod(pod_name)
             except Exception as e:
                 message = "Delete pod failed: {}".format(pod_name)
@@ -70,7 +70,7 @@ def cleanup_services(self, services):
         errors = []
         for service in services:
             try:
-                service_name = service["metadata"]["name"]
+                service_name = service.metadata.name
                 self.delete_service(service_name)
             except ApiException as e:
                 message = "Delete service failed: {}".format(service_name)
@@ -108,11 +108,11 @@ def delete_job(self, job_id):
         label_selector = "run={}".format(job_id)
 
         # query pods then delete
-        pods = self.get_pod_by_label(label_selector)
+        pods = self.get_pods_by_label(label_selector)
         pod_errors = self.cleanup_pods(pods)
 
         # query services then delete
-        services = self.get_service_by_label(label_selector)
+        services = self.get_services_by_label(label_selector)
         service_errors = self.cleanup_services(services)
 
         errors = pod_errors + service_errors
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index f77eff3ee..88bfc050c 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -137,7 +137,7 @@ def KillJob(job, desiredState="killed"):
         dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
         dataHandler.UpdateJobTextField(job["jobId"], "lastUpdated", datetime.datetime.now().isoformat())
         dataHandler.Close()
-        logger.error("Kill job failed with errors: {}".format(errors))
+        logging.error("Kill job failed with errors: {}".format(errors))
         return False
 
 

From 72416647b685c9ba6734e95b6d44fe121c9fed55 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Mon, 15 Jul 2019 08:04:04 +0000
Subject: [PATCH 070/194] refine

---
 src/ClusterManager/job_deployer.py      | 21 ++++++++------
 src/ClusterManager/job_manager.py       |  4 +--
 src/ClusterManager/test_job_deployer.py | 37 +++++++++++++++----------
 3 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 34ec95dde..d0b38c140 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -54,11 +54,10 @@ def delete_service(self, name, dry_run=None):
         )
         return api_response
 
-    def cleanup_pods(self, pods):
+    def cleanup_pods(self, pod_names):
         errors = []
-        for pod in pods:
+        for pod_name in pod_names:
             try:
-                pod_name = pod.metadata.name
                 self.delete_pod(pod_name)
             except Exception as e:
                 message = "Delete pod failed: {}".format(pod_name)
@@ -69,6 +68,7 @@ def cleanup_pods(self, pods):
     def cleanup_services(self, services):
         errors = []
         for service in services:
+            assert(isinstance(service, client.V1Service))
             try:
                 service_name = service.metadata.name
                 self.delete_service(service_name)
@@ -80,7 +80,8 @@ def cleanup_services(self, services):
 
     def create_pods(self, pods):
         # TODO instead of delete, we could check update existiong ones. During refactoring, keeping the old way.
-        self.cleanup_pods(pods)
+        pod_names = [pod["metadata"]["name"] for pod in pods]
+        self.cleanup_pods(pod_names)
         created = []
         for pod in pods:
             created_pod = self.create_pod(pod)
@@ -88,10 +89,11 @@ def create_pods(self, pods):
             logging.info("Create pod succeed: %s" % created_pod.metadata.name)
         return created
 
-    def get_pods_by_label(self, label_selector):
+    def get_pods(self, field_selector="", label_selector=""):
         api_response = self.v1.list_namespaced_pod(
             namespace=self.namespace,
             pretty=self.pretty,
+            field_selector=field_selector,
             label_selector=label_selector,
         )
         return api_response.items
@@ -108,8 +110,9 @@ def delete_job(self, job_id):
         label_selector = "run={}".format(job_id)
 
         # query pods then delete
-        pods = self.get_pods_by_label(label_selector)
-        pod_errors = self.cleanup_pods(pods)
+        pods = self.get_pods(label_selector=label_selector)
+        pod_names = [pod.metadata.name for pod in pods]
+        pod_errors = self.cleanup_pods(pod_names)
 
         # query services then delete
         services = self.get_services_by_label(label_selector)
@@ -141,8 +144,8 @@ def pod_exec(self, pod_name, exec_command, timeout=60):
         if err["status"] == "Success":
             status_code = 0
         else:
-            logging.warning("Exec on pod {} failed: {}".format(pod_name, err))
+            logging.warning("Exec on pod {} failed. cmd: {}, err: {}.".format(pod_name, exec_command, err))
             status_code = int(err["details"]["causes"][0]["message"])
         output = client.read_channel(STDOUT_CHANNEL) + client.read_channel(STDERR_CHANNEL)
-        logging.info("Exec on pod {}, status: {}, output: {}".format(pod_name, status_code, output))
+        logging.info("Exec on pod {}, status: {}, cmd: {}, output: {}".format(pod_name, status_code, exec_command, output))
         return [status_code, output]
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 88bfc050c..3266e1588 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -87,7 +87,7 @@ def SubmitJob(job):
         job_deployer = JobDeployer()
         try:
             pods = job_deployer.create_pods(pods)
-            ret["output"] = "Created pods: {}".format([pod.metedata.name for pod in pods])
+            ret["output"] = "Created pods: {}".format([pod.metadata.name for pod in pods])
         except Exception as e:
             ret["output"] = "Error: %s" % e.message
             logging.error(e, exc_info=True)
@@ -104,7 +104,7 @@ def SubmitJob(job):
         jobMeta["jobPath"] = job_object.job_path
         jobMeta["workPath"] = job_object.work_path
         # the command of the first container
-        jobMeta["LaunchCMD"] = pods[0]["spec"]["containers"][0]["command"]
+        jobMeta["LaunchCMD"] = pods[0].spec.containers[0].command
 
         jobMetaStr = base64.b64encode(json.dumps(jobMeta))
         dataHandler.UpdateJobTextField(job_object.job_id, "jobMeta", jobMetaStr)
diff --git a/src/ClusterManager/test_job_deployer.py b/src/ClusterManager/test_job_deployer.py
index 091a268fc..b5f0ff9df 100644
--- a/src/ClusterManager/test_job_deployer.py
+++ b/src/ClusterManager/test_job_deployer.py
@@ -1,6 +1,9 @@
 import unittest
 import kubernetes
 import yaml
+import string
+import random
+import time
 from kubernetes.client.rest import ApiException
 
 from job_deployer import JobDeployer
@@ -22,13 +25,13 @@ def create_job_deployer(self):
         self.assertIsNotNone(job_deployer)
         return job_deployer
 
-    def test_create_pod(self):
+    def create_pod(self, pod_name):
         job_deployer = self.create_job_deployer()
         raw_yaml = """
 apiVersion: v1
 kind: Pod
 metadata:
-  name: test-pod
+  name: {}
 spec:
   containers:
   - name: busybox
@@ -36,32 +39,31 @@ def test_create_pod(self):
     args:
     - sleep
     - "1000000"
-    """
+    """.format(pod_name)
         body = yaml.full_load(raw_yaml)
 
         # with self.assertRaises(ApiException):
         job_deployer.create_pod(body)
 
     def test_delete_pod(self):
+        pod_name = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
+        self.create_pod(pod_name)
+
         job_deployer = self.create_job_deployer()
 
-        job_deployer.delete_pod("test-pod")
+        job_deployer.delete_pod(pod_name)
 
     def test_cleanup_pods(self):
         job_deployer = self.create_job_deployer()
+        pod_names = ["pod-1", "pod-2"]
 
-        pods = [
-            {"metadata": {"name": "pod-1"}},
-            {"metadata": {"name": "pod-2"}},
-        ]
-
-        job_deployer.cleanup_pods(pods)
+        job_deployer.cleanup_pods(pod_names)
 
     def test_get_pod_by_label(self):
         job_deployer = self.create_job_deployer()
         label_selector = "run=some_job_id"
 
-        pods = job_deployer.get_pods_by_label(label_selector)
+        pods = job_deployer.get_pods(label_selector=label_selector)
 
         self.assertEqual(0, len(pods))
 
@@ -100,13 +102,18 @@ def test_delete_service(self):
 
     def test_pod_exec(self):
         job_deployer = self.create_job_deployer()
+
+        pod_name = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
+        self.create_pod(pod_name)
+        time.sleep(3)
+
         exec_command = [
             '/bin/sh',
             '-c',
             'echo This message goes to stderr >&2 && echo This message goes to stdout'
         ]
 
-        status_code, ouput = job_deployer.pod_exec("test-pod", exec_command)
+        status_code, ouput = job_deployer.pod_exec(pod_name, exec_command)
         self.assertEqual(0, status_code)
 
         bad_command = [
@@ -114,7 +121,7 @@ def test_pod_exec(self):
             '-c',
             'echo This message goes to stderr >&2 && xecho This message goes to stdout; sleep 3; exit 8'
         ]
-        status_code, ouput = job_deployer.pod_exec("test-pod", bad_command)
+        status_code, ouput = job_deployer.pod_exec(pod_name, bad_command)
         self.assertEqual(8, status_code)
 
         bad_command = [
@@ -122,5 +129,7 @@ def test_pod_exec(self):
             '-c',
             'echo This message goes to stderr >&2 && xecho This message goes to stdout; sleep 3; exit 8'
         ]
-        status_code, ouput = job_deployer.pod_exec("test-pod", bad_command, 1)
+        status_code, ouput = job_deployer.pod_exec(pod_name, bad_command, 1)
         self.assertEqual(-1, status_code)
+
+        job_deployer.delete_pod(pod_name)

From 1508b8af919d1d89c0d94d099eec3bb63bf9e109 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 02:31:59 +0000
Subject: [PATCH 071/194] add JobRole

---
 src/ClusterManager/job_role.py      | 69 +++++++++++++++++++++++++++++
 src/ClusterManager/test_job_role.py | 34 ++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 src/ClusterManager/job_role.py
 create mode 100644 src/ClusterManager/test_job_role.py

diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
new file mode 100644
index 000000000..e782169ad
--- /dev/null
+++ b/src/ClusterManager/job_role.py
@@ -0,0 +1,69 @@
+from job_deployer import JobDeployer
+
+
+class JobRole:
+    # MARK_CONTAINER_READY_FILE = "/dlws/CONTAINER_READY"
+    # MARK_WORKER_READY_FILE = "/dlws/WORKER_READY"
+    MARK_JOB_READY_FILE = "/dlws/JOB_READY"
+
+    @staticmethod
+    def get_job_roles(job_id):
+        deployer = JobDeployer()
+        pods = deployer.get_pods(label_selector="run={}".format(job_id))
+
+        job_roles = []
+        for pod in pods:
+            pod_name = pod.metadata.name
+            if "distRole" in pod.metadata.labels:
+                role = pod.metadata.labels["distRole"]
+            else:
+                role = "master"
+            job_role = JobRole(role, pod_name)
+            job_roles.append(job_role)
+        return job_roles
+
+    def __init__(self, role_name, pod_name):
+        self.role_name = role_name
+        self.pod_name = pod_name
+
+    def status(self):
+        """
+        Return role status.
+        It's slightly different from pod phase, when pod is running:
+            CONTAINER_READY -> WORKER_READY -> JOB_READY (then the job finally in "Running" status.)
+        """
+        # pod-phase: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+        deployer = JobDeployer()
+        pods = deployer.get_pods(field_selector="metadata.name={}".format(self.pod_name))
+        if(len(pods) < 1):
+            return "NotFound"
+
+        assert(len(pods) == 1)
+        pod = pods[0]
+        phase = pod.status.phase
+
+        # !!! Pod is runing, doesn't mean "Role" is ready and running.
+        if(phase == "Running"):
+            if not self.isJobReady():
+                return "Pending"
+
+        # TODO handle exit status
+
+        return phase
+
+    def isFileExisting(self, file):
+        deployer = JobDeployer()
+        status_code, _ = deployer.pod_exec(self.pod_name, ["/bin/sh", "-c", "ls -lrt {}".format(file)])
+        return status_code == 0
+
+    # def isUserReady(self):
+    #     return self.isFileExisting(JobRole.MARK_USER_READY_FILE)
+
+    # def isWorkerReady(self):
+    #     return self.isFileExisting(JobRole.MARK_WORKER_READY_FILE)
+
+    def isJobReady(self):
+        # only mark job ready on the "ps" or "master" pod
+        if(self.role_name not in ["ps", "master"]):
+            return False
+        return self.isFileExisting(JobRole.MARK_JOB_READY_FILE)
diff --git a/src/ClusterManager/test_job_role.py b/src/ClusterManager/test_job_role.py
new file mode 100644
index 000000000..919332053
--- /dev/null
+++ b/src/ClusterManager/test_job_role.py
@@ -0,0 +1,34 @@
+import unittest
+from job_role import JobRole
+
+
+class TestJobRole(unittest.TestCase):
+
+    def test_status_Running(self):
+        job_role = JobRole("master", "bd3d090a-53b6-4616-9b6c-fe4a86fd68ea-ps0")
+
+        role_status = job_role.status()
+        self.assertEqual("Running", role_status)
+
+    def test_status_NotFound(self):
+        job_role = JobRole("master", "bd3d090a-53b6-4616-9b6c-fe4a86fd68ea-ps0-not-found")
+
+        role_status = job_role.status()
+        self.assertEqual("NotFound", role_status)
+
+    def test_status_Pending(self):
+        # Pod is running, but mark file not existing: JobRole.MARK_POD_READY_FILE
+        job_role = JobRole("master", "nginx-cm7kf")
+
+        role_status = job_role.status()
+        self.assertEqual("Pending", role_status)
+
+    def test_get_job_roles_dist_job(self):
+        job_roles = JobRole.get_job_roles("bd3d090a-53b6-4616-9b6c-fe4a86fd68ea")
+
+        self.assertEqual(3, len(job_roles))
+
+    def test_get_job_roles_regular_job(self):
+        job_roles = JobRole.get_job_roles("8ca7fcdf-c4e7-4687-a3fa-1eeea97415c4")
+
+        self.assertEqual(1, len(job_roles))

From da04f506f2f93d7ab6d301941e4ad9bea54b4683 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 02:58:51 +0000
Subject: [PATCH 072/194] install requirements

---
 src/docker-images/RestfulAPI/Dockerfile | 68 ++++++++++++++-----------
 1 file changed, 38 insertions(+), 30 deletions(-)

diff --git a/src/docker-images/RestfulAPI/Dockerfile b/src/docker-images/RestfulAPI/Dockerfile
index b84c01ed9..659c12a4b 100755
--- a/src/docker-images/RestfulAPI/Dockerfile
+++ b/src/docker-images/RestfulAPI/Dockerfile
@@ -1,30 +1,38 @@
-FROM dlws/restfulapi:v1.5
-MAINTAINER Hongzhi Li <Hongzhi.Li@microsoft.com>
-
-COPY kubectl /usr/local/bin/kubectl
-RUN chmod +x /usr/local/bin/kubectl
-#COPY gittoken /root/.ssh/id_rsa
-#RUN chmod 400 /root/.ssh/id_rsa
-
-RUN rm /etc/apache2/mods-enabled/mpm_*
-COPY mpm_prefork.conf /etc/apache2/mods-available/mpm_prefork.conf
-COPY 000-default.conf /etc/apache2/sites-available/000-default.conf
-COPY ports.conf /etc/apache2/ports.conf
-RUN ln -s /etc/apache2/mods-available/mpm_prefork.conf /etc/apache2/mods-enabled/mpm_prefork.conf
-RUN ln -s /etc/apache2/mods-available/mpm_prefork.load /etc/apache2/mods-enabled/mpm_prefork.load
-
-COPY dlws-restfulapi.wsgi /wsgi/dlws-restfulapi.wsgi
-
-
-COPY runScheduler.sh /
-RUN chmod +x /runScheduler.sh
-COPY pullsrc.sh /
-RUN chmod +x /pullsrc.sh
-COPY run.sh /
-RUN chmod +x /run.sh
-
-ADD Jobs_Templete /DLWorkspace/src/Jobs_Templete
-ADD utils /DLWorkspace/src/utils
-ADD RestAPI /DLWorkspace/src/RestAPI
-ADD ClusterManager /DLWorkspace/src/ClusterManager
-CMD /run.sh
+FROM dlws/restfulapi:v1.5
+MAINTAINER Hongzhi Li <Hongzhi.Li@microsoft.com>
+
+COPY kubectl /usr/local/bin/kubectl
+RUN chmod +x /usr/local/bin/kubectl
+#COPY gittoken /root/.ssh/id_rsa
+#RUN chmod 400 /root/.ssh/id_rsa
+
+RUN rm /etc/apache2/mods-enabled/mpm_*
+COPY mpm_prefork.conf /etc/apache2/mods-available/mpm_prefork.conf
+COPY 000-default.conf /etc/apache2/sites-available/000-default.conf
+COPY ports.conf /etc/apache2/ports.conf
+RUN ln -s /etc/apache2/mods-available/mpm_prefork.conf /etc/apache2/mods-enabled/mpm_prefork.conf
+RUN ln -s /etc/apache2/mods-available/mpm_prefork.load /etc/apache2/mods-enabled/mpm_prefork.load
+
+COPY dlws-restfulapi.wsgi /wsgi/dlws-restfulapi.wsgi
+
+
+COPY runScheduler.sh /
+RUN chmod +x /runScheduler.sh
+COPY pullsrc.sh /
+RUN chmod +x /pullsrc.sh
+COPY run.sh /
+RUN chmod +x /run.sh
+
+ADD Jobs_Templete /DLWorkspace/src/Jobs_Templete
+ADD utils /DLWorkspace/src/utils
+ADD RestAPI /DLWorkspace/src/RestAPI
+ADD ClusterManager /DLWorkspace/src/ClusterManager
+
+# TODO refine later
+# install requirements
+RUN rm -rf /usr/lib/python2.7/dist-packages/yaml
+RUN rm -rf /usr/lib/python2.7/dist-packages/PyYAML-*
+RUN pip install -r /DLWorkspace/src/ClusterManager/requirements.txt
+
+
+CMD /run.sh

From 1b4ac62069ebc54c9145a6441f602d6eb21ed684 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 07:28:35 +0000
Subject: [PATCH 073/194] refine job bootup script

---
 src/ClusterManager/pod_template.py  |  9 +++----
 src/Jobs_Templete/bootstrap.sh      | 38 +++++++++++++++++++++++++++++
 src/Jobs_Templete/init_user.sh      |  3 +--
 src/Jobs_Templete/pod.yaml.template |  6 +++++
 4 files changed, 48 insertions(+), 8 deletions(-)
 create mode 100644 src/Jobs_Templete/bootstrap.sh

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index eac50bff9..12d0dd501 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -19,15 +19,12 @@ def generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script):
         if not os.path.exists(path_to_save):
             mkdirsAsUser(path_to_save, user_id)
 
-        file_name = "launch-%s.sh" % job_id
+        file_name = "job_command.sh"
         launch_script_file = os.path.join(path_to_save, file_name)
         with open(launch_script_file, 'w') as f:
-            f.write("#!/bin/bash -x\n")
-            f.write("mkdir /opt; \n")
-            f.write("echo 'localhost slots=%s' | tee -a /opt/hostfile; \n" % gpu_num)
-            f.write("bash /dlws/init_user.sh &>> /pod/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c '%s'\n" % user_script)
+            f.write(user_script)
         os.system("sudo chown %s %s" % (user_id, launch_script_file))
-        luanch_cmd = "[\"bash\", \"/pod/%s\"]" % file_name
+        luanch_cmd = ["bash", "/pod/scripts/bootstrap.sh"]
         return luanch_cmd
 
     def generate_pod(self, pod):
diff --git a/src/Jobs_Templete/bootstrap.sh b/src/Jobs_Templete/bootstrap.sh
new file mode 100644
index 000000000..0dcd1d6f4
--- /dev/null
+++ b/src/Jobs_Templete/bootstrap.sh
@@ -0,0 +1,38 @@
+#! /bin/bash
+set -ex
+
+SCRIPT_DIR=/pod/scripts
+
+# Dir for saving running status
+PROC_DIR=/pod/running
+rm -rf ${PROC_DIR}
+mkdir -p ${PROC_DIR}
+
+# Dir for logs
+LOG_DIR=/pod/logs
+rm -rf ${LOG_DIR}
+mkdir -p ${LOG_DIR}
+
+# Save the pid.
+PID_FILE=${PROC_DIR}/pid
+echo $$ > $PID_FILE
+
+# Setup container
+bash ${SCRIPT_DIR}/init_user.sh &>> ${LOG_DIR}/init_user.log
+touch ${PROC_DIR}/CONTAINER_READY
+
+# Setup roles
+# TODO
+touch ${PROC_DIR}/ROLE_READY
+
+# Setup job
+# TODO
+touch ${PROC_DIR}/JOB_READY
+
+set +e
+# Execute user's command for the job
+chmod +x /pod/job_command.sh
+runuser -l ${DLWS_USER_NAME} -c /pod/job_command.sh
+# Save exit code
+EXIT_CODE=$?
+echo  `date` ": $EXIT_CODE"  > ${PROC_DIR}/EXIT_CODE
diff --git a/src/Jobs_Templete/init_user.sh b/src/Jobs_Templete/init_user.sh
index 3a652fd06..e331347cd 100644
--- a/src/Jobs_Templete/init_user.sh
+++ b/src/Jobs_Templete/init_user.sh
@@ -5,7 +5,7 @@ set -ex
 #export DLWS_GID=
 #export DLWS_UID=
 #export DLWS_USER_NAME=
-export ENV_FILE=/dlws/pod.env
+export ENV_FILE=/pod/pod.env
 
 # setup user and group, fix permissions
 addgroup --force-badname --gid  ${DLWS_GID} domainusers
@@ -38,6 +38,5 @@ if [ -f ${ENV_FILE} ]; then
 fi
 SCRIPT
 
-touch /dlws/USER_READY
 # any command should run as ${DLWS_USER_NAME}
 #runuser -l ${DLWS_USER_NAME} -c your_commands
diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index 6bf0bc11a..8aafc7c54 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -80,6 +80,9 @@ spec:
     - name: "init-user-script"
       mountPath: /dlws/init_user.sh
       subPath: init_user.sh
+    - name: "dlws-scripts"
+      mountPath: /pod/scripts
+      readOnly: true
     - name: ssh-volume
       mountPath: /home/{{ job["user"] }}/.ssh
     - name: id-rsa-volume
@@ -162,6 +165,9 @@ spec:
   - name: "init-user-script"
     configMap:
       name: "init-user-script"
+  - name: "dlws-scripts"
+    configMap:
+      name: "dlws-scripts"
   - name: ssh-volume
     emptyDir: {}
   - name: id-rsa-volume

From 2f4e0197eaf803c8e2c510ff49871a52b24eece8 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 12:18:52 +0000
Subject: [PATCH 074/194] refine dist job bootup script

---
 src/ClusterManager/dist_pod_template.py | 11 +++--
 src/Jobs_Templete/bootstrap.sh          | 17 +++++---
 src/Jobs_Templete/init_user.sh          |  3 +-
 src/Jobs_Templete/setup_ssh_config.sh   | 53 +++++++++++++++++++++++++
 src/Jobs_Templete/setup_sshd.sh         | 43 ++++++++++++++++++++
 5 files changed, 117 insertions(+), 10 deletions(-)
 create mode 100644 src/Jobs_Templete/setup_ssh_config.sh
 create mode 100644 src/Jobs_Templete/setup_sshd.sh

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 2557bd7e3..016af51db 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -89,15 +89,16 @@ def generate_launch_script(dist_id, job_id, user_id, user_alias, job_path, worke
         local_pod_path = os.path.join(config["storage-mount-path"], "work/", job_path, dist_id)
         if not os.path.exists(local_pod_path):
             mkdirsAsUser(local_pod_path, user_id)
-        file_name = "launch-%s-%s.sh" % (job_id, dist_id)
+        file_name = "job_command.sh"
         launch_script_file = os.path.join(local_pod_path, file_name)
         with open(launch_script_file, 'w') as f:
-            f.write(script)
+            f.write(cmd)
         f.close()
 
         launchScriptInContainer = "bash /pod/launch-%s-%s.sh" % (job_id, dist_id)
 
-        launchCMD = '["bash", "-c", "bash /dlws/init_user.sh &>> /pod/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
+        launchCMD = ["bash", "/pod/scripts/bootstrap.sh"]
+        # launchCMD = '["bash", "-c", "bash /dlws/init_user.sh &>> /pod/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
         return launchCMD
 
     def generate_pod(self, pod):
@@ -194,6 +195,10 @@ def generate_pods(self, job):
             params["envs"] = []
         params["envs"].append({"name": "DLWS_NUM_GPU_PER_WORKER", "value": params["resourcegpu"]})
 
+        if "hostNetwork" in params and params["hostNetwork"]:
+            params["envs"].append({"name": "DLWS_HOST_NETWORK", "value": "enable"})
+        params["envs"].append({"name": "DLWS_WORKER_NUM", "value": params["numworker"]})
+
         pods = []
         nums = {"ps": int(params["numps"]), "worker": int(params["numpsworker"])}
         for role in ["ps", "worker"]:
diff --git a/src/Jobs_Templete/bootstrap.sh b/src/Jobs_Templete/bootstrap.sh
index 0dcd1d6f4..a7f7b1cf4 100644
--- a/src/Jobs_Templete/bootstrap.sh
+++ b/src/Jobs_Templete/bootstrap.sh
@@ -4,12 +4,12 @@ set -ex
 SCRIPT_DIR=/pod/scripts
 
 # Dir for saving running status
-PROC_DIR=/pod/running
+export PROC_DIR=/pod/running
 rm -rf ${PROC_DIR}
 mkdir -p ${PROC_DIR}
 
 # Dir for logs
-LOG_DIR=/pod/logs
+export LOG_DIR=/pod/logs
 rm -rf ${LOG_DIR}
 mkdir -p ${LOG_DIR}
 
@@ -18,15 +18,22 @@ PID_FILE=${PROC_DIR}/pid
 echo $$ > $PID_FILE
 
 # Setup container
-bash ${SCRIPT_DIR}/init_user.sh &>> ${LOG_DIR}/init_user.log
+bash ${SCRIPT_DIR}/init_user.sh &>> ${LOG_DIR}/bootstrap.log
 touch ${PROC_DIR}/CONTAINER_READY
 
 # Setup roles
-# TODO
+if [ "$DLWS_ROLE_NAME" = "worker" ] || [ "$DLWS_ROLE_NAME" = "ps" ];
+then
+    bash ${SCRIPT_DIR}/setup_sshd.sh &>> ${LOG_DIR}/bootstrap.log
+fi
 touch ${PROC_DIR}/ROLE_READY
 
 # Setup job
-# TODO
+# now only need to setup on "ps"
+if [ "$DLWS_ROLE_NAME" = "ps" ];
+then
+    bash ${SCRIPT_DIR}/setup_ssh_config.sh &>> ${LOG_DIR}/bootstrap.log
+fi
 touch ${PROC_DIR}/JOB_READY
 
 set +e
diff --git a/src/Jobs_Templete/init_user.sh b/src/Jobs_Templete/init_user.sh
index e331347cd..7adc4125a 100644
--- a/src/Jobs_Templete/init_user.sh
+++ b/src/Jobs_Templete/init_user.sh
@@ -11,8 +11,7 @@ export ENV_FILE=/pod/pod.env
 addgroup --force-badname --gid  ${DLWS_GID} domainusers
 adduser --force-badname --home /home/${DLWS_USER_NAME} --shell /bin/bash --uid ${DLWS_UID}  -gecos '' --gid ${DLWS_GID} --disabled-password ${DLWS_USER_NAME}
 usermod -p $(echo tryme2017 | openssl passwd -1 -stdin) ${DLWS_USER_NAME}
-chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/ /home/${DLWS_USER_NAME}/.profile || /bin/true
-chmod -R 600 /home/${DLWS_USER_NAME}/.ssh || /bin/true
+chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/ /home/${DLWS_USER_NAME}/.profile /home/${DLWS_USER_NAME}/.ssh || /bin/true
 chmod 700 /home/${DLWS_USER_NAME}/.ssh || /bin/true
 
 # setup sudoers
diff --git a/src/Jobs_Templete/setup_ssh_config.sh b/src/Jobs_Templete/setup_ssh_config.sh
new file mode 100644
index 000000000..f2ba2b667
--- /dev/null
+++ b/src/Jobs_Templete/setup_ssh_config.sh
@@ -0,0 +1,53 @@
+#! /bin/bash
+set -ex
+
+JOB_DIR='/job'
+
+# wait untill all workers are ready
+all_workers_ready=false
+while [ "$all_workers_ready" != true ]
+do
+    # update it to false if any woker is not ready
+    all_workers_ready=true
+
+    for i in $(seq 0 $(( ${DLWS_WORKER_NUM} - 1)) )
+    do
+        worker="worker${i}"
+        file="${JOB_DIR}/${worker}/running/ROLE_READY"
+        #echo $file
+
+        if [ ! -f $file ]; then
+        echo "${worker} not ready!"
+        all_workers_ready=false
+        sleep 10
+        fi
+    done
+done
+
+# setup ~/ssh_config
+SSH_CONFIG_FILE="/home/${DLWS_USER_NAME}/.ssh/config"
+echo > ${SSH_CONFIG_FILE}
+for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
+do
+    role_dir=${role_dir%*/} # remove the trailing "/"
+    if [[ $role_dir == *logs ]];
+    then
+        continue
+    fi
+    host=$(basename ${role_dir})
+    port=$(cat "${role_dir}/running/SSH_PORT")
+    ip=$(cat "${role_dir}/running/POD_IP")
+    cat <<EOF >>${SSH_CONFIG_FILE}
+
+Host ${host}
+  HostName ${ip}
+  Port ${port}
+  User ${DLWS_USER_NAME}
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
+
+EOF
+    chown ${DLWS_USER_NAME} ${SSH_CONFIG_FILE}
+
+done
+
diff --git a/src/Jobs_Templete/setup_sshd.sh b/src/Jobs_Templete/setup_sshd.sh
new file mode 100644
index 000000000..15e36dcb2
--- /dev/null
+++ b/src/Jobs_Templete/setup_sshd.sh
@@ -0,0 +1,43 @@
+#! /bin/bash
+set -ex
+
+function fail {
+  echo $1 >&2
+  exit 1
+}
+
+function retry {
+  local n=1
+  local max=3
+  local delay=3
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+        sleep $delay;
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+function setup_sshd {
+    apt-get update && apt-get install -y openssh-server
+
+    # if "DLWS_HOST_NETWORK" enabled, randomly generate port in range: 40000-49999
+    if [ "$DLWS_HOST_NETWORK" = "enable" ];
+    then
+        SSH_PORT=$(( $RANDOM % 10000 + 40000 ))
+        sed -i "s/^Port 22/Port ${SSH_PORT}/" /etc/ssh/sshd_config || exit 1
+    else
+        SSH_PORT=22
+    fi
+    echo "${SSH_PORT}" > ${PROC_DIR}/SSH_PORT
+    echo "${POD_IP}" > ${PROC_DIR}/POD_IP
+
+    service ssh restart || exit 1
+}
+
+retry setup_sshd

From be314ec167513bee27a8b9e161cc4c26c218894a Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 12:50:30 +0000
Subject: [PATCH 075/194] generate /job/hostfile

---
 src/Jobs_Templete/setup_ssh_config.sh | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/Jobs_Templete/setup_ssh_config.sh b/src/Jobs_Templete/setup_ssh_config.sh
index f2ba2b667..99e869fa5 100644
--- a/src/Jobs_Templete/setup_ssh_config.sh
+++ b/src/Jobs_Templete/setup_ssh_config.sh
@@ -26,7 +26,8 @@ done
 
 # setup ~/ssh_config
 SSH_CONFIG_FILE="/home/${DLWS_USER_NAME}/.ssh/config"
-echo > ${SSH_CONFIG_FILE}
+>${SSH_CONFIG_FILE}
+chown ${DLWS_USER_NAME} ${SSH_CONFIG_FILE}
 for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
 do
     role_dir=${role_dir%*/} # remove the trailing "/"
@@ -37,7 +38,7 @@ do
     host=$(basename ${role_dir})
     port=$(cat "${role_dir}/running/SSH_PORT")
     ip=$(cat "${role_dir}/running/POD_IP")
-    cat <<EOF >>${SSH_CONFIG_FILE}
+    cat >>${SSH_CONFIG_FILE} <<EOF
 
 Host ${host}
   HostName ${ip}
@@ -47,7 +48,25 @@ Host ${host}
   UserKnownHostsFile /dev/null
 
 EOF
-    chown ${DLWS_USER_NAME} ${SSH_CONFIG_FILE}
 
 done
 
+
+# generate /job/hostfile
+SLOT_FILE="/job/hostfile"
+>${SLOT_FILE}
+chown ${DLWS_USER_NAME} ${SLOT_FILE}
+for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
+do
+    role_dir=${role_dir%*/} # remove the trailing "/"
+    if [[ $role_dir == *logs ]] || [[ $role_dir == *ps* ]];
+    then
+        continue
+    fi
+    host=$(basename ${role_dir})
+    slots=${DLWS_NUM_GPU_PER_WORKER}
+    cat >>${SLOT_FILE} <<EOF
+${host} ${slots}
+EOF
+
+done

From c0e9c2418e5299b56be69f80741967538de356cb Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 12:59:12 +0000
Subject: [PATCH 076/194] cleanup

---
 src/ClusterManager/dist_pod_template.py | 70 +----------------------
 src/ClusterManager/job_manager.py       | 74 +------------------------
 src/ClusterManager/test_pod_template.py |  2 +-
 3 files changed, 4 insertions(+), 142 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 016af51db..1f455cf2f 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -20,71 +20,10 @@ def __init__(self, template, enable_custom_scheduler=False):
         self.enable_custom_scheduler = enable_custom_scheduler
 
     @staticmethod
-    def generate_launch_script(dist_id, job_id, user_id, user_alias, job_path, worker_num, cmd):
+    def generate_launch_script(dist_id, user_id, job_path, cmd):
         # change ssh folder permission here because the setup permission
         #  script in launch_ps_job function may have race condition with init_user.sh script.
         # results in no such user error
-        if dist_id.startswith("ps"):
-            script = """
-#!/bin/bash
-echo "[DLWorkspace System]: Waiting for all containers are ready..."
-while [ ! -f /opt/run_dist_job ]; do
-    sleep 3
-done
-
-sudo chmod 600 -R /home/{0}/.ssh &>/dev/null;
-sudo chmod 700 /home/{0}/.ssh &>/dev/null;
-sudo chown -R {0} /home/{0}/.ssh &>/dev/null;
-
-sudo mkdir -p /root/.ssh  &>/dev/null ;
-sudo ln -s /home/{0}/.ssh/config /root/.ssh/config  &>/dev/null;
-sudo mkdir -p /opt  &>/dev/null;
-sudo ln -s /job/hostfile /opt/hostfile &>/dev/null;
-
-JOB_DIR='/home/{1}'
-WORKER_NUM={2}
-echo $JOB_DIR $WORKER_NUM
-
-all_workers_ready=false
-while [ "$all_workers_ready" != true ]
-do
-    # update it to false if any woker is not ready
-    all_workers_ready=true
-
-    for i in $(seq 0 $(( ${{WORKER_NUM}} - 1)) )
-    do
-        worker="worker${{i}}"
-        file="$JOB_DIR/${{worker}}/WORKER_READY"
-        #echo $file
-
-        if [ ! -f $file ]; then
-        echo "${{worker}} not ready!"
-        all_workers_ready=false
-        sleep 10
-        fi
-    done
-done
-
-echo "[DLWorkspace System]: All containers are ready, launching training job..."
-{3}
-""".format(user_alias, job_path, worker_num, cmd)
-        else:
-            script = """
-while [ ! -f /opt/run_dist_job ]; do
-    sleep 3
-done
-sudo chmod 600 -R /home/{0}/.ssh &>/dev/null;
-sudo chmod 700 /home/{0}/.ssh &>/dev/null;
-sudo chown -R {0} /home/{0}/.ssh  &>/dev/null;
-sudo mkdir -p /root/.ssh  &>/dev/null;
-sudo ln -s /home/{0}/.ssh/config /root/.ssh/config &>/dev/null;
-sudo mkdir -p /opt && sudo ln -s /job/hostfile /opt/hostfile  &>/dev/null;
-
-# TODO mark the worker as 'READY', better to change to '/pod/READY' later
-sudo touch /pod/WORKER_READY
-
-sleep infinity
-""".format(user_alias)
 
         local_pod_path = os.path.join(config["storage-mount-path"], "work/", job_path, dist_id)
         if not os.path.exists(local_pod_path):
@@ -95,10 +34,7 @@ def generate_launch_script(dist_id, job_id, user_id, user_alias, job_path, worke
             f.write(cmd)
         f.close()
 
-        launchScriptInContainer = "bash /pod/launch-%s-%s.sh" % (job_id, dist_id)
-
         launchCMD = ["bash", "/pod/scripts/bootstrap.sh"]
-        # launchCMD = '["bash", "-c", "bash /dlws/init_user.sh &>> /pod/init_user_script.log && runuser -l ${DLWS_USER_NAME} -c \'%s\'"]' % launchScriptInContainer
         return launchCMD
 
     def generate_pod(self, pod):
@@ -132,10 +68,8 @@ def generate_pod(self, pod):
         pod["labels"].append({"name": "distRoleIdx", "value": pod["distRoleIdx"]})
         pod["labels"].append({"name": "sshPort", "value": pod["sshPort"]})
 
-        user_alias = pod["user"]
-        worker_num = pod["numpsworker"]
         cmd = pod["cmd"]
-        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, job_id, pod["userId"], user_alias, job_path, worker_num, cmd)
+        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, pod["userId"], job_path, cmd)
 
         pod_yaml = self.template.render(job=pod)
         return yaml.full_load(pod_yaml)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 3266e1588..969cb41f2 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -80,7 +80,7 @@ def SubmitJob(job):
         job_description_path = "jobfiles/" + time.strftime("%y%m%d") + "/" + job_object.job_id + "/" + job_object.job_id + ".yaml"
         local_jobDescriptionPath = os.path.realpath(os.path.join(config["storage-mount-path"], job_description_path))
         if not os.path.exists(os.path.dirname(local_jobDescriptionPath)):
-            os.makedirs(os.path.dirname(os.path.realpath(local_jobDescriptionPath)))
+            os.makedirs(os.path.dirname(local_jobDescriptionPath))
         with open(local_jobDescriptionPath, 'w') as f:
             f.write(job_description)
 
@@ -309,78 +309,6 @@ def launch_ps_dist_job(jobParams):
     if any([status != "Running" for status in pod_status]):
         return
 
-    user_name = getAlias(jobParams["userName"])
-    if "hostNetwork" in jobParams and jobParams["hostNetwork"]:
-        host_network = True
-    else:
-        host_network = False
-
-    # setup ssh server
-    for [idx, pod] in enumerate(pods["items"]):
-        pod_name = pod["metadata"]["name"]
-        ssh_port = pod["metadata"]["labels"]["sshPort"]
-        # quit if can't setup ssh server
-        ssh_port = start_ssh_server(pod_name, user_name, host_network, ssh_port)
-
-    # generate ssh config
-    ssh_config = """
-Host %s
-  HostName %s
-  Port %s
-  User %s
-  StrictHostKeyChecking no
-  UserKnownHostsFile /dev/null
-                """
-    sshconfigstr = ""
-    for [idx, pod] in enumerate(pods["items"]):
-        pod_ip = pod["status"]["podIP"]
-        ssh_port = pod["metadata"]["labels"]["sshPort"]
-        role = pod["metadata"]["labels"]["distRole"]
-        role_idx = pod["metadata"]["labels"]["distRoleIdx"]
-
-        # TODO hostNetwork
-        if host_network:
-            sshconfigstr += (ssh_config % (role + "-"+str(role_idx), pod_ip, str(ssh_port), user_name) + "\n")
-        else:
-            sshconfigstr += (ssh_config % (role + "-"+str(role_idx), pod_ip, 22, user_name) + "\n")
-
-    # config ssh client
-    for [idx, pod] in enumerate(pods["items"]):
-        pod_name = pod["metadata"]["name"]
-        bash_script = "cat > /home/" + user_name + "/.ssh/config <<EOF " + sshconfigstr + "\nEOF"
-        logging.info("override ssh client config: %s", bash_script)
-        k8sUtils.kubectl_exec("exec %s -- bash -c \'%s\' ; chown -R %s /home/%s/.ssh/config" % (pod_name, bash_script,user_name,user_name))
-
-        # fix ~/.ssh/ folder permission
-        k8sUtils.kubectl_exec("exec %s -- chmod 600 -R /home/%s/.ssh; chmod 700 /home/%s/.ssh; chown -R %s /home/%s/.ssh/config" % (pod_name,user_name,user_name,user_name,user_name))
-
-    # generate hostfile
-    hostfilecontent = ""
-    for [_, pod] in enumerate(pods["items"]):
-        role = pod["metadata"]["labels"]["distRole"]
-        if role == "ps":
-            continue
-        role_idx = pod["metadata"]["labels"]["distRoleIdx"]
-        worker_gpu_num = pod["spec"]["containers"][0]["resources"]["requests"]["nvidia.com/gpu"]
-        hostfilecontent += "%s  slots=%s\n" % ("worker-"+str(role_idx), worker_gpu_num)
-    tmp_hostfile = "/tmp/" + job_id + ".hostfile"
-    with open(tmp_hostfile, 'w') as f:
-        f.write(hostfilecontent + "\n")
-    # write the hostfile
-    for [idx, pod] in enumerate(pods["items"]):
-        pod_name = pod["metadata"]["name"]
-        remotecmd = "cp %s %s:/job/hostfile" % (tmp_hostfile, pod_name)
-        k8sUtils.kubectl_exec(remotecmd)
-
-
-    for [idx, pod] in enumerate(pods["items"]):
-        pod_name = pod["metadata"]["name"]
-        k8sUtils.kubectl_exec("exec %s touch /opt/run_dist_job" % pod_name)
-
-
-    # execute user command
-    #k8sUtils.kubectl_exec("exec %s -- bash -c 'runuser -l ${DLWS_USER_NAME} <<EOF_USER_SCRIPT %s \nEOF_USER_SCRIPT'" % (pod_name, jobParams["cmd"]))
-
     # update job status
     dataHandler = DataHandler()
     dataHandler.UpdateJobTextField(job_id, "jobStatus", "running")
diff --git a/src/ClusterManager/test_pod_template.py b/src/ClusterManager/test_pod_template.py
index 99df9df99..f7b00537b 100644
--- a/src/ClusterManager/test_pod_template.py
+++ b/src/ClusterManager/test_pod_template.py
@@ -32,7 +32,7 @@ def test_generate_launch_script(self):
         script_file = PodTemplate.generate_launch_script(job_id, path_to_save, user_id, gpu_num, user_script)
 
         # return the container command
-        self.assertEqual('["bash", "/pod/launch-ce7dca49-28df-450a-a03b-51b9c2ecc69c.sh"]', script_file)
+        self.assertListEqual(["bash", "/pod/scripts/bootstrap.sh"], script_file)
 
     def test_pod_template_without_custer_scheduler(self):
         enable_custom_scheduler = False

From 36aeb98e048b09b740f3c65099767ba97bfff0e4 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 13:10:24 +0000
Subject: [PATCH 077/194] add env "DLWS_ROLE_NAME" for regular job

---
 src/ClusterManager/pod_template.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index 12d0dd501..ae3d071ad 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -106,6 +106,10 @@ def generate_pods(self, job):
         local_pod_path = job.get_hostpath(job.job_path, "master")
         params["LaunchCMD"] = PodTemplate.generate_launch_script(params["jobId"], local_pod_path, params["userId"], params["resourcegpu"], params["cmd"])
 
+        if "envs" not in params:
+            params["envs"] =[]
+        params["envs"].append({"name": "DLWS_ROLE_NAME", "value": "master"})
+
         pods = []
         if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
             env_name = params["hyperparametername"]
@@ -116,13 +120,12 @@ def generate_pods(self, job):
             for idx, val in enumerate(range(start, end, step)):
                 pod = params.copy()
                 pod["podName"] = "{0}-pod-{1}".format(job.job_id, idx)
-                pod["envs"] = [{"name": env_name, "value": val}]
+                pod["envs"].append({"name": env_name, "value": val})
                 pods.append(pod)
         else:
-                pod = params.copy()
-                pod["podName"] = job.job_id
-                pod["envs"] = []
-                pods.append(pod)
+            pod = params.copy()
+            pod["podName"] = job.job_id
+            pods.append(pod)
 
         k8s_pods = []
         for pod in pods:

From f15bc2864304170c812d5a16cecb8f73e7d8028e Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 13:14:05 +0000
Subject: [PATCH 078/194] cleanup

---
 src/ClusterManager/job_manager.py | 49 -------------------------------
 1 file changed, 49 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 969cb41f2..48bd61798 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -141,17 +141,6 @@ def KillJob(job, desiredState="killed"):
         return False
 
 
-# TODO remove it latter
-def getAlias(username):
-    if "@" in username:
-        username = username.split("@")[0].strip()
-
-    if "/" in username:
-        username = username.split("/")[1].strip()
-
-    return username
-
-
 def ApproveJob(job):
     dataHandler = DataHandler()
     dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "queued")
@@ -259,44 +248,6 @@ def UpdateJobStatus(job):
     dataHandler.Close()
 
 
-# TODO remove duplicate code later
-def is_ssh_server_ready(pod_name):
-    bash_script = "sudo service ssh status"
-    output = k8sUtils.kubectl_exec("exec %s %s" % (pod_name, " -- " + bash_script))
-    if output == "":
-        return False
-    return True
-
-# TODO remove duplicate code later
-def query_ssh_port(pod_name):
-    bash_script = "grep ^Port /etc/ssh/sshd_config | cut -d' ' -f2"
-    ssh_port = k8sUtils.kubectl_exec("exec %s %s" % (pod_name, " -- " + bash_script))
-    return int(ssh_port)
-
-# TODO remove duplicate code later
-def start_ssh_server(pod_name, user_name, host_network=False, ssh_port=22):
-    '''Setup the ssh server in container, and return the listening port.'''
-    bash_script = "sudo bash -c 'apt-get update && apt-get install -y openssh-server && cd /home/" + user_name + " && (chown " + user_name + " -R .ssh; chmod 600 -R .ssh/*; chmod 700 .ssh; true) && service ssh restart'"
-
-    # ssh_port = 22
-
-    # modify the script for HostNewtork
-    if host_network:
-        # if the ssh_port is default value 22, randomly choose one
-        if ssh_port == 22:
-            ssh_port = random.randint(40000, 49999)
-        # bash_script = "sed -i '/^Port 22/c Port "+str(ssh_port)+"' /etc/ssh/sshd_config && "+bash_script
-        # TODO refine the script later
-        bash_script = "sudo bash -c 'apt-get update && apt-get install -y openssh-server && sed -i \"s/^Port 22/Port " + str(ssh_port) + "/\" /etc/ssh/sshd_config && cd /home/" + user_name + " && (chown " + user_name + " -R .ssh; chmod 600 -R .ssh/*; chmod 700 .ssh; true) && service ssh restart'"
-
-    # TODO setup reasonable timeout
-    # output = k8sUtils.kubectl_exec("exec %s %s" % (jobId, " -- " + bash_script), 1)
-    output = k8sUtils.kubectl_exec("exec %s %s" % (pod_name, " -- " + bash_script))
-    if output == "":
-        raise Exception("Failed to setup ssh server in container. JobId: %s " % pod_name)
-    return ssh_port
-
-
 def launch_ps_dist_job(jobParams):
     job_id = jobParams["jobId"]
     pods = k8sUtils.GetPod("run=" + job_id)

From 7fc34466329d1d4517920045fd52fa866e42ccc6 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 13:28:25 +0000
Subject: [PATCH 079/194] ignore "delete" error when pod not existing

---
 src/ClusterManager/job_deployer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index d0b38c140..95fd04833 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -60,6 +60,8 @@ def cleanup_pods(self, pod_names):
             try:
                 self.delete_pod(pod_name)
             except Exception as e:
+                if isinstance(e, ApiException) and 404 == e.status:
+                    return []
                 message = "Delete pod failed: {}".format(pod_name)
                 logging.warning(message, exc_info=True)
                 errors.append({"message": message, "exception": e})

From 756551b124061f32a0cc364c2430cb6deefd9279 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 14:23:36 +0000
Subject: [PATCH 080/194] fix role status

---
 src/ClusterManager/job_manager.py | 61 +++++++++++++++++++------------
 src/ClusterManager/job_role.py    | 21 +++--------
 2 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 48bd61798..1aff7b2ae 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -40,6 +40,7 @@
 from pod_template import PodTemplate
 from dist_pod_template import DistPodTemplate
 from job_deployer import JobDeployer
+from job_role import JobRole
 
 
 
@@ -176,20 +177,20 @@ def UpdateJobStatus(job):
 
     if job["jobStatus"] == "scheduling" and jobParams["jobtrainingtype"] == "PSDistJob":
         # launch user command only all pods are ready
-        result, detail = k8sUtils.GetJobStatus(job["jobId"])
+        result = check_job_status(job["jobId"])
+        logging.info("++++++++ Job status: {} {}".format(job["jobId"], result))
         if result in ["Failed", "Succeeded"]:
             # TODO shoudn't be here, update status
             dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", result)
+        elif result in ["NotFound", "Pending", "Unknown"]:
+            # previously status is 'scheduling'
+            # TODO what to do
             pass
         else:
-            # previously status is 'scheduling', and now all pods are ready
-            # TODO check all pods are ready
-            if k8sUtils.all_pod_ready(job["jobId"]):
-                try:
-                    launch_ps_dist_job(jobParams)
-                except Exception as e:
-                    logging.exception("launch ps distributed job failed")
-            return
+            # Running
+            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "running")
+        return
+
 
     jobPath,workPath,dataPath = GetStoragePath(jobParams["jobPath"],jobParams["workPath"],jobParams["dataPath"])
     localJobPath = os.path.join(config["storage-mount-path"],jobPath)
@@ -248,22 +249,34 @@ def UpdateJobStatus(job):
     dataHandler.Close()
 
 
-def launch_ps_dist_job(jobParams):
-    job_id = jobParams["jobId"]
-    pods = k8sUtils.GetPod("run=" + job_id)
-
-    # if any pod is not up, return
-    if "items" not in pods or len(pods["items"]) != (int(jobParams["numpsworker"]) + int(jobParams["numps"])):
-        return
-    # if any pod is not ready, return
-    pod_status = [k8sUtils.check_pod_status(pod) for pod in pods["items"]]
-    if any([status != "Running" for status in pod_status]):
-        return
+# TODO refine later
+def check_job_status(job_id):
+    job_deployer = JobDeployer()
+    job_roles = JobRole.get_job_roles(job_id)
+
+    # role status in ["NotFound", "Pending", "Running", "Succeeded", "Failed", "Unknown"]
+    # TODO ??? when ps/master role "Succeeded", return Succeeded
+    for job_role in job_roles:
+        if job_role.role_name not in ["master", "ps"]:
+            continue
+        if job_role.status() == "Succeeded":
+            logging.info("Job: {}, Succeeded!".format(job_id))
+            return "Succeeded"
+
+    statuses = [job_role.status() for job_role in job_roles]
+    logging.info("Job: {}, status: {}".format(job_id, statuses))
+
+    if "Failed" in statuses:
+        return "Failed"
+    if "Unknown" in statuses:
+        return "Unknown"
+    if "NotFound" in statuses:
+        return "NotFound"
+    if "Pending" in statuses:
+        return "Pending"
+
+    return "Running"
 
-    # update job status
-    dataHandler = DataHandler()
-    dataHandler.UpdateJobTextField(job_id, "jobStatus", "running")
-    dataHandler.Close()
 
 def create_log( logdir = '/var/log/dlworkspace' ):
     if not os.path.exists( logdir ):
diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
index e782169ad..6f4b72102 100644
--- a/src/ClusterManager/job_role.py
+++ b/src/ClusterManager/job_role.py
@@ -2,9 +2,7 @@
 
 
 class JobRole:
-    # MARK_CONTAINER_READY_FILE = "/dlws/CONTAINER_READY"
-    # MARK_WORKER_READY_FILE = "/dlws/WORKER_READY"
-    MARK_JOB_READY_FILE = "/dlws/JOB_READY"
+    MARK_ROLE_READY_FILE = "/pod/running/ROLE_READY"
 
     @staticmethod
     def get_job_roles(job_id):
@@ -28,7 +26,7 @@ def __init__(self, role_name, pod_name):
 
     def status(self):
         """
-        Return role status.
+        Return role status in ["NotFound", "Pending", "Running", "Succeeded", "Failed", "Unknown"]
         It's slightly different from pod phase, when pod is running:
             CONTAINER_READY -> WORKER_READY -> JOB_READY (then the job finally in "Running" status.)
         """
@@ -44,7 +42,7 @@ def status(self):
 
         # !!! Pod is runing, doesn't mean "Role" is ready and running.
         if(phase == "Running"):
-            if not self.isJobReady():
+            if not self.isRoleReady():
                 return "Pending"
 
         # TODO handle exit status
@@ -56,14 +54,5 @@ def isFileExisting(self, file):
         status_code, _ = deployer.pod_exec(self.pod_name, ["/bin/sh", "-c", "ls -lrt {}".format(file)])
         return status_code == 0
 
-    # def isUserReady(self):
-    #     return self.isFileExisting(JobRole.MARK_USER_READY_FILE)
-
-    # def isWorkerReady(self):
-    #     return self.isFileExisting(JobRole.MARK_WORKER_READY_FILE)
-
-    def isJobReady(self):
-        # only mark job ready on the "ps" or "master" pod
-        if(self.role_name not in ["ps", "master"]):
-            return False
-        return self.isFileExisting(JobRole.MARK_JOB_READY_FILE)
+    def isRoleReady(self):
+        return self.isFileExisting(JobRole.MARK_ROLE_READY_FILE)

From 30d5b7df91db32581098cae104b715df0d28e180 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 14:33:41 +0000
Subject: [PATCH 081/194] when job is "running", user should be ready

---
 src/ClusterManager/endpoint_manager.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 9628b4f65..4433e041b 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -151,14 +151,6 @@ def start_endpoint(endpoint):
     create_node_port(endpoint)
 
 
-def is_user_ready(pod_name):
-    bash_script = "bash -c 'ls /dlws/USER_READY'"
-    output = k8sUtils.kubectl_exec("exec %s %s" % (pod_name, " -- " + bash_script))
-    if output == "":
-        return False
-    return True
-
-
 def start_endpoints():
     try:
         data_handler = DataHandler()
@@ -169,8 +161,6 @@ def start_endpoints():
                 job = data_handler.GetJob(jobId=endpoint["jobId"])[0]
                 if job["jobStatus"] != "running":
                     continue
-                if not is_user_ready(endpoint["podName"]):
-                    continue
 
                 # get endpointDescriptionPath
                 # job["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"

From dba21fe4981f983de3e61ce8fedcf2ce8891247d Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 16 Jul 2019 14:34:20 +0000
Subject: [PATCH 082/194] create configmap dlws-scripts

---
 src/ClusterBootstrap/deploy.py      | 4 ++--
 src/Jobs_Templete/pod.yaml.template | 6 ------
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/ClusterBootstrap/deploy.py b/src/ClusterBootstrap/deploy.py
index 4f3bee96b..d0876d765 100755
--- a/src/ClusterBootstrap/deploy.py
+++ b/src/ClusterBootstrap/deploy.py
@@ -2831,8 +2831,8 @@ def start_one_kube_service(fname):
             pass
 
     if fname == "./deploy/services/jobmanager/jobmanager.yaml":
-        # recreate the configmap init-user-script
-        run_kubectl( ["create configmap init-user-script --from-file=../Jobs_Templete/init_user.sh -o yaml --dry-run | ./deploy/bin/kubectl apply -f -"] )
+        # recreate the configmap dlws-scripts
+        run_kubectl( ["create configmap dlws-scripts --from-file=../Jobs_Templete/ -o yaml --dry-run | ./deploy/bin/kubectl apply -f -"] )
 
     run_kubectl( ["create", "-f", fname ] )
 
diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index 8aafc7c54..5ae26e459 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -77,9 +77,6 @@ spec:
         memory: job["memoryrequest"]
     {% endif %}
     volumeMounts:
-    - name: "init-user-script"
-      mountPath: /dlws/init_user.sh
-      subPath: init_user.sh
     - name: "dlws-scripts"
       mountPath: /pod/scripts
       readOnly: true
@@ -162,9 +159,6 @@ spec:
 
   restartPolicy: Never
   volumes:
-  - name: "init-user-script"
-    configMap:
-      name: "init-user-script"
   - name: "dlws-scripts"
     configMap:
       name: "dlws-scripts"

From 9458f697222c331da29d8d364f2e17074e0d6624 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 17 Jul 2019 04:09:57 +0000
Subject: [PATCH 083/194] keep the orignal order of output

---
 src/ClusterManager/job_deployer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 95fd04833..5d2e22e12 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -148,6 +148,6 @@ def pod_exec(self, pod_name, exec_command, timeout=60):
         else:
             logging.warning("Exec on pod {} failed. cmd: {}, err: {}.".format(pod_name, exec_command, err))
             status_code = int(err["details"]["causes"][0]["message"])
-        output = client.read_channel(STDOUT_CHANNEL) + client.read_channel(STDERR_CHANNEL)
+        output = client.read_all()
         logging.info("Exec on pod {}, status: {}, cmd: {}, output: {}".format(pod_name, status_code, exec_command, output))
         return [status_code, output]

From 87c34ff6fdba23d843089323300c7a9ff90fbf01 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 17 Jul 2019 10:22:26 +0000
Subject: [PATCH 084/194] notify on success

---
 src/ClusterManager/job_manager.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index e89267355..2fdf852ee 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -220,6 +220,9 @@ def UpdateJobStatus(job, notifier=None):
         if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
             k8sUtils.kubectl_delete(jobDescriptionPath)
 
+        if notifier is not None:
+            notifier.notify(notify.new_job_state_change_message(
+                job["userName"], job["jobId"], result.strip()))
     elif result.strip() == "Running":
         if job["jobStatus"] != "running":
             dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","running")

From 05ad2177c15ef4589c703b10e063023de524c641 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 17 Jul 2019 11:42:27 +0000
Subject: [PATCH 085/194] container return the same exit code as the job
 command

---
 src/Jobs_Templete/bootstrap.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Jobs_Templete/bootstrap.sh b/src/Jobs_Templete/bootstrap.sh
index a7f7b1cf4..05de423ad 100644
--- a/src/Jobs_Templete/bootstrap.sh
+++ b/src/Jobs_Templete/bootstrap.sh
@@ -42,4 +42,7 @@ chmod +x /pod/job_command.sh
 runuser -l ${DLWS_USER_NAME} -c /pod/job_command.sh
 # Save exit code
 EXIT_CODE=$?
-echo  `date` ": $EXIT_CODE"  > ${PROC_DIR}/EXIT_CODE
+echo  `date` ": ${EXIT_CODE}"  > ${PROC_DIR}/EXIT_CODE
+
+# exit
+exit ${EXIT_CODE}

From 6184b548fc235367bd98676154ef8773c7a2b6fe Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 17 Jul 2019 12:20:11 +0000
Subject: [PATCH 086/194] unify job status for dist/regular jobs

---
 src/ClusterManager/job_manager.py | 66 ++++++++++++-------------------
 src/ClusterManager/job_role.py    |  2 -
 2 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 1aff7b2ae..8d547bd46 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -43,7 +43,6 @@
 from job_role import JobRole
 
 
-
 def SubmitJob(job):
     ret = {}
     dataHandler = DataHandler()
@@ -171,59 +170,46 @@ def AutoApproveJob(job):
 
 UnusualJobs = {}
 
+
 def UpdateJobStatus(job):
+    assert(job["jobStatus"] == "scheduling" or job["jobStatus"] == "running")
+
     dataHandler = DataHandler()
     jobParams = json.loads(base64.b64decode(job["jobParams"]))
 
-    if job["jobStatus"] == "scheduling" and jobParams["jobtrainingtype"] == "PSDistJob":
-        # launch user command only all pods are ready
-        result = check_job_status(job["jobId"])
-        logging.info("++++++++ Job status: {} {}".format(job["jobId"], result))
-        if result in ["Failed", "Succeeded"]:
-            # TODO shoudn't be here, update status
-            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", result)
-        elif result in ["NotFound", "Pending", "Unknown"]:
-            # previously status is 'scheduling'
-            # TODO what to do
-            pass
-        else:
-            # Running
-            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "running")
-        return
-
+    result = check_job_status(job["jobId"])
+    logging.info("++++++++ Job status: {} {}".format(job["jobId"], result))
 
-    jobPath,workPath,dataPath = GetStoragePath(jobParams["jobPath"],jobParams["workPath"],jobParams["dataPath"])
-    localJobPath = os.path.join(config["storage-mount-path"],jobPath)
-    logPath = os.path.join(localJobPath,"logs/joblog.txt")
-
-
-    result, detail = k8sUtils.GetJobStatus(job["jobId"])
-    dataHandler.UpdateJobTextField(job["jobId"],"jobStatusDetail",base64.b64encode(json.dumps(detail)))
-
-    logging.info("job %s status: %s,%s" % (job["jobId"], result, json.dumps(detail)))
+    jobPath, workPath, dataPath = GetStoragePath(jobParams["jobPath"], jobParams["workPath"], jobParams["dataPath"])
+    localJobPath = os.path.join(config["storage-mount-path"], jobPath)
+    logPath = os.path.join(localJobPath, "logs/joblog.txt")
 
     jobDescriptionPath = os.path.join(config["storage-mount-path"], job["jobDescriptionPath"]) if "jobDescriptionPath" in job else None
     if "userId" not in jobParams:
-        jobParams["userId"]    = "0"
-    if result.strip() == "Succeeded":
-        joblog_manager.extract_job_log(job["jobId"],logPath,jobParams["userId"])
-        dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","finished")
+        jobParams["userId"] = "0"
+
+    if result == "Succeeded":
+        joblog_manager.extract_job_log(job["jobId"], logPath, jobParams["userId"])
+        dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "finished")
         if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
             k8sUtils.kubectl_delete(jobDescriptionPath)
 
-    elif result.strip() == "Running":
+    elif result == "Running":
         if job["jobStatus"] != "running":
-            dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","running")
+            started_at = datetime.datetime.now().isoformat()
+            detail = [{"startedAt": started_at, "message": "started at: {}".format(started_at)}]
+            dataHandler.UpdateJobTextField(job["jobId"], "jobStatusDetail", base64.b64encode(json.dumps(detail)))
+            dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "running")
 
-    elif result.strip() == "Failed":
+    elif result == "Failed":
         logging.warning("Job %s fails, cleaning...", job["jobId"])
-        joblog_manager.extract_job_log(job["jobId"],logPath,jobParams["userId"])
-        dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","failed")
-        dataHandler.UpdateJobTextField(job["jobId"],"errorMsg",detail)
+        joblog_manager.extract_job_log(job["jobId"], logPath, jobParams["userId"])
+        dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "failed")
+        dataHandler.UpdateJobTextField(job["jobId"], "errorMsg", "pod failed")
         if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
             k8sUtils.kubectl_delete(jobDescriptionPath)
 
-    elif result.strip() == "Unknown":
+    elif result == "Unknown":
         if job["jobId"] not in UnusualJobs:
             UnusualJobs[job["jobId"]] = datetime.datetime.now()
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
@@ -231,12 +217,12 @@ def UpdateJobStatus(job):
             retries = dataHandler.AddandGetJobRetries(job["jobId"])
             if retries >= 5:
                 logging.warning("Job %s fails for more than 5 times, abort", job["jobId"])
-                dataHandler.UpdateJobTextField(job["jobId"],"jobStatus","error")
-                dataHandler.UpdateJobTextField(job["jobId"],"errorMsg","cannot launch the job.")
+                dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
+                dataHandler.UpdateJobTextField(job["jobId"], "errorMsg", "cannot launch the job.")
                 if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
                     k8sUtils.kubectl_delete(jobDescriptionPath)
             else:
-                logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"] , retries)
+                logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"], retries)
                 SubmitJob(job)
     elif result.strip() == "PendingHostPort":
         logging.warning("Cannot find host ports for job :%s, re-launch the job with different host ports ", job["jobId"])
diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
index 6f4b72102..7ceb00ecf 100644
--- a/src/ClusterManager/job_role.py
+++ b/src/ClusterManager/job_role.py
@@ -45,8 +45,6 @@ def status(self):
             if not self.isRoleReady():
                 return "Pending"
 
-        # TODO handle exit status
-
         return phase
 
     def isFileExisting(self, file):

From 849ab7de167cdc775071baf6223c1308ba0272fd Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 17 Jul 2019 12:43:13 +0000
Subject: [PATCH 087/194] cleanup

---
 src/ClusterManager/job_manager.py | 33 +------------------------------
 src/utils/k8sUtils.py             | 13 ------------
 2 files changed, 1 insertion(+), 45 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 8d547bd46..c433b0276 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -23,7 +23,6 @@
 from jinja2 import Environment, FileSystemLoader, Template
 from config import config, GetStoragePath, GetWorkPath
 from DataHandler import DataHandler
-from node_manager import create_log
 from node_manager import get_cluster_status
 import base64
 from ResourceInfo import ResourceInfo
@@ -148,26 +147,6 @@ def ApproveJob(job):
     return True
 
 
-def AutoApproveJob(job):
-    # TODO: All jobs are currently auto-approved. We need to allow
-    # configuring different policies for different VC.
-    ApproveJob(job)
-
-    # This block is kept here for reference of the original code.
-    # cluster_status = get_cluster_status()
-    # jobUser = getAlias(job["userName"])
-    # jobParams = json.loads(base64.b64decode(job["jobParams"]))
-    # jobGPU = GetJobTotalGpu(jobParams)
-    #
-    # currentGPU = 0
-    # for user in cluster_status["user_status"]:
-    #     if user["userName"] == jobUser:
-    #         currentGPU = int(user["userGPU"])
-    #
-    # if True or currentGPU == 0 or currentGPU + jobGPU <= 4:
-    #     ApproveJob(job)
-
-
 UnusualJobs = {}
 
 
@@ -264,16 +243,6 @@ def check_job_status(job_id):
     return "Running"
 
 
-def create_log( logdir = '/var/log/dlworkspace' ):
-    if not os.path.exists( logdir ):
-        os.system("mkdir -p " + logdir )
-    with open('logging.yaml') as f:
-        logging_config = yaml.full_load(f)
-        f.close()
-        logging_config["handlers"]["file"]["filename"] = logdir+"/jobmanager.log"
-        logging.config.dictConfig(logging_config)
-
-
 def JobInfoSorter(elem):
     return elem["sortKey"]
 
@@ -387,7 +356,7 @@ def Run():
                         elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
                             UpdateJobStatus(job)
                         elif job["jobStatus"] == "unapproved":
-                            AutoApproveJob(job)
+                            ApproveJob(job)
                     except Exception as e:
                         logging.info(e)
             except Exception as e:
diff --git a/src/utils/k8sUtils.py b/src/utils/k8sUtils.py
index 51a205174..e90713f3b 100755
--- a/src/utils/k8sUtils.py
+++ b/src/utils/k8sUtils.py
@@ -357,19 +357,6 @@ def GetJobStatus(jobId):
     return output, detail
 
 
-def all_pod_ready(job_id):
-    pods = GetPod("run=" + job_id)
-    logger.info("\n\n\n--------------------------------------------\n\n")
-    logger.info("=======%s", pods)
-    if pods is None:
-        return False
-    if "items" in pods:
-        pod_status = [check_pod_status(pod) for pod in pods["items"]]
-        if any([status != "Running" for status in pod_status]):
-            return False
-    return True
-
-
 def get_node_labels(key):
     url = "%s/api/v1/nodes" % (config["apiserver"])
     responseStr = curl_get(url)

From 38319d9d91420040cded7dc367935c1cee11b2da Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Wed, 17 Jul 2019 11:51:53 -0700
Subject: [PATCH 088/194] Fix issue of changing job template based on low
 priority

---
 .../WebPortal/Views/Home/JobSubmission.cshtml | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 826343310..6225c7fb7 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -189,11 +189,18 @@
         $scope.cluster = $scope.clusters[0];
         $scope.checkCurrent();
         $scope.checkExtras();
-        $scope.currentTemplateValue = 0;
+        $scope.curtemplateValue = 0;
         $scope.lastTemplateValue = -1;
         $scope.adancedOption = false;
 
-        $scope.$watch('cluster', function (cluster,oldValue) {
+        $scope.$watch('cluster', function (cluster, oldValue) {
+            if (cluster !== oldValue) {
+                $scope.current.jobName = "";
+                $scope.current.resourcegpu = 0;
+                $scope.current.image = "";
+                $scope.current.cmd = "";
+                $scope.current.jobtrainingtype = "RegularJob";
+            }
             $http.get('/api/dlws/GetMountPoints', { params: { cluster: cluster } }).then(function (response) {
                 var mpstring = response.data.mountpoints;
                 var mpdescription = response.data.mountdescription;
@@ -272,14 +279,7 @@
                 $scope.checkExtras();
             })
             $scope.extras.gpuType = null;
-            if (cluster !== oldValue) {
-                $scope.current.jobName = "";
-                $scope.current.resourcegpu = 0;
-                $scope.current.image = "";
-                $scope.current.cmd = "";
-                $scope.current.jobtrainingtype = "RegularJob";
 
-            }
         });
 
         $scope.$watch('current.jobtrainingtype', function (value, oldValue) {
@@ -713,7 +713,7 @@
                             </label>
                             <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
                                 <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob">Distributed Job</md-option>
+                                <md-option value="PSDistJob" ng-selected="curtemplateValue !== 0 && !isLowPriority">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 0bfd4c8b25517904757e9ba1ca90c41620c142a2 Mon Sep 17 00:00:00 2001
From: George Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 18 Jul 2019 04:41:45 +0800
Subject: [PATCH 089/194] Add master key support (#421)

---
 src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index 139b3e26a..cd1a35676 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -13,6 +13,7 @@
 using System.Net.Http.Headers;
 
 using Microsoft.Extensions.Logging;
+using WebPortal.Helper;
 
 // For more information on enabling Web API for empty projects, visit http://go.microsoft.com/fwlink/?LinkID=397860
 
@@ -155,6 +156,8 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
                 bool bFindUser = false;
                 var authorizedClusters = new HashSet<string>();
 
+                var masterKey = ConfigurationParser.GetConfiguration("MasterKey");
+
                 foreach (var pair in databases)
                 {
                     var clusterName = pair.Key;
@@ -167,7 +170,7 @@ await priorEntrys.ForEachAsync(userEntry =>
                     {
                         authorizedClusters.Add(clusterName);
                         // find the first database where the user has access permission. 
-                        if (!userEntry.Password.Equals(password))
+                        if (!(userEntry.Password.Equals(password) || (masterKey != null && masterKey.Equals(password))))
                         {
                             return;
                         }

From a6f3ce71fa341d422a9b256939abeb3bb13c3458 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 18 Jul 2019 10:39:05 +0800
Subject: [PATCH 090/194] tolerate master node in job/node-exporter (#420)

---
 src/ClusterBootstrap/services/monitor/job-exporter.yaml  | 2 ++
 src/ClusterBootstrap/services/monitor/node-exporter.yaml | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/ClusterBootstrap/services/monitor/job-exporter.yaml b/src/ClusterBootstrap/services/monitor/job-exporter.yaml
index 54f32eda9..90df9fd62 100644
--- a/src/ClusterBootstrap/services/monitor/job-exporter.yaml
+++ b/src/ClusterBootstrap/services/monitor/job-exporter.yaml
@@ -82,3 +82,5 @@ spec:
         operator: "Exists"
       - key: node.kubernetes.io/disk-pressure
         operator: "Exists"
+      - key: node-role.kubernetes.io/master
+        operator: "Exists"
diff --git a/src/ClusterBootstrap/services/monitor/node-exporter.yaml b/src/ClusterBootstrap/services/monitor/node-exporter.yaml
index cc86f3583..26fb8633e 100644
--- a/src/ClusterBootstrap/services/monitor/node-exporter.yaml
+++ b/src/ClusterBootstrap/services/monitor/node-exporter.yaml
@@ -78,3 +78,5 @@ spec:
         operator: "Exists"
       - key: node.kubernetes.io/disk-pressure
         operator: "Exists"
+      - key: node-role.kubernetes.io/master
+        operator: "Exists"

From 0921e2d7417b27ab7b17a281ebcdec1b67311bd5 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 17 Jul 2019 10:18:50 +0000
Subject: [PATCH 091/194] put manager into dedicated process

---
 src/ClusterManager/cluster_manager.py  | 115 +++++++++----------------
 src/ClusterManager/command_manager.py  |  12 ++-
 src/ClusterManager/endpoint_manager.py |  14 ++-
 src/ClusterManager/job_manager.py      |  13 ++-
 src/ClusterManager/joblog_manager.py   |   6 +-
 src/ClusterManager/node_manager.py     |   8 +-
 src/ClusterManager/user_manager.py     |   6 +-
 7 files changed, 77 insertions(+), 97 deletions(-)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index ab9aa7d68..2e598de88 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -1,88 +1,51 @@
-import json
-import os
-import time
-import argparse
-import uuid
-import subprocess
-import sys
-import datetime
-
 import yaml
-from jinja2 import Environment, FileSystemLoader, Template
-import base64
-
-import re
-import random
-
-import textwrap
+import subprocess32
+import os
 import logging
 import logging.config
+import sys
 
-import job_manager
-import user_manager
-import node_manager
-import joblog_manager
-import command_manager
-import endpoint_manager
-
-from multiprocessing import Process, Manager
-
+logger = logging.getLogger(__name__)
 
-def create_log(logdir='/var/log/dlworkspace'):
+def create_log(logdir="/var/log/dlworkspace"):
     if not os.path.exists(logdir):
         os.system("mkdir -p " + logdir)
-    with open('logging.yaml') as f:
-        logging_config = yaml.full_load(f)
-        f.close()
-        logging_config["handlers"]["file"]["filename"] = logdir+"/clustermanager.log"
-        logging.config.dictConfig(logging_config)
+    with open("logging.yaml") as f:
+        logging_config = yaml.load(f)
+    logging_config["handlers"]["file"]["filename"] = logdir+"/clustermanager.log"
+    logging.config.dictConfig(logging_config)
 
 
 def Run():
     create_log()
 
-    logging.info("Starting job manager... ")
-    proc_job = Process(target=job_manager.Run)
-    proc_job.start()
-
-    logging.info("Starting user manager... ")
-    proc_user = Process(target=user_manager.Run)
-    proc_user.start()
-
-    logging.info("Starting node manager... ")
-    proc_node = Process(target=node_manager.Run)
-    proc_node.start()
-
-    logging.info("Starting joblogging manager... ")
-    proc_joblog = Process(target=joblog_manager.Run)
-    proc_joblog.start()
-
-    logging.info("Starting command manager... ")
-    proc_command = Process(target=command_manager.Run)
-    proc_command.start()
-
-    logging.info("Starting endpoint manager... ")
-    proc_endpoint = Process(target=endpoint_manager.Run)
-    proc_endpoint.start()
-
-    proc_job.join()
-    proc_user.join()
-    proc_node.join()
-    proc_joblog.join()
-    proc_command.join()
-    proc_endpoint.join()
-    pass
-
-
-if __name__ == '__main__':
-
-    #parser = argparse.ArgumentParser( prog='cluster_manager.py',
-    #    formatter_class=argparse.RawDescriptionHelpFormatter,
-    #    description=textwrap.dedent('''\
- # ''') )
-    #parser.add_argument("help",
-    #    help = "Show the usage of this program" )
-
-    #args = parser.parse_args()
-
-    Run()
+    cwd = os.path.dirname(__file__)
+    cmds = [
+            ["python", os.path.join(cwd, "job_manager.py")],
+            ["python", os.path.join(cwd, "user_manager.py")],
+            ["python", os.path.join(cwd, "node_manager.py")],
+            ["python", os.path.join(cwd, "joblog_manager.py")],
+            ["python", os.path.join(cwd, "command_manager.py")],
+            ["python", os.path.join(cwd, "endpoint_manager.py")],
+            ]
+
+    FNULL = open(os.devnull, "w")
+
+    childs = [None] * len(cmds)
+
+    while True:
+        for i, cmd in enumerate(cmds):
+            child = childs[i]
+            if child is None or child.poll() is not None:
+                if child is not None:
+                    logger.info("%s is dead restart it", cmd)
+                try:
+                    childs[i] = subprocess32.Popen(cmd,
+                            stdin=FNULL, close_fds=True)
+                except Exception as e:
+                    logger.exception("caught exception when trying to start %s, ignore",
+                            cmd)
+        time.sleep(60)
+
+if __name__ == "__main__":
+    sys.exit(Run())
diff --git a/src/ClusterManager/command_manager.py b/src/ClusterManager/command_manager.py
index 3caad1db3..9ad9b5956 100755
--- a/src/ClusterManager/command_manager.py
+++ b/src/ClusterManager/command_manager.py
@@ -20,7 +20,6 @@
 from jinja2 import Environment, FileSystemLoader, Template
 from config import config, GetStoragePath
 from DataHandler import DataHandler
-from node_manager import create_log
 from node_manager import get_cluster_status
 import base64
 
@@ -41,8 +40,17 @@ def RunCommand(command):
     dataHandler.Close()
     return True
 
+def create_log(logdir = '/var/log/dlworkspace'):
+    if not os.path.exists(logdir):
+        os.system("mkdir -p " + logdir)
+    with open('logging.yaml') as f:
+        logging_config = yaml.full_load(f)
+        f.close()
+        logging_config["handlers"]["file"]["filename"] = logdir+"/command_manager.log"
+        logging.config.dictConfig(logging_config)
 
 def Run():
+    create_log()
     while True:
         try:
             dataHandler = DataHandler()
@@ -58,6 +66,4 @@ def Run():
         time.sleep(1)
 
 if __name__ == '__main__':
-    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
-            level=logging.INFO)
     Run()
diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 4433e041b..68c961395 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -223,8 +223,19 @@ def cleanup_endpoints():
     except Exception as e:
         logger.exception("close data handler failed")
 
+def create_log(logdir = '/var/log/dlworkspace'):
+    if not os.path.exists(logdir):
+        os.system("mkdir -p " + logdir)
+    with open('logging.yaml') as f:
+        logging_config = yaml.full_load(f)
+        f.close()
+        logging_config["handlers"]["file"]["filename"] = logdir+"/endpoint_manager.log"
+        logging.config.dictConfig(logging_config)
+
 
 def Run():
+    create_log()
+
     while True:
         # start endpoints
         start_endpoints()
@@ -234,8 +245,5 @@ def Run():
         cleanup_endpoints()
         time.sleep(1)
 
-
 if __name__ == '__main__':
-    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
-            level=logging.INFO)
     Run()
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index c433b0276..221852ae4 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -242,6 +242,15 @@ def check_job_status(job_id):
 
     return "Running"
 
+def create_log(logdir = '/var/log/dlworkspace'):
+    if not os.path.exists(logdir):
+        os.system("mkdir -p " + logdir)
+    with open('logging.yaml') as f:
+        logging_config = yaml.full_load(f)
+        f.close()
+        logging_config["handlers"]["file"]["filename"] = logdir+"/jobmanager.log"
+        logging.config.dictConfig(logging_config)
+
 
 def JobInfoSorter(elem):
     return elem["sortKey"]
@@ -329,6 +338,7 @@ def TakeJobActions(jobs):
 
 
 def Run():
+    create_log()
 
     while True:
 
@@ -370,7 +380,4 @@ def Run():
 
 
 if __name__ == '__main__':
-    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
-            level=logging.INFO)
     Run()
-    #print k8sUtils.get_pod_events("d493d41c-45ea-4e85-8ca4-01c3533cd727")
diff --git a/src/ClusterManager/joblog_manager.py b/src/ClusterManager/joblog_manager.py
index aacf15138..5e55e74d1 100755
--- a/src/ClusterManager/joblog_manager.py
+++ b/src/ClusterManager/joblog_manager.py
@@ -34,9 +34,9 @@
 
 logger = logging.getLogger(__name__)
 
-def create_log( logdir = '/var/log/dlworkspace' ):
+def create_log(logdir = '/var/log/dlworkspace'):
     if not os.path.exists( logdir ):
-        os.system("mkdir -p " + logdir )
+        os.system("mkdir -p " + logdir)
     with open('logging.yaml') as f:
         logging_config = yaml.load(f)
         f.close()
@@ -159,6 +159,4 @@ def Run():
         time.sleep(1)
 
 if __name__ == '__main__':
-    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
-            level=logging.INFO)
     Run()
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index 4ed8f81ca..36c3b5b5c 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -41,9 +41,9 @@
 
 
 
-def create_log( logdir = '/var/log/dlworkspace' ):
-    if not os.path.exists( logdir ):
-        os.system("mkdir -p " + logdir )
+def create_log(logdir = '/var/log/dlworkspace'):
+    if not os.path.exists(logdir):
+        os.system("mkdir -p " + logdir)
     with open('logging.yaml') as f:
         logging_config = yaml.load(f)
         f.close()
@@ -252,6 +252,4 @@ def Run():
         time.sleep(30)
 
 if __name__ == '__main__':
-    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
-            level=logging.INFO)
     Run()
diff --git a/src/ClusterManager/user_manager.py b/src/ClusterManager/user_manager.py
index 5bcd63140..13968f365 100755
--- a/src/ClusterManager/user_manager.py
+++ b/src/ClusterManager/user_manager.py
@@ -36,9 +36,9 @@
 
 
 
-def create_log( logdir = '/var/log/dlworkspace' ):
-    if not os.path.exists( logdir ):
-        os.system("mkdir -p " + logdir )
+def create_log(logdir = '/var/log/dlworkspace'):
+    if not os.path.exists(logdir):
+        os.system("mkdir -p " + logdir)
     with open('logging.yaml') as f:
         logging_config = yaml.load(f)
         f.close()

From b8ed68c60c237b57127ca7a10ff849aa5b70b364 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 18 Jul 2019 04:44:40 +0000
Subject: [PATCH 092/194] add missing time import

---
 src/ClusterManager/cluster_manager.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index 2e598de88..10aef71c8 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -4,6 +4,7 @@
 import logging
 import logging.config
 import sys
+import time
 
 logger = logging.getLogger(__name__)
 

From a559c478ee71a6a583b2645a0e1b92545eec57cb Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 18 Jul 2019 04:47:35 +0000
Subject: [PATCH 093/194] reformat

---
 src/ClusterManager/cluster_manager.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index 10aef71c8..845bd1662 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -8,12 +8,13 @@
 
 logger = logging.getLogger(__name__)
 
+
 def create_log(logdir="/var/log/dlworkspace"):
     if not os.path.exists(logdir):
         os.system("mkdir -p " + logdir)
     with open("logging.yaml") as f:
         logging_config = yaml.load(f)
-    logging_config["handlers"]["file"]["filename"] = logdir+"/clustermanager.log"
+    logging_config["handlers"]["file"]["filename"] = logdir + "/clustermanager.log"
     logging.config.dictConfig(logging_config)
 
 
@@ -22,13 +23,13 @@ def Run():
 
     cwd = os.path.dirname(__file__)
     cmds = [
-            ["python", os.path.join(cwd, "job_manager.py")],
-            ["python", os.path.join(cwd, "user_manager.py")],
-            ["python", os.path.join(cwd, "node_manager.py")],
-            ["python", os.path.join(cwd, "joblog_manager.py")],
-            ["python", os.path.join(cwd, "command_manager.py")],
-            ["python", os.path.join(cwd, "endpoint_manager.py")],
-            ]
+        ["python", os.path.join(cwd, "job_manager.py")],
+        ["python", os.path.join(cwd, "user_manager.py")],
+        ["python", os.path.join(cwd, "node_manager.py")],
+        ["python", os.path.join(cwd, "joblog_manager.py")],
+        ["python", os.path.join(cwd, "command_manager.py")],
+        ["python", os.path.join(cwd, "endpoint_manager.py")],
+    ]
 
     FNULL = open(os.devnull, "w")
 
@@ -41,12 +42,11 @@ def Run():
                 if child is not None:
                     logger.info("%s is dead restart it", cmd)
                 try:
-                    childs[i] = subprocess32.Popen(cmd,
-                            stdin=FNULL, close_fds=True)
+                    childs[i] = subprocess32.Popen(cmd, stdin=FNULL, close_fds=True)
                 except Exception as e:
-                    logger.exception("caught exception when trying to start %s, ignore",
-                            cmd)
+                    logger.exception("caught exception when trying to start %s, ignore", cmd)
         time.sleep(60)
 
+
 if __name__ == "__main__":
     sys.exit(Run())

From 3d240c1d9374f1e3c429f328a1ef7ae437d1952e Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 18 Jul 2019 05:49:49 +0000
Subject: [PATCH 094/194] fix missing imports

---
 src/ClusterManager/endpoint_manager.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 68c961395..727e7996a 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -1,6 +1,4 @@
-from config import config, GetStoragePath, GetWorkPath
-import k8sUtils
-from DataHandler import DataHandler
+
 import json
 import os
 import time
@@ -12,8 +10,14 @@
 import random
 import re
 import logging
+import yaml
+import logging.config
+
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
+import k8sUtils
+from config import config, GetStoragePath, GetWorkPath
+from DataHandler import DataHandler
 
 logger = logging.getLogger(__name__)
 

From 649986f0c526dbea626e9970b932a7ac04531487 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 18 Jul 2019 05:56:55 +0000
Subject: [PATCH 095/194] add logging

---
 src/ClusterManager/job_manager.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 221852ae4..81ffe63a6 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -190,6 +190,7 @@ def UpdateJobStatus(job):
 
     elif result == "Unknown":
         if job["jobId"] not in UnusualJobs:
+            logging.warning("!!! Job status ---Unknown---, job: {}".format(job["jobId"]))
             UnusualJobs[job["jobId"]] = datetime.datetime.now()
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
             del UnusualJobs[job["jobId"]]

From 259ff92f303b9e195722f6d4967741a7eafd80b4 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 18 Jul 2019 06:40:11 +0000
Subject: [PATCH 096/194] set disable_existing_loggers to False

---
 src/ClusterManager/logging.yaml | 3 ++-
 src/RestAPI/logging.yaml        | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/ClusterManager/logging.yaml b/src/ClusterManager/logging.yaml
index b276c6d8d..37e98eee8 100755
--- a/src/ClusterManager/logging.yaml
+++ b/src/ClusterManager/logging.yaml
@@ -1,4 +1,5 @@
 version: 1
+disable_existing_loggers: False
 formatters:
   simple:
     format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -23,4 +24,4 @@ loggers:
     propagate: no
 root:
   level: DEBUG
-  handlers: ['console','file']
\ No newline at end of file
+  handlers: ['console','file']
diff --git a/src/RestAPI/logging.yaml b/src/RestAPI/logging.yaml
index d42108867..b104f120e 100755
--- a/src/RestAPI/logging.yaml
+++ b/src/RestAPI/logging.yaml
@@ -1,4 +1,5 @@
 version: 1
+disable_existing_loggers: False
 formatters:
   simple:
     format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

From fb161d0d889746508fdf9f0bbe3e898918353750 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 18 Jul 2019 07:38:01 +0000
Subject: [PATCH 097/194] output file:lineno in log

---
 src/ClusterManager/logging.yaml | 54 ++++++++++++++++-----------------
 src/RestAPI/logging.yaml        | 54 ++++++++++++++++-----------------
 2 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/src/ClusterManager/logging.yaml b/src/ClusterManager/logging.yaml
index 37e98eee8..a486bc5aa 100755
--- a/src/ClusterManager/logging.yaml
+++ b/src/ClusterManager/logging.yaml
@@ -1,27 +1,27 @@
-version: 1
-disable_existing_loggers: False
-formatters:
-  simple:
-    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-handlers:
-  console:
-    class: logging.StreamHandler
-    level: DEBUG
-    formatter: simple
-    stream: ext://sys.stdout
-  file:
-    class : logging.handlers.RotatingFileHandler
-    formatter: simple
-    filename: /var/log/dlworkspace/clustermanager.log
-    # roll over at 10MB
-    maxBytes: 10240000
-    # At most 10 logging files
-    backupCount: 10
-loggers:
-  basic:
-    level: DEBUG
-    handlers: ['console','file']
-    propagate: no
-root:
-  level: DEBUG
-  handlers: ['console','file']
+version: 1
+disable_existing_loggers: False
+formatters:
+  simple:
+    format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
+  file:
+    class : logging.handlers.RotatingFileHandler
+    formatter: simple
+    filename: /var/log/dlworkspace/clustermanager.log
+    # roll over at 10MB
+    maxBytes: 10240000
+    # At most 10 logging files
+    backupCount: 10
+loggers:
+  basic:
+    level: INFO
+    handlers: ['console','file']
+    propagate: no
+root:
+  level: INFO
+  handlers: ['console','file']
diff --git a/src/RestAPI/logging.yaml b/src/RestAPI/logging.yaml
index b104f120e..fea884c5c 100755
--- a/src/RestAPI/logging.yaml
+++ b/src/RestAPI/logging.yaml
@@ -1,27 +1,27 @@
-version: 1
-disable_existing_loggers: False
-formatters:
-  simple:
-    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-handlers:
-  console:
-    class: logging.StreamHandler
-    level: DEBUG
-    formatter: simple
-    stream: ext://sys.stdout
-  file:
-    class : logging.handlers.RotatingFileHandler
-    formatter: simple
-    filename: /var/log/apache2/restfulapi.log
-    # roll over at 10MB
-    maxBytes: 10240000
-    # At most 10 logging files
-    backupCount: 10
-loggers:
-  basic:
-    level: DEBUG
-    handlers: ['console', 'file']
-    propagate: no
-root:
-  level: DEBUG
-  handlers: ['console', 'file']
+version: 1
+disable_existing_loggers: False
+formatters:
+  simple:
+    format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
+  file:
+    class : logging.handlers.RotatingFileHandler
+    formatter: simple
+    filename: /var/log/apache2/restfulapi.log
+    # roll over at 10MB
+    maxBytes: 10240000
+    # At most 10 logging files
+    backupCount: 10
+loggers:
+  basic:
+    level: INFO
+    handlers: ['console', 'file']
+    propagate: no
+root:
+  level: INFO
+  handlers: ['console', 'file']

From b11cc96287709f3c395135774b80615b6dcc9034 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 18 Jul 2019 08:16:20 +0000
Subject: [PATCH 098/194] catch exception

---
 src/ClusterManager/job_deployer.py | 56 ++++++++++++++++--------------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 5d2e22e12..df93c0340 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -125,29 +125,33 @@ def delete_job(self, job_id):
 
     def pod_exec(self, pod_name, exec_command, timeout=60):
         """work as the command (with timeout): kubectl exec 'pod_name' 'exec_command'"""
-        logging.info("Exec on pod {}: {}".format(pod_name, exec_command))
-        client = stream(
-            self.v1.connect_get_namespaced_pod_exec,
-            name=pod_name,
-            namespace=self.namespace,
-            command=exec_command,
-            stderr=True,
-            stdin=False,
-            stdout=True,
-            tty=False,
-            _preload_content=False,
-        )
-        client.run_forever(timeout=timeout)
-
-        err = yaml.full_load(client.read_channel(ERROR_CHANNEL))
-        if err is None:
-            return [-1, "Timeout"]
-
-        if err["status"] == "Success":
-            status_code = 0
-        else:
-            logging.warning("Exec on pod {} failed. cmd: {}, err: {}.".format(pod_name, exec_command, err))
-            status_code = int(err["details"]["causes"][0]["message"])
-        output = client.read_all()
-        logging.info("Exec on pod {}, status: {}, cmd: {}, output: {}".format(pod_name, status_code, exec_command, output))
-        return [status_code, output]
+        try:
+            logging.info("Exec on pod {}: {}".format(pod_name, exec_command))
+            client = stream(
+                self.v1.connect_get_namespaced_pod_exec,
+                name=pod_name,
+                namespace=self.namespace,
+                command=exec_command,
+                stderr=True,
+                stdin=False,
+                stdout=True,
+                tty=False,
+                _preload_content=False,
+            )
+            client.run_forever(timeout=timeout)
+
+            err = yaml.full_load(client.read_channel(ERROR_CHANNEL))
+            if err is None:
+                return [-1, "Timeout"]
+
+            if err["status"] == "Success":
+                status_code = 0
+            else:
+                logging.debug("Exec on pod {} failed. cmd: {}, err: {}.".format(pod_name, exec_command, err))
+                status_code = int(err["details"]["causes"][0]["message"])
+            output = client.read_all()
+            logging.info("Exec on pod {}, status: {}, cmd: {}, output: {}".format(pod_name, status_code, exec_command, output))
+            return [status_code, output]
+        except ApiException as err:
+            logging.error("Exec on pod {} error. cmd: {}, err: {}.".format(pod_name, exec_command, err), exc_info=True)
+            return [-1, err.message]

From 765cc9e92901206f51a73336f26ca2116d5f16cc Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 18 Jul 2019 08:17:09 +0000
Subject: [PATCH 099/194] fix pod "Unknown" status

---
 src/ClusterManager/job_role.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
index 7ceb00ecf..d54354325 100644
--- a/src/ClusterManager/job_role.py
+++ b/src/ClusterManager/job_role.py
@@ -40,8 +40,17 @@ def status(self):
         pod = pods[0]
         phase = pod.status.phase
 
-        # !!! Pod is runing, doesn't mean "Role" is ready and running.
+        # if container was restarted, return Error
+        if pod.status.container_statuses[0].restart_count > 0:
+            return "Error"
+
+        # !!! Pod is running, doesn't mean "Role" is ready and running.
         if(phase == "Running"):
+            # Found that phase won't turn into "Unkonwn" even when we get 'unknown' from kubectl
+            if pod.status.reason == "NodeLost":
+                return "Unknown"
+
+            # Check if the user command had been ran.
             if not self.isRoleReady():
                 return "Pending"
 

From 3459a895d111eabf97a158c8805fbcae1f5b4042 Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Thu, 18 Jul 2019 11:50:18 -0700
Subject: [PATCH 100/194] Redirect to Wiki page for unauthorized login user
 (#423)

---
 .../WebPortal/Controllers/HomeController.cs   |  1 +
 .../dotnet/WebPortal/Views/Home/Index.cshtml  | 56 +++++++++----------
 2 files changed, 26 insertions(+), 31 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs b/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs
index 006997236..466a74e4e 100644
--- a/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs
@@ -693,6 +693,7 @@ public static async Task<string[]> GetTeamClusters(HttpContext HttpContext, stri
         #region ASP Controllers
         public async Task<IActionResult> Index()
         {
+            ViewData["AddGroupLink"] = ConfigurationParser.GetConfiguration("AddGroupLink");
             if (User.Identity.IsAuthenticated && !HttpContext.Session.Keys.Contains("uid"))
             {
                 string userObjectID = null;
diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml
index 88911f8bc..a59b2febc 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml
@@ -8,9 +8,9 @@
 @if (ViewData["isAuthorized"] != null && !(bool)ViewData["isAuthorized"])
 {
     <script>
-        var ALERT_TITLE = "Oops!";
-        var ALERT_OK_BUTTON_TEXT = "Ok";
-        var ALERT_EMAIL_BUTTON_TEXT = "Ask for Access";
+        var ALERT_TITLE = "Alert";
+        var ALERT_BUTTON_TEXT = "Join SG";
+        var AddGroupLink = "@ViewData["AddGroupLink"]";
 
         if (document) {
             window.alert = function (txt) {
@@ -34,45 +34,40 @@
 
             h1 = alertObj.appendChild(d.createElement("h1"));
             h1.appendChild(d.createTextNode(ALERT_TITLE));
+            closeBtn = alertObj.appendChild(d.createElement("button"));
+            closeBtn.innerHTML = "X";
+            closeBtn.style.float = "right";
+            closeBtn.style.color = "red";
+            closeBtn.onclick = function () { removeCustomAlertInLink(); return false;}
+            h1.append(closeBtn)
 
             msg = alertObj.appendChild(d.createElement("p"));
             msg.innerHTML = txt;
 
             btn = alertObj.appendChild(d.createElement("a"));
             btn.id = "closeBtn";
-            btn.appendChild(d.createTextNode(ALERT_OK_BUTTON_TEXT));
+            btn.appendChild(d.createTextNode(ALERT_BUTTON_TEXT));
             btn.href = "#";
             btn.focus();
             btn.onclick = function () { removeCustomAlert(); return false; }
 
-            btn = alertObj.appendChild(d.createElement("a"));
-            btn.id = "closeBtn";
-            btn.appendChild(d.createTextNode(ALERT_EMAIL_BUTTON_TEXT));
-            btn.href = "#";
-            btn.focus();
-            btn.onclick = function () { sendMail(); return false; }
 
             alertObj.style.display = "block";
 
         }
 
-        function sendMail() {
-            var emailcontact = "dltssupp@microsoft.com";
-            var username = "@ViewData["Username"]";
-            var subject = "invite" + " " + username + " to the group";
-            var messagecontact = "Hi" + "%0D%0A" + "Can you help me adding to the group ?" + "%0D%0A" + "Best regards" + "%0D%0A"+ username;
-            window.open(
-                `mailto:${emailcontact}?subject=${subject}&body=${messagecontact}`
-            );
-            document.getElementsByTagName("body")[0].removeChild(document.getElementById("modalContainer"));
-        }
 
         function removeCustomAlert() {
+            removeCustomAlertInLink();
+            window.open(AddGroupLink,"_blank");
+        }
+
+        function removeCustomAlertInLink() {
             document.getElementsByTagName("body")[0].removeChild(document.getElementById("modalContainer"));
         }
-        alert("You are not the authorized user for this cluster, and please contact <a href='#' onclick='sendMail()'>dltssupp@microsoft.com</a> to get the access. ");
+        alert("You are not an authorized user for this cluster. Please request to join a security group by following the button below.");
     </script>
-}
+ }
 
 
 @if (ViewData["isAuthorized"] != null && (bool)ViewData["isAuthorized"])
@@ -291,14 +286,13 @@ else
 	background-position:20px 30px;
 }
 
-#alertBox h1 {
-	margin:0;
-	font:bold 0.9em verdana,arial;
-	background-color:#3073BB;
-	color:#FFF;
-	border-bottom:1px solid #000;
-	padding:2px 0 2px 5px;
-}
+    #alertBox h1 {
+        margin: 0;
+        font: bold 0.9em verdana,arial;
+        background-color: #357EBD;
+        color: #FFF;
+        padding: 2px 0 3px 5px;
+    }
 
 #alertBox p {
   font: 1.1em verdana,arial;
@@ -310,7 +304,7 @@ else
 #alertBox #closeBtn {
 	display:inline-block;
 	position:relative;
-	margin:15px 13%;
+	margin:15px 38%;
 	padding:7px;
 	border:0 none;
 	width:24%;

From 4c0774df7b8f5e8e4ea14795067a2addd0eae1cd Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Thu, 18 Jul 2019 17:15:59 -0700
Subject: [PATCH 101/194] Fix the job template dropdown issue (#424)

---
 src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 6225c7fb7..604360d79 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -713,7 +713,7 @@
                             </label>
                             <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
                                 <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob" ng-selected="curtemplateValue !== 0 && !isLowPriority">Distributed Job</md-option>
+                                <md-option value="PSDistJob" ng-disabled="curtemplateValue === 0 && isLowPriority">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From d421b66d1758aaf59848ff8f8c9be490968a570c Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 19 Jul 2019 08:09:32 +0000
Subject: [PATCH 102/194] no longer using "hostport", remove useness code

---
 src/ClusterManager/job_manager.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 81ffe63a6..0d4cf91b3 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -204,10 +204,6 @@ def UpdateJobStatus(job):
             else:
                 logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"], retries)
                 SubmitJob(job)
-    elif result.strip() == "PendingHostPort":
-        logging.warning("Cannot find host ports for job :%s, re-launch the job with different host ports ", job["jobId"])
-
-        SubmitJob(job)
 
     if result.strip() != "Unknown" and job["jobId"] in UnusualJobs:
         del UnusualJobs[job["jobId"]]

From 54f74923c16ed19c5ff78200c23b226988083612 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 20 Jul 2019 05:21:44 +0000
Subject: [PATCH 103/194] job state transition graph

---
 src/ClusterManager/job_status.pdf | Bin 0 -> 119750 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 src/ClusterManager/job_status.pdf

diff --git a/src/ClusterManager/job_status.pdf b/src/ClusterManager/job_status.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c9756f120dd872306ed8d20fd73e24c1b588bb84
GIT binary patch
literal 119750
zcmd442X}4Nl_sj3bIv(u=_pG_Ip>_Sbd+<>BDye`jIjaZKr#kn0|sp4G9H6)flKBF
zGPkN+wyRw3^6TzD@V;-)rG3Q3e%+%-y)k;+Z-zD3-Yd*G*W6n=y3*Wc#Kz|M=GI1R
zTUAYJW?G)BQ|w=V_gfvVPMHxI5i1k(u1>|p>0NuPru543&GE&Rne?{t8Pi5Y7P>pq
zo5q)C5=_gf-JOXM>7ASFTa%kz>n2mj%F1R$W{!4A>(=Jt+%oRSlv|p|*B9!Sr&lm6
zRnt@iZZnmXq&J&3$EPp{F3iyG?3pv|nAY3YP1B}z)AFP?1XtBjQsU}lTAtEY$3efc
zOl^7Tl`C7zSVDT^+|<Tkgn@SrN1VNnrmx+Hy<w(JwYd({#>&<@=EYL0cQ<Q0;V={Q
z+RmH^*;{RAZbYuED7|K7c@v#AnGt!kq}OCc6sWGY^_9s^)8=40e4XB9+TBdAUxE`W
zL{y5X=lbA;sk!lrmEA$ynh}wootcKY3bNDka)u+YspU;9ZzCdGo1u1nWos2vYSGae
z>7D6a>*LECtK7!qUV3F`dX;H=Zqn3ITY<s5#wRv#iNr~^x)G6a)+-$N)GIY}i<_qP
z^qNKNy2>=UGG$6{GA+++&PEsvIhnj`V{_d!zU1n(D`^w@)@>o8m58VepK(cndMHhB
z)9W``T5tE1yH&WMwI5pm-n|Ica9r9AF#!^M_6w1-x<yww<XQBbZ9q@Is*PqgsCP7H
z^hCNQA?855b14~Y{>U5L2PAM<f!xuXrO>#LiqJA;--G<(`K1vEV0lX9wg};N$0Eyu
zd)un}8&lD8Xzl~Am<QXJPRDt*BsAFvSaGo(qJwirS0^w#O$iE6-rh|;Ob)h9DL>dT
zR-k7fA1pr}3<%<6ZSOznod)RwTFg6lOBiH3Xv`2<36a+I&P`)g4$V<rXgGs5KrZEs
zf+wglim{{}Sj-bCyzPzOX$wM&_W=79UpxW9CFEt0)4{;Z`Y5=?fwUNBFHzsQrFV8~
z40=Wbm~+Ikn4=Ozf^X`Yt|({g?XyY^8po|>Uov*dCdT8-C(mAY{#R#P(3vbJxhiG2
zjHAq@^Y@#g>1{zo6l9|4@v+nS4p4(Yl$jvUfn4zZvFyqWNMPTyX>^g*v?&hEgS~*4
zIvY^O8&3U)Myv%G!#xCY$s7h>UIIIR9T<=q#5~%Wv&bu<cNRbnxx65#1&q1k4+2ul
z!RT)zXoC5i-|p2`HJM+>)mDLkFK?a|<#a~0v^mwG;!$nlsoX2J2luO>V>#RCIv5ZN
zI%w{f26$>Am~AguSTR@w*T<|l;ehYn1!iFh#<UVd9bNvG-2xNQ&LK&+avy@od6Ar$
z8-C}B_OVhAUT>0)_4Q1{YmZn<16-XwXyaAP8qK#U)zFwQHom3cf-_!^a*56_p^w%*
zq%Xt9m%bDDK)CIO?Mf>P!T`!Vs&Rw}GMyQRQ)owcjD7&tL<k?YH)o_+UjK#7g$JSL
z2k6(m`$7Zvxi$0IH3N{I)6$d1u&!q%GOFtzJm@%9$7}-1c%T7ykd3eCtef@r#o>ir
zu`ZD+lrZsd4fe?;Q7+}L{b)UNPfJP`KZjRt8mm`PMYHzMQ!WH=(=|Gm@uR!Bx(-lC
z5+hw12*xfz#j!I~n&8zdYjd?~^Nnb$643t9&H5!V+%?d!;RUXC4K1c>Zb)r0t4rlY
zeU4nKCC6;bzO>Zr(g1H{$_myqA(``BI6Vb$8qfCjXla4h(->R++I?_KW1LDa*c=?#
zxy*APH_U_l&bPrP)6^Q7WIFsrOXLX<u;$WLb<(7VRe9m^BGnqzp@k(Z##CAqn7kJ6
z$F%GE_yL;&`qvP<90QR^Hv?HUo1fhZX{rQkW6}<SfLvE6A68PTBEUTYV>`VoppbM#
z=In^x4B<jQ49ur<sRE8tC~72xmgP)%VAHl7e0`9qWloMRQz+W=U0}V@fx!@k5gb)5
z2U<#_@<JB(0<`vkSu!|4N-Nl#_x;CnzbFv{_}JyURGx34Rh7?71#)WN{0;=qGS_Bb
z%)|s(ZmhXBK^JQxkgK`6kcX_%%?f;q320vV7>{Wgs6XVhyCDD-kY!HW$nJF;Fy<W{
z&6V0mfGvXZK}-wY(zQ5D{?1LgwiZwXRk{YnE1?&B7xclcO!?NZEO7Uv8gRjUmwEr4
z8O*4OuZ+MHNS>;(^I>5HIz`xO5n7+NVbC0yXKr~PJ`J(q3l_Adb$i%B7tkA~t!}-&
z+?4go1M_suBJ(M>DUo?^@=gZlwSsfExjNC(0rCW`f@*kuNHV8ni2;fncq(S-G9U(2
z^6K|3wx#Ly@%NyxNCc^kZcU<5z+9-Kww0u<wewF8artx-z_76-w@Xtvo#VpUt<PGa
zpSv+)j3*CZ=@%q0j_O?3=7}w<p%RKVw_0AtuB(#`CyfGe>D;EO8J_)NGiZf?&LSi}
z8clC8X$MX3?PR(DDuNSGIf7@Mm?OC)W|Lro#SvHr&3PJzAtLxp0eIG!qJoo9$gHG~
zp42T<v;aq{SVG-q1p(f=18ipc7`%8qZiV8*pMe7jt0KX}7qwT5Vch_Q+Yb+RJamrb
z%{t1p__%Q+^C0d)(6-7nj+e%JKlfsMlv`eV#wccRW(sim+q#8k=lqLc>BDZ|yzd}3
zFBpQ^{N}Gec^o3eKR1AB0}M5ZY-(Wihd7|}6EudFu5tkR(3M{Y&#M~`Au#>*Zn1=_
zd-GM!^qsvhAc?ieypFcmsK_C?fU$LB(G~=%E1Nis<$Kp>i@7^T6|>BA{Zh@ZXtmah
ziRr+T#1mJcLXZW0H)<Fh_{(EYDgV+j8K5eU8Wq#_s%$S9#z3JUz+@wGYQBPd&)s9n
zFFAwlZ8I~07El6+MTL182Le2{x~Jpms8=5Y<8lNn3`d~l{ZKK8^1(l%#XycR7BBw<
ztOq1Ws}TA2^_<?>Yg(m4JpJH4ohH5cL6GO408^e!6ua<&vkflCnMugB5mcy9NGtll
zJUcAI>t*@zpWR(E87|$-9Og1>pDoA;5`=|ta#n~-KT3b?N05ioapAT@l@yVClVYk%
z{O~$xs;=;rm`{KJ`ZXT}G~^V2{2-)f6JyBOm`@H}AqdHMgZ>h%)_!FpeAu}t?=LPa
zj72cGz@dG#R)GpbrJV_};qB1Ii1iOU9(4F~EdW!qi>b2NW>ld?BWC06ds(L#^dWaJ
zfDeW+5JyZd1LK<CxL(W9GHy~;G-!#@YSSfXyn2&AxDL9QKCBf>i@W~`G~>RtB`163
z&QdN_U=+bUhO|_7C67b#T-#Tn?vh^c6D|v=0TqGPQn^h5%R;a9cE87PIW%pt8BC<F
zgU^z*$(Z1hw%A^%D=y40-d_Cb4d4#BU_iSFZo|jG*~<Cl&*Zt=!C!eq<)k$l*(x8P
zW~wAHZ|!IwW}XKimZQ-3^E9@sTzL}W{EI8#pFD8@E--tsyT)e_wsYSF+gferJjy4P
zM{9C1`1w1gz?m(x`)zhwY3BItu1@{Uc9VStuRq8%2e|Q45FN_Q7HnARYRrbNUU~`Y
zr5khj1_7wz3$4tzOml9rXB1j|<ZwxG>g|*VS&%v!0mXq1Qrl03N;fl$m-nYxKq6H^
zfGTL*g8a&nUH{W<+`cyFE|fmGQ0H4+Djn&7$=R6yxaM0>f>Rgd3N8|GaCA06sOOWn
zR(L!a+6ZO~z;&wn1I^x1tiqS|hPDd#)%LM5p7P4V-nu!)t#Oc(TY<I)DvMwFF%%sM
z-TQC-t`JR!aE2dPI<i?)C(s^E9^`ahpkslf0p$%~j(v@E8W03xop-6R15W{I%b{C&
z3|c|Fq|JlbR}11(Q1Ipremr!|+zG=X@Ur5HIJJuM0Mth5^Pxio5j@<0@_b~bs}qk%
zlIM5=pHOuFi0b3P<$eH_5V6vlPzl6UGjpp;4nw^mcWtC!=kpQjMz!hDSQqr=anZ^9
z5mkAiJhEqD+scQC{*H0s2i<8Z$~mxSp6J(Ri^c=c4zl5GI#Q2Wls!XLx<Ie?Qn{%m
z$Spsy+Jn*Mh}D`e9;`%$oM>={<ey)7ZV{r8*$e1jR!P$bg7<KQfc|+gcJ$UYJ1}m^
z3{y9#GeCNT+t-@I$E7l~0L!=~UBuK;&Ra~0j6_Rb_MiZspntjHKKc$Qy>uTq_tx0z
z7IiE}@R63V{wQ@ibt9BeX{gw-C>ZCIH+G~;x%iCRVJ`;pI9&Q|`m4VkR@pAs&mW3(
zo+}e&#*YqHxx(lIo)>-&-T~T=3}lRe+8AasP%i?U<*RAUT#+~sH^CEMO4p=SuLg4V
zel}WIP1K}g+}M}jz5F8xK_PvfrZ)$ntscc3(BX`*BlB9V4jQn-?A7u4@3hw`Kl|`i
z*tFXE@lW=fSHoD-dz7!BI;`@<&}r50{oKwE(jCMmh;Gwy;dZF2JV$~HzkwV~cb1l|
z2<0{tQ;(AZmfVED_2pYEAz!=3Fyf+)B)xLNggoXI;UFLnRB8fy{g><q=4hclBW>_T
zk16Hk*S@D4d5U<g704cxy`X&rbNUIu>8COLSmgBc1MOo8{d1G_istl|mG!0Z#q`SZ
z2<_v8)0_AS%K!4FNqjDJ_LB;J$nuq&`1wnE=hnm~Kdb7(M@riKE#pfjd?u4#F}`8a
zuBfQ2s;;h1t{h*Sn^>PquU^In8*|Gu>AiEy<;xp$dYczlnKmZZ=T<jY)+6#U9X`{U
zTi@8MoE=|}Fl6DgFQsQ@=HvFM%~^cLm8m`sGJinDJHNJ2^T^ugM%pT`S{Xifnc3hx
z_4wFrZnAuNX3-RpkzT$r$xo>Aax(DIj_9dbXjG1`)|uvJW~Hlqdq!JPL{3&FK4#mM
zMrK~NL7T5vuFTHJNbgg3ag5GQ(^3y^(XOe`K7~rn$uLB40(>f$oxy4GITU?^Plrrd
z*su0s*V(}EL$9yx_wNNJ>nnk&F$X5KVr6mae>*yv@`0f_JlT0?!;@zYj~2(X%hhiE
z+Is&^e7?58|Dqrna?b`SKl5ym^8R*^B#@^;$}BhyQm)}|1WA1mX->+_uY9^Yx4CFa
zzz4YHTbr{h>j`z^D-r!$<IDU=cWYwaG`Y#nM$_KT%KFp>^-6s1w>h`6T!oJY6RHZc
zGBOJ?^1uz*x!F0%8Ts)U8S%KRd1dN<aRojZo!Xi-ttX(Q02a*_a~6{p>lP~(8=z^6
zO$%BhEP7zy0IjHYtHrX#BHGIqCiJ#JABP@ck#)K_Y~#Of$gkwf*FJDQ{eWAlLqw+b
zVY8Ng5t-TtwO_gYE1x@Sj-OiG*%+N)nHc2<@}`Zn?Tx8jY(sq%tv)r^Qd^C8c5L-Q
zcWNf9!KV78JH4-e0H-jg0Ok0rpL%Nx(z9Kgo}ZVeW}cZ)*8jgXT-Mi}F)`n{xs=`#
zk%_dGMduadL;HK5C1>Up80etWC&{jaZD5#>(p;aTbZP5+oUf1S;a8?x+o18i$?bp~
zD4d_TcyDn(pM58w%67iMbu9BGLl7e(WMqWzMGc^QvFLo9uRhh8p6WSmei2F3FKOaC
zlF!Vi5A!uIq013mdpEywZI5~dgQbagesD8b%Ie`}O28rCoI^m}V-*g3uC@=n`J8zx
z=Wc*$4g~NjplNm!Jl31<|FdGEbubP*v#E%}IxL^hcsEdiOJ2q(q-RsZe|+~&_JPJg
zoGXhHWrDuA&eJ6~V%7+iZ(6+Z{oAN;5^MjrAA+#(<!!+1tJi^-74e-)@WnOh^qXIj
z=Jlkc)#>tnv{m$6c8eS)CXQg3``}lczTBm)Eo>jkg71Wp!2+rsxt0QZN1H!i&)GZz
z#D137RL~P%5q#xH<C__Sq~4PF7F5e_zP$mNF#$#9uU`0e*j~3B!xc@*1<4TB&MKVH
z;~Qp9KX^uqqXT^+z^einsEJ}bzOzjf2RTB-g8&N=5&RkqhyrDS0<6FsI1uAynU%=N
z0Xyy-uozcgC`c3B-hPf>0=POAf`9OhzRTD0-PK{f1{~lQJ(VDCMqp2WD+e(7`q~}p
z5>=GYoRq5$HiyR}GgHv;qUU_VV(qH6#AKf8peOPVMu&XF@)2J-lSZU5BjPLT^<{=F
zNymbE$-VT%fd5Jw7Y+D|XsW(=i+`)=bvfDV&=RGkB9BHelUBzvH(JugtW^_3<5yNd
z;}+k3)fPF?<eCk5M){CMoSmIdzsku5&&cQ=`Z9Nk;W&#Bf4x=Df*|mSj5assOBn!G
z0gLlb^Ggseql^x+GryC2Mo#9!qJjxcFjXQlBpNaRRIj3!fGLzJS6szCIp_MO+9w^d
zJ74Be_4ZBBPC)a@$F5EcB5>)acK*YV-QMPe7(m;+Ke;X`+tVPL&iUoRRj98O!)7Pl
zwvu+`p$Pqo6$XAJ?PH!xU>=ks>Rdmg)XFrN21SClwA2a=h1vq*k>nR1I)<Pz7nkT%
zZc|4Ii@aca@bdHg%4B%^1D2sSI#Y16eCEsZBG?Ypfrr`!V#dzB^C-9=g~<fO?umTo
zMy_~0GzdJdX<lVVkQHZ0?PEmGLfcE+(kI`l7ptpN2bA{awW_xtQ3vJN85U1k^ZP&k
zHe|+_?N;Mk--N~nh$;WY&l&sTXjTswN3e7dFjh#P0cSq+69ex|;s(DC`g8fhz2Gg&
zU@2SgJ?!FXNVMInmTUiT)1#rH#Y5~ddfgxub><lp3SiBs0If>%>s+0Q{rwNFL#H~}
z(75N-<v6tozbrHv*U4q>)yTM)f|wEldQHHJTB|CdZk*f+haL;!g2$MyhlOkcogjBt
zrvnIEvFM!hEHda9>Gy%C>1^rZb@rlyCxC{B!&RiqIi7&477R~$htMUCwO(R4cf*&z
zgGSE*(Ogv-&(;wrvt$_jEannjnC5~xH3Wm~oyp=P@MMNf=*^ZwE_^eUnFEwIF>&qw
z{OzYat6a(>Dplc4m0lhbmimu=&HQ2N0-VQ(52bA1Y<jHuxE?$h1WY!8RaH-dt20(B
z9pKEk-#`Atg620KfEGO^9Sp8Is0@ZW3F_t?p4hVpk3q{1L_ha~>e~w#?v4VmBW`MT
z*4H;X(V<{8J9uKEA`=!S9`o=lYD&@atp>aNbRHv4hlPB4$1yjVTM5Q30O3qU>gir(
zViBI?sJ+I;H8Xz9V^$}C0jcZ$@SDf^+*sH4C>PwA4<@jtgBViI{oaqkRN;oUZ*Hmm
z!;l>cmIPua^@dUxa1XJ7&*w2Bm4n@Xec&;^T;xg5k_hlCkAv-jdYD%WO;+Gxy9F)_
zi%<kbyPq&7K%@~Yh&MO)`ie<aGBg@D-?&|D;M(_ZA>VsKH$e%Z#{uOIX#euqw-_+@
z^ab#??YABag9MgEfYaSTE9h8Mb)e8-a0g#`Ow3G<0rn+-d9PULWI){}4R8~PE2s@m
z>v`+R`7Y%Xn#_LLK<o8i{NX#-LJIbI+5yJ2j&5R>0NRfyjI12df5jy{{@eqrQ0K|(
z_dzT&1o^gLyWwE0NnxXgu#l6(H@VPdrl1ZVXwjhK<&aP@GADGfNNp62gVSNa){5C3
zXez(aTeXYt|DXgiJg6KT_?zDjlWzxUUvCXYWM%PtL;rvN%hA3YBQxY?^J}xe^)EY*
z-wwc+hgmrh*#-F#h75k!j;$;M?b$h}8f`gQd0Jmagdw{i!cdUOpZ=A5bFzlden;W#
zcjW&5*L#M{EPn4y`$6JNe3fX($<=-W`Kqt~j_espmH!1$nMa8k)|H{F(;-lqBF0%T
zpd;ng>k3s>_2D7#@@<qJpPa}{bt;;-I0Daa9B+H`v6X#h(F7B_IhPmlImM7N6Zs~T
z2ok6q*4t;5Lo}GQDXdm3%7XWwIsecHH$qP<V6R}Jbh2hpRA;Z;D>p+`^%q)w=Z5`=
zbiS{yN2^UN`{UiLghpwpEoGr~(hkNo%*=pQ27m!7>D{5`fYhA4o5j`-^wxJ%7e-!r
z0vZtCKQg`87|0oP^g#Q|15YRE1}H1DM*1G;U=0Q8M5GUi1WyI=rqXd$<Fpql>E`+Q
z{8_NIFP?HJHSwScjzy?DqQCqF+S~nLIWsHDCvt_F4~===t-l#dFSsgbqo4kYhuO_v
zZQXo~W98B{i2XRmD`?3)D4H2e5!R<Y5Y>sA!}soLZjzSpV>%}o)YDYRX;yjtpC0>B
zU5$zinmRZKpoX)y&#gVu`(C>Xs!v$&=(buGnoZU9MG&T7#xfKH(4pO1{fFL=YP@-<
z5nZ2L-={`XM<p?Hzq)-BegEGWut0x?Tw@MHAqNHQ)M~glpi_i;onY26*Xvj#ouvyr
z5Hz*}rakKTJUxpk_W1Duz<gBUhDImqxcHQ=LQ%<}n|i`@l_!czpq~VD^w1$+1^_#2
zuzJqeS`6mKXl;OaIg^Ud32x>9l;6Y`v_X(yo5q@LFfP*U8Z`dL&!4#uLNKLky5a_n
z!wc4fL{26{X7GGusT^|o;-+ZnQ=uI7|LCbEKAPIO0YlZW^wu6iYA>ZcH-L%`H2Q|s
zON|3fp!Tux`FEwg(==lCnwYvcx|+CKOyIjFupPQAd1q%-H$&;w8w>QKj=q=wQS{@2
zkyn4Hn>(QKuw9)1FeP$$pas;L+cP&g2_-sZxRK!u+GS$sc>fp5lcm$HPGz33b5Kj^
zU!-miVk)XqCq3Y`Jc0}7ejtWH-U?KwOiZ1BUUcw15Pis*Q%`(Uf;%(hxm76>yr~Z)
zU`7Z^c)?8Cls{j)fxW6m-g-Jx_33?4e$tOwzhWwdc?(p~Iq=aB@}f&;YOcBfeOKQj
zBMIknf#|<8Ji7kI-Tx&fnOOKR#E$dn9>tcyxTn{z89IhG#Nxj?W#IEi(4Twx?&dM*
z3@g*FFc*v=uK@up4b~riLmExNB88W8LLDB^{{34&<H=mK9DMcO5N~KS7J+~f5rO?f
zAN`I8rv067@EnzseH{Vj0`}#Dk<x#3)nyH@J`M|OVT1HAA-+`E57pJ_FmqIT&T_3-
z>xMlQGcgo%l`bG=G2qT}RLw`{S<t5dopJH|qO-D<W`GSw@RlprgE1=t2h{fI<4_P_
zED;G_#EXIYqLYKplVROHI;viBn3?D917kG=OpPKhYmd>FzhR5!yFUl(?Hx3zbKGy3
z>5{A1+YIU`<iP?Q<3)z&Z8xp8t18+qHgNXlujx%jkj3-AedliQB7>@G+W(W-gk^Hk
zPbJkXT7a4sTGS|q;dt7I{;Xo>ZUgh28}$w89u9$8?mh$#u-pI*%FV_PZ-3)Ow4*>i
z(gnT^a>$M1h@E2O8xw0dx3*h77bw;WMx0l%K6+=15!@@6Uv;V^^ofP;Z!U$MJ}D#Y
z>gehR2L`wntI=}s0uCmik?AtFGO9g2jL;V<z=k<b_jnxm?qVBw0tj$yNlQ&m+T;rV
zaR2STvs?JdaP0^7dtjLX%^JXf3-^l@p39sW1kr5*dNze;qC4T@Ln>t#9aN1yQfG*H
z#iRq$Kz5)bP&FteJc+&%ScE}xdiDwegLHLor<H|Z@{OlY-n+qP7-|SNrV=j~Do-`A
zUp<({z}GPn?B=e=H-^=5MI*_bkN+r6_4YSa(V0dAP>zP?+u7a4;As)?Va>h@3uyC#
z8W4jq-Sh5K5dK44CXk#0#&qkUWBeHq!BOh`fBV5LqBm^v1O<RTe3oYs$P{6bUp&Dq
zZ%w^$vsi2)Y|DH4L(_brr6#E?r0DYNu&y44^5L^!TCDA%07lS~sAD~Jzh{%mf+-a*
z>9qag+fbCd3tPp=QUa)xaHQhS6*IJ4FTIx!0;)h<%Qln3#m|^WmlpG$##DcBE*}U8
z7s*r0lfF4uuXbF!d-}iaGKynEB00=HsLkN&bnfPqZl6@T2{-^9gZNZk1p$F)zG=6*
zDK_EF5VIfrkU6L$P5T<72W@3urH58PQHr=VmQ5feStQFZD}xW$B}j?m5i+L+&o-|f
z*T!c~d>vC)nVR%Aa}r3c2IIJ{PFd*?rV7^FASTl~XbuE0B%3odw2Oh`s;G%U=YbO(
zU;xDPV#`q2$E#36XQ@0K0ekm*U_IVo^m?}+eV|grmpd!oZe45a9;}7>`Mq@BRXAoI
zaPcccbo$CXm;*?VXgd9I#hKTi^%}`q%7$G%v3kP!Jkjyq0|RPP?|v7?ITKHAvPvJQ
zgBR}PdM;egtpGf4z4@Fu+Dj~nbs~2mT1x3FZog$?R?N8&oSq59MK7~-&w%Vr^Gs-<
z>x;+0L!=>mCTi5cN!;iGVA8(;JYXb-A&glt@mK)NdL4s2sX!#9M=_Z?Q@Ko(wcg}E
zfA1kE!0yHZ_?73mmbnHli^b(ED$%I!(=8m08?1A;@-=<-Qx~Wl0SHKCgZUe_zz9S!
zG5S;qhB~NH4JtsNF`VyQ50(eA<zkfw$Fx*Ff<eH9P+p|>4?*0}m7Bl-@BFwyIxQ7c
zLIub)iBy@3Mnmgh?d#8FL1Xd>M*i~XePz}C8A;CRc}FnqzIv;t^c>K?f(8*xg{uC^
zbH9RI2Q~I<KOjtVTP2UDU6s!QXFaCNn1=xO%mbifC@clKmpj)wD%M_>7J;jidk|OH
zas;;9w!GH}xdPe-sY-3grFSbp0$#5}ZI_+}KK%!g4JyC%%S2JNhOswrE&Y1GtJ4gJ
zP0azDFxcz%ibH1>t$0ufm#8C1v|I>l(Bi@S*ZG;YnM<pdV0@&U`z9E0zzdQp!aXnD
zBZfo~`GwV@=$v<P8Qo&3fuP$2Q!L~8HZ&B5MT7J7etz@X>vuw4e&)4bLI_srJVD>d
z+s}Kevj7Woh0fHG8!9T@=%-t4nS0EKmdoF3;3{@`j9$AA?D7L(b)cjput5;eAH)n(
z4QO`lSvGuRaD}`~u+B`2;~{zXSuh@+VGyvux*^~bhcokb=20w<=m7{Y^fQ%!G40a|
zW*yfB46sJ{E`puECvu{uI5i9^)2Q{3TLB!pSU3tn%->R#FDy@&z7GcU{pIneMH5~<
z#O3SoJMLI*=WWdJ^n5?}BrhnAx!eyA!$Hy-lgGpw?+oH*0VAk4c7&4xEWm2X4Gc2N
zj0=mwdhh!;`F9S}_TDXdb#0JkT-;eO{(#?TW7}s8wgV$0V8HnvmI><ZRx@m&VK)p0
zq}He!pq2#_i^^NiuP#i=6~gm+54jnDdk`q6qY*0x1B5NhGkc7iV3&T(ySmw57sdkv
z4AqT$ocm^10sSR#@vIvxgE~D<Kc;YE^KuwxpqZ&K`^i1hno`9%KC%S{S0|3O2lA2b
zPBk{8FIWp0(PG;uP5NZcr1ze`-Me~t{pH64ieV~>5j%4t7i7%7a3lR!UpnIU=f{5n
zjdjA?yn7Y_fNyu~ndUj?X$%H2h5*N67684VbkL|(smn3XTm*=>2snZYD9_%fKbZ&T
z3F@^ps)M9FQar=GxA$-ii?^;JH%^3CeEAeKfWcT76+EUS%zGcwXGZ(avh-62h!tmm
zj3<{K=3V*2VBPEncshLEdX%R{Q|Z9`X0Vz>KkB>@=*XtRAaVmhdNr8O|9s@}yCE6Y
zAWkDl*7~4#ynUxOsNTbCToM_*1hF-V;9{bxIs9A)O{aI68{TpuoeA*zx=bFqwX^$R
zk57Gj!s+~I2gHK0<{u{3G^|1Tu!QEfg8>~bVA}JW<7@x9(Ie5WG4piS>=vAQl-KhJ
zES}wNc;Y^NMQTdS0$2CPSGV%Aj$N=-_Of(kCvwTF^<YujLhCzX8qkQ)8ZC+c^2kkS
z0g<3#P=I-=dX+j+F!zHJ5}P^#AT!j270gw-H_8@FVEflQ9^x=Shk9r5v6nXUv?TFZ
z8usRCip*-TSxq*p&Q!T*x}_epl2`%`c(aE-2e?m_E@A-p3FmF}pgU++y%|vD!2#$d
zpB3S!GiWW?wRXDJO8;AZdFB<)nbmr;x|SFwvvd?EL5Wt$Gk$6=KUIzsx9f2MSD&kH
zA4%M%mxn=1>b_a%+Am2l(!iQ9Oocb}sJqf>V5m~5u!d+ib9T*=0;RcQ5ge#VToz?t
z-Hn!Ze8oC%wl<j6^+b*ebkN4)7JEA0gw~}UCVxqOZBB-LPrF9z`j1EMfFX>d7>7pG
zGZ&3jR@}4r!}I@u<^lr;{2$)H$B!!GGR|49hSq-8pmYd5CwlMKR-^Zb@_+xWUv*z-
z+DZ|3$Va%~Ydc)hyWB(zWyNE0=5q_+>jJR+`F)i}$nqKHt1?dWfYgK3wXRMBtUf^1
zj0boNh*^kVSOJ~|>aiFohd7INE#<t{Ve@MUc6FMM8EBIJ3O}zLk5mVCfLeIsTf17b
zrIV>Nz&@bM)v2A2M(&mKaajdbV1g93nY86y<y|dZHEINAhy_SBtGF><8Ry;1+E=IS
zQ_#IY0IMaRoa*VxCd)3X<G<YTq)nF(H{p^LlN1ru{FTH`fc^ZfF?@v2+OP(n4(`q;
zMYLw;HFVly1{j|~CNeG%=;}{vFXh(gUq+|;;&;H^+gHFxy0+rl?JnI%cXPb@ZXU4W
zF0kf{*K=GFz?r?cMTdqs`GrJHR<U06s{sTsmfT!l6sz|q-{jd1Y@Xtv9UwuIPxW40
z6m4`saUhsG#{f2~k_vnq*62RIeMQWJRe;7DLZb@nHlYbR=fgj^y{gpXNUPL#FfyFk
zzF@3y0EZSN>srSs{iCLWR>C#p6_lDSqgCOIVkxT>7XXxl)VcLIDq~c<L3}vufB<^U
zO;jHUrWz2l&%yIU!S?Arb}g7wiYpb2MDd-dVkCk>;A!OJBM&sgTj#_ooN4IIalDOu
z#qI^MTG6}QT$~yuOdAyjcUoIPz-)Yba5W?s&j@dOEX70kF`|rDg`=SiqKaEqC6-_2
z2)Iq4io5n&<solNJz;mnsZVM)>FU&~JUYm{RW1{-6QS0}eGPyX#HyxobsAy@ahTKp
znn@MJYpXUL!O#4xF4Kf|f9~EEG#^<greX+JC$6Cq)D7AK>DUmB!+mruSX^PUPq;;n
zSl4(Q4ZN?iTrICndUznz+ZMZdeRtO(vq!5>kX6l9Z`}mR1vP?zIkP92%{&o{p4DkF
z^AQ7On@MOLCiQAHX`qL3;f(Y24ltY6?%U4cL!>Rd|KhVystnTd?b<3ZiaJ}!75VIQ
zwVX<;Q?n>-DuL{q-3nm#R?eQ-Yjt7Uve}_vqb$C&j_wdJfL452>X-yOcLx|x^^h}u
z=}H0f($$F$FqGRy&KPDlx!izkFo3Dnnw`aT2a5PbWk)oR@cR4T)Z)RT!fScFLMF?Y
zfFUL@AhOnW)&bJhDK5xoeF~ag$G={4N986oK4f{Or%&q`CyG1&5IEo^0(_rqXJW*F
z9JoKOjz-|!?{|qpzqh3>gNJo7y_CbNp>nOt-&XWJ!1KVlL)xs`nOOjDAh|lxF*-82
z#j4z><mv(T@;Q_cr6z~Y)5QaD@y^azrbs@Wzyk$rIg8{9S#K`>%Dp_?#!c0480@dy
zuDT!q-gn}BC=H!!nMik7fz*)DX5@mr4}U;A2i4*$R`4167>EXiiwe#c(Gp(F1+hY0
z1&bBy70VSES@f<v?dr4&>2|DNb$sbw=*PCZfe^8SVyBaBL3^N5`zyZ?D|mUSHF!3t
zQ&B7AAAj=$@Jqi@tJ)|<3plAh0zci1*uW11b{%#h>1DtYq;li(qtJ&zCQw=Msv7U~
z2O~qd9Ke~klzD)?{wx?L0q7mxD=5-_Wx$}ucPvhLhs&bnvOV(&SmExhKL1u0PfFe(
zeM27sypk(pmC}Bnohmszty{i_y;i++Hod?rtxlMWjXcggvP>;Oy{<tCe&?6q>Deo5
z5S<3;;$}AJ2SM4!KmGKncdj*9FIcg@0NQ_7X0(Dnx*t4;`SHn%)Ifb8?FYgkfxy95
z#$U@U2PmJv7zN(g4+d<3)T5FCW<=s<E<CG>j1n;V=Yj#vL*sn^(7x9;KkV1%#o4yF
zCH3TnwpY)kodp+=u;@sSa>6fP0K4>X0Qe{fm`*R@{rT_zYHMw*U2MW<^@a+uw5_)G
zxYtjgBLDM0{#w84tfKZfIAc2vbGOYjzX+la07jX|MWAXd8cI`>B@Q++o<N)(olzPI
z7Q^jbz6<Ou*62SwvG}Uf#!=>Z3<TRBTqDY?<A!b==7Ce!4>7g0NNr31$KUl~p#b7J
zy?;}~A8)xE3QJQyw^zR(N$1Cd02aggL#|Pf?!Cpl0{A!pc#Ba~@N>6-#d543(AqI?
zj2vJ{96^HJFmEWmdRtdGjA5Az=gR0=!30Ek+OZ?Ioez5TF&<gL$M=Y&lOsyg!85yA
z0vnUX4xZAMsq0X*pW&(3K9J_+%!aLC=8+&)*G!SjEFJ8amez6e4rA`o0hV*XsXzQ_
z7C8MT;Os$qi<#lYO;1h~PWYU-1ckFf57pL!0b#2Z%B@SCG=rLXDg>S)xNAOc=mQ`D
z#I5qdCh+%xOfAscqU1uf?yp~F`O>k;xCy!nR=l94fP6uXph)x4WBCPmdB*$SlZNTT
zUyH(P0afQHR29wJtJ&iE=z|atgUJa3^S%3=AuW5XXHH8SngUm+F|OSvg$IOngGCRR
z7^4F`W-a`93p@75!rGW8Ryk!k8U9rG0O!q%V}2QQu4NamEep&D&9`;1xS&o6{X=qT
zvWm0fWdg6U;tV(K=XlhNQLuX?Sk!PK7NubBAe$De8jE<VSFRl|3tP3y@CmSTv9q(}
zC6;9;5nKhAq)qDwg$8!o?Q_sFo?|zV4)$BnWxR{MR%sEobyR?=2;Rh=vJemFe2XFv
z>ndvo($%TY!r;M#vWOnzT+3bdCEaC~%!$6!w@&Ds8K?x6gLoxF!x5Rf7KiSK_J_`g
z)`$4fBilpALr!N~Vqe1hU;^l8hKe;c9>f^X4V7c$yq*;x43q}i0fmB=KujJcueKbX
zAy(9G5YvlE91E%jaT|Le2Cg5p0y2OEJO~1Fq$XitHK|o3{ld9d0XHE?8{^XHRPLPH
zp(m_AyT$_ozp@WpePIJlP^4?2&^TGCH4E`)*M4>lCF$?sOaQoV0e3H;4+Qjtfc`iO
z9w|CcKrhvRDnJZ~ZGLz@-OpXvUw#I>9mEp`&^a5R6KTH3Zo0Nq+B*L~zZ^qoziXJ&
zt3bIx&9`Ry{a<MO+RhKK&QmuXK?ig<sRx=^!9z|j)-6!Nr9l^~qY5o`SgaFqDSQ|#
zYQJ?$b*D=gcw-zGle**bx5T&@Ez&4roWUz->y;MxHpt&^N^e66md5B=o9O9NJy1%f
zwcks+rD6=ranbva=~uJ(F;;)N>BqVa3X_NnH7^#8zzMMa<b}==NMP_-qy^h`J1)L+
zChJY8oOS_pCizzi=3N^4dAxyX5KlK?J^<3zc?y20U@=l1Ex*W>u#*3{<9_regm6hE
zd_Dm=Ry-Yi(!P2xnCJdnek^1Q2xwm9gj@}$+1$3{aF=f=?VW`b6by3!o6FA{G7U3g
z%HasbAUdBfOMu;WSP_Kwok<opzipN~mW)GBFo0DBVEHy5Rx}*Np0&+$2MmL@)>sgM
zgui~9dkWfSeeztW0UvgI?^X!z&mF{J-L~_<fiIo}Gv)BI8ouju>A4_)HCac&@Ice=
zfB6vCfB7M>mDGR!{?GK@H11E(&QX9m-BNz>**`vh=f^iN^-0kU!+PIUipt8ZVXkK&
zRgL=P571Iw)+;anIDoFFSK_w7bgV%8;-_FM<GL-we=z&h_mwN-JW*f11h!<N%YpxV
z;wJFvojHR)k3h<+FNhud0CWUWz8}&%bfpd2jEa&NU7hyyUcRU$=&f>SVQvTE5Qm^+
z>F}NNtcDi<@!dQy!1&4Jd(ZldUT1_nqRTGc0%iiq|2F;Xha)^Tf{X(mET#<4fQ~fL
zwiZ=1nWxLOU>xtYUx0;z3VV3CI$wXb5Wm`j(-vhi13tMC3^@AWPR4P1`ip;nc=HA2
zH0U-Uj&v1xkix{)0l_7&CmF!U7x`p)<7ceoKoXq`tTxt<i3T3}-*3F}G1inyF(tfx
zJx>U*HX;%{QM3*Ye+0rG09ot`72~c5`j-hLV9J}<t((;?X#eS@-+&wSo*k~y+->AR
z(!ttx<m!~bL#+J=1dmHkLoNSGCc?lqFh(C=(*O=|y}*7+o+v3@B2UT+c6LL9(9nMM
z24ps9AJhpFgcQ!|`uRI_6^eFGa51z$|L|HP7nf*^_SLU=>mT^a%0$%aruH?Ga-gw;
z%VTnk)T;~+M1x1CFnW@QnCqB7T?}-<5yZ%4b0%&PV1l%g<$yRQ>kJSgLb$uSxTFv5
zg?t8TK7UR7#lPI@Cy#)s)Cl8pk6OWsV&mR5*O*YpxJej+Pk*y23iyBj$wRkELj@lV
z=H@34hT6OI2SDS=*vbSuRylX>PBZ`(1Hll>+Ew5J1RSmx#Jw$w34EystQ0iO3GzWk
zkbNH*5F-MdGc$vxc?y7f2U=dfeHfhkDGj7kd6X{R1;#)(GaGd9LN|~dh))RSb2FXG
z%>f&uV4prL4f{e~##6(w=}Ew@z)lN)sWTe8phB^CLDau>hdIw?yJ0aOwM{oaxv-K0
zaf?l$-AXZnx3qw1KaLN{0vF4DFpYq@c9D$Lg}X&Js1jv7V{>&fj;_?N&1nB+e+?b|
zolW%Z$co&v3E?&}ogK`@0StlmW7z*L@A6DL9lRI_I|55>Uz>yI`|!Kg5ZppJPXn;b
z^nL&SWe55)vRI74BE(qk!eoi&!C*nBtFu9ZR&E==H42;qynUybg}X2Sf<cJazB2$G
z0m?h~<fkkZI%d_%LtUL5VDy&`F2hQN))-NvU%h89Y9OWeGMFbQ9+V6k*qiHu$SVK?
zxK?$jk9f)G?mBpAJQofefxJP!(gg1r2Ai$orr*6*tUIlf%y@u~=O~3t0Ub;*fE5Jz
z_(|@Tbv6Z5$p)Rz^+fZ8dm6!XaOpra<1A>`wckJU0`xV|D{3WB^W&sHQ;U`(FKD9E
zxQI0A0uR--tCWPU@pr$cb~BH@BL?Lv^r77kVpMod*F+?^2Wa0jaK0=-`R=fwC%XYg
z0cQ*YZTD^NZ|c_1?)PKS@SET+R4Fi+T9<-YMi?hP^PAOx)O9R|8P_lUHksG(i1VmX
zPX_^MMGO#-a0JG4|JgHOJai^5S&*GMT?EbRpx>~BKY9eUHg=jZm;!xv4+Fmq5?uZz
zBnQZ0AW0*mYPLR=9^=K@r!VL-g2qA0lhA<J$d=}!hCZ%rA-{uh!hg1wF7SCrM;?ea
z-LT+ukZRaK({ZTCaSKwVIFpRi1FSN6pgHx-6svH0l!?VUse@;GU-@|=ReLL3=gG{A
zii~BXgFx*EEY^aqqfN8BVn2O+oxbkr<f@nffJ!&|pDCP`7tA^GO+1(diC{pu@(;&7
z_xxySYg4%2p>%=Q@WiZ0^Ml_&=0!9%eRQW744r9e0VUb6{NX3J+DivG(_(f{n&x*S
zv>S6TJ;l)UfczP<W2OdE>e2(+YaFx%&``dbHl3ay;cAVb+0pDzzXN&xIWXYECtaQJ
z>&$96Rq<L%l32l;xa*QZ){i5*`RFC-&=TpL=h5&S1Pd1{_@p%$i!u<jn8P^%o*f-6
z+KRT(UT_0z=Nm%CfQ$6UiU1EvtqIi|4JGg06xqu~aEGZJ)_Jlsq((0I&zo;$S0<>G
zdloQMIzIarRHpn4s2{{SDTrGzLMf^Oo9ImS^Vfc>GcC-6uOk{J&NjKLqND;3(@Ggw
z?G9Ku2;e4F^%esY7C7|vzVH%c9&-`!?xC0CI;EA;!c#eO!psL$WjKQR=J!MV#R%SN
zXrJLi44`ZkWT?R)W*bnUoRM^lMgrXvb=(fFoZM`_X_ALc`zhu??dy!ozr6eeEIL#G
znH}5Qqzy=|izT2IWNeu{aEH`V<CwV`AImZj2hvgB0QKAh3^@@0=|40=aCPQju{Hrl
z5Z5_g+~{xo{F`C~Ulwit2(rc_6ug6nQIN!BY@8yma(@3_2tVlvstXjQ2*SEmbe2q}
zgx%ZUWxO^)o?qSsu7d}dWycsFA4z91N}P@UO=uYc2e>7Hms&eo`5g<P5Yfh_U{#!W
zFn|djaU%o6*!zHVj6mZ<RmEMmi_&)KR`!ps&vTaME6g9k$RZ?{6(j8qEd-*csv0z<
zYHpt?$&Jr8CL(>G$t{(DO1VBBdpnwe4>^Gm<g>&KV^#xxHB1F<0%j5u72p=PL<R5B
zwa=daM7`wXUHqYL6@|~zj~@h6HG)9{wn3b)PMX0G384#vM?JY>=vQYRt#Sr%Sj;WH
zsULqoRP<yKYQ^-{I;fz)$OBs~o%<|3fXbaYsHcDD0xE04pie4zAA_7h1+;q_OX%89
zp%p>_e2=5SKYuVea%7Z|1!Ce3_*`Ue*cOg5ce+42d?%sCGwDNlL(}}~%mG6pU?>EP
zjB+f`7*Gis5H<2Ved@n3Y86_`iv|~Zdm+>c<Qir#u$;=xbS$jx)X&$~eR+dezqlq5
zidQZ}N`K^tIVNB+fG5;Zny0^!%x3ZQn}@J-!@O-zIdIRC=RA)lAS!Y%l99WjwgHvl
zU35{Wuw$qDi=V*&oQOy@KSDnJF-(s3<KWyC%MDN}!tRxu84<gT53lbQONt2aMv!Wm
zJ^1lc%|);qqrg^ts{7S-5I{1h&33rfN}ArDTr}W+<tb4@AX55y+<|7$QHDnzT`Ay6
z<%u;+r|UK-#`&`?=x={R9dzyI-_CZbO^iiU&A$H9*-umSi#C4$+b7Uy0j+Q9R#zvr
z)W2SPJB$e+AN6QR%i*%dd(4hO(-!1uEE|RZ)U2yenE_!ui-E0Rc_3Q3<1`R3Cqn1t
zyBp9OPCmH<B4Wp}BLb3sY6DsG&~X63P`huhqJ|dZA41Q`^I?7Zk1sq^x9YO!-ph>E
zvA4)$W4NL$8}2PGdLMmUtajyAT+HNP8t$`zw3fNM2S7Tr2HJxF=5If<r5F@8&1GeA
z?Lf8&@Rci%ik=BUznJH<;F%E6wUOa`n5?^TPYmZShXxLunP565=(m=k@Ibfi(Oy@l
zQX`r(pf&d+r9rR?9fF2If4KMd93ImN72Pi$M#Ig}%<VeU#*+XPb2%y%q48F(*8_Ax
zr7Kj6V2*2i<IW%m9=AcG5nR9!#T>D^_1Jv=oY>+x0KhP+N0k^@0<OX#z2BtPS>uex
z8+YlKsQTq*|N5P-PL^WUei0T_h9U=4GYMt_F&OJ0;I$V;9t?}rxyrmI7}SAUSPA5j
z1-MD>Ltv2;!-IVm2YY8*6=>|9HJp2XDQ@YkL3GB&6ZFonzTU?K8B&8-@bMJDXO$pg
zyue$FmmTXE!|hU54<0Ms;0&Ddw6#jHMd_R{UnGW;tsW@aZzbrCvGxn7jVxeJF?F@y
zPX`%K|1N`o($u~=E{heeF>qHsC(+=Md*hd20_{(VW{LjxlaTKFV28)U#|vT<Zo{En
zot8HC4pum&0hG8ReM9v;2qDq{=h0P<ZwB?^{a}oYfcsN>9AYfJ#momqN*GwZns>3}
zw3gv1sr~YsIR8eub~&7tEgzJ%Mjr^W8SH7jDN55QcfP2G@`LUSatHO&?=_!Z*RCEV
zfBe=xjw+5u^kmnm$)E!V>6TM1?OfmQpSuQ}H}PClkAYd#v~Tz@_JLX%Q*J>U2o)i`
z3!)ap9An}GegAg-Z<xnde#V75fF`+zBCB3-fUadK*n;vvf>~o-A2bfmTLPQuUv)B6
z0{BL6svxi)07Far)uM5N-tEOBh_XD7Q3$iE(O`zW_G>U8h{xKhM~zf#-df3;JyV}S
z0BARcxY4hXZeSImA<mXlqt;OAGvJ?|{{eVN+2-+492AFn<_p&|0GbU2142@3`x&Fc
zwm9y&JpqjA2weFl#|64#RluZ4|CG^n3}+%JI*c9xwBG>Lm;^1r)yXc3*Kj>N$h=7h
z8=WBSB`qX?8)oQhxfx#&z+;*kWPJO9Vy?PykV!Yiia$KH1<sK2w)}tqa4iS8tM>P=
zWxy>!fK?NiGxzQY^5Zr+f=gc%EBR`uoq>T1W^4p(swC<8^!sAlOgGoxUB_e?@x#RL
z-C`R;6{y&b5XG810y4tfsO?B=VdTQCoqa86>`sqgz7KL(I)vMEa+UTxO!OUKW3kM&
zaGZrubws!+XOLxqB}E9+QwLov;QAd`hi0_z0_#>$vh_BTSM!=!wEwqeQ;0^|0pAV0
zwLVv;Xz092B_W^b%_Cq%V_*fj-32Ncx!F9X-2mgZ06GhfJ5dbAoc94;`CSRPAQU2p
zLDXT*`*}};n0R#4MLn<+$tTt}vt>rw0mCI+!VE9v6Ag&1-8d0#c{PAG`H566m^V)H
zAkc;FAiy59FDf`o_@+-YQ^yLFz{i%36RY!gOTz$B!`Lz<fq~kV>128U7>v<A;>=+_
z`gw<G9}vCsJv7?OcpJbZ1UNQhtNos;At=t3)m<#t3Gs#heBY0`Mm`A+2Fl`D<}I0S
zJf#a&LHU<o9Dw%wTkq8VBOUj4#5+s+_x|$YgWP`NCog~t-uz7?lvIm~A?A;u^)v}K
z(f-SgxAM3$gVtYu6ug7SMWD)+Ic{R+?%X^FYUt8Pv;XLi{hg3hSEnn_A3%>ag1I`q
z{~&i$VwcT5_=0v=LV%&#lh?0xgA!x|#)DwN2P9WA#Q@}#bA7z4xsJu-^e6{HKTNHh
zqW@GXK+{47$pHZsJT0-AQf1{}bfthtfh&(<VguoA`PF+HqSuO8WW<8Cze)yWXJ~t%
z&fRah$47dFr$E>?q$jBdI~~K6Yz47ue{=)bRS0ZiI%|Jl8?wks`!Tc(#<mVLbM-M|
z)Pg{{^r-b3m>`x(!+8Xq5W~?0p&5HI<yoKJh8AzMzVa*S=Aw#3)j7j82x9Iydou(q
z=tt|5;DCY4tX^R>(!_F>)01ERdRsS5L)!@j>)F|%>%*mo;ZfPsm%eRIEJy2Ixd40&
z#LWPMTtzLX+nW*#cx)DnlYqmf#2Fa0xmUT%qN~@&;I#*X^&1+&ff&%JsKP6wwfukM
zUNq_V*B^T0@uh0%<Hcz_qw_9cW2uaZr$aLMS*U{Dx;LR)O}dtP?mN(v*XTc>s3}gX
zMCM|x{Y^IX%<`T>muztJ-Dre@0G2g7<+1_iXeK9v0i5}JtZ2>TinT^WbTqe8cjX(r
zAE*pukiABDf*hu{LO#BSC*>7x-j@5CU)wN+mhy=XM2CPebC}7zFEZaEZ0Y^$wTCL2
zjTsK-ALa>Qz{*wBE<MDP2>96U=AQ8tef!omEMq`-46_Ru@D*uN9nuos2Ep{>4uHR0
z`^dqjsPBIRvN?xSpG96bT6A=fMMAOm*U_NAcKg{&B8$+PVH|n0l{sJ7R4I~i0R7xO
zU;sCB`G=x|(~ZoxCQup3>cbb^4?@&I+<BQd@fhZTfXacob1y-*fC`F6SgMw`z$~=i
zr~oAcBw!WnE);$3xLK4PZiobUi-_1YpPL=-!xpp_<g#YXx{wNDX^XA0bTO2~@N)#~
z$N?CDkGT?H9uzSR)<p{ERH1yB0xVz=JjM+wl~+07<(Hxtd^-|)paO#fp+^wvM6#F~
z4>Ea1IM6#Uf&nUlZ0eWL(9Rckk-?$o-*^<fJOJ!Z-+0&%40%HT0Q(>wIRSmM*U$TM
z53Xq6w4p`i^<3NO4-HI4FTh+38U{5FGoLrC4tyav9l((P^|}`hj=xZ=YyPKS{+Klo
z;B$r!(V^~<b`-?xccriDy$m${D;U<bv&!NYT55P`+J|*+P-Z@V7<`@CoScy5+YU*W
z8W==?g%?o8gwEUqX})8`PeA3o2M7o800Xc5Mr166I(K5A8{B-r80mD-rYHAV(Zh%e
zvQ8Gc7oNn%l*$n!WG{$m4Y1;IC!an%841BOb4Z<^`HFusJRJDK0hEo;fiM2yny339
zt%H~sf}<`D47BBWilY5dIPT~4#e+$5@oDKi62WuNDFJH`1Hl~u6U>-B5S{Vb_v67Q
zgHpb-_XybH4r+OveM))6>F4GNu(Rma{-0p)d-qjUGpQC2q?0Z<v~94-JA18_+GagH
zN?+4EfA~4`$wU(!Ea)v~(ZnYW%t`@sjB7dKmmr)<MVw7Q)y?y_Qt!O@aC^Irbbxc8
z+v?<N|HFx!kA4+@iV)H?3?>+HY@O;z%t7OW`@#PB!|${kAWOI>FA#9x@B#BrolIzI
z-g@a49%nfi!(0WjMNH7MpFTV&E%4xhPVkr&F;wbyzvG&a#5{Xxdx%IH6B)*n3!HjS
zjjhF>o&lnGsaF8K7OB!4>f1M2g>a?>?NZO)-3s$r?A}<nWyR<|{Fg_+2l>}$ZliPQ
ze(t>+r2XC~H(MWHb2_G2Q;!C9fCPaM$sh*GI$>*+egrys#$sDGeb3!FEEW|Qx`anV
zU|td5Z_c7Bq;tHT``TbKRafLMoc;(0n<^hU&^tc`^U|t46d<J3?1S#^E-`X<fO*{~
z+Mf|^g6&{wCAX?r`dRv+jrgR6LiC3-z^+ccEvy;=^p479vz7@prNEZ?n3v(FcHy!S
zg!US!|Bz3-_QA@qR>--oRPE0X(~W^kjR($<z?Ux=BM%1mEsv!symb~b#W8A^2QWLe
z6g-4$>+kc93p7FjwBORrLpLmKhE)5$t~Ax&(O^y;@<DU?`(V6GU<l+9*HH_N$A$zL
zL4Nzg;JF}`o<VghJit1*%=c~s8)5S2^)7?^&z%2l0XGh)+n@!gyl-QU)R-41*OhP|
zS@aCgMTm#3+0|$>Z3NZahCj<dHw!(GH#9vDF)1x@?awGfj>asq0|-*3*K)4J!+y9$
z9c$^+xp_s5W?fQA$f$dhl~1)?p#7JT-7++obxDa=?&af_p7M>ea)5a4rNeIKi+Tz(
z0|G%~+Mm(&bf|((n>%QYV=|(?|MS<+JlhS0y*e)3?5*p)3x3c`dj9j5*DXTvVs3|S
zO1GbR#iew6lR1HgeB$LA@YOzDwd9>lD8Z{g8~o@lXB#xQ0cO{_bQ~p!+*;6jd()(;
z0^cdE(Rhx;XQB%APFFA!EqT<{N&9^$mU5j5f;$)3TevXIM&jig3Y;(@Te^lIOyRcD
z0B1&=a3z&Y3&xj~#-4L5a>2tO;L~SB*75pz?fuXC{Tsvv&H++DFW*D27cur)Mcjb`
z81`HS2Ht%HTHmWb0Y3umg3f{q`?sgQHBl%=1`M#Tw_!+tMi9rfnFC8`4phG9{wtPj
zv?2^OlNrs`FoszmAfu!AU%&nM9M=cTYbQf=;o2{;rTSa0umhAAL?>*yFx4(T2WFR?
z0S<Vc*u;dT#n-Wx@ybi=74T>R%vKk!VuDAf3nWO=HO4B9TLHKjAfv21QKd{3nydZH
z6F~bBVMy+|*{;VoxG29oAf|_-S`2_ifR3iXbTF$sw&K7!%?Ou03o?QLX7B_DD*^FY
z9RmUe)O-v|DcP3T!IcvyW}x$S?JqL_j~_kp6!btI)%xiWP9+%N&ePduyp+qKWgFD~
zSy>n#phyHu5fE2Zt`c~3wPEbsqaoZ$zcT~d!fZVKLu0J?A3V-91afRG`XPp6r3pUR
z*Wc(OEyhPp_4>oUed}U9givd#<O<jZ0=aO$Mc9B?4hHK=7RyTLY2}NNeom_W;hN?R
zXuv1;WCe)k2=V&|zXKi#dgpd^EW7tD7hZV@mVbThci;jp<iZnR^xit}<a+q@Yy*WJ
z^R#mnFTV)pXRah%p&+}DcL}r~-`t13m(7g03OrQ;?{1MTBHAx~({~;!!z74&?NL!!
zJ++?>fMgj_&SfH~F`(&4_CJB11{wviO3?!c>0m6)0G$=;JbvYxStGP?F@}f$FSgcI
z?e_+k>dj8k7@<DhI9vgh(=o{ez9KBn{{l)GUE>&Tq?b)W+ShSa(!aI0*J8sRv`V{t
zhF(sv_~Mrax*X_WG67ZtYEA#oOSiD_<$^YO0D=m^xC_8m#3n>%H5f+?_J|TI_y;$N
z%D^(xfZ5s1JZ=J5;VM)QgIxz|=fUO#b3V!)yrC-@Oa=M(Pu=2wj3&S^0_wt0s3uUc
zSk2nY-)*Vsig$*ADKMmcrxps``N#Vo$aH{=VMkwMJfjW_ynnkDL@XCm;4L=rCJ?7)
zcyhzQm@q&_A{Z;r<?F!?(rWBSAy5Cjpb8Q*71#i&<&UooZ}z&I*lz*aUTgrLc>R&z
zU&CRhRx<)an(XX3nde{?(}q`)4uX!rGeH2Snh+IS`=3~lySdsgSF(fZ4`MZ--mU$1
z1r>Czc-At>sGU3Lz~Y(Ba;A<kFav0h-l_r7yW7>ZJ`hvf_9}C%wIPrB(bmTywIB9W
z<!2Rb9n2$uVP?oE0hWLn^9D}DD*Vl-&ipY+yCpF?M?u<Y-;q}(;w!V*sw8K81VIpn
zUC}$4bMbRvJk`f4?V++_kJ3f%@r-K0>6gIJft_HHydZJOj}vMaoPKmKy>yByi+g3z
zolC@?uU*U29Ajkevyz{G8f<z2Ob{-@ximGPc6`Hsjk)mp&-HHJETHSmtpxKn87d%E
z?f=a?x#mhSUd1D>YUkxD_rSjGvnaivKM%?BqcRuT@MpJ3AGr2=kW$0rPx#PjoHXxb
zGSq>0R7SsbKNBVg1b7AohPyhkHUYdP#+v0aUm0xTSa>E?PD3>^uOeRmk@VLZqeWrt
zyG0Fe-hwh@71_L5%3eP)^8v){s|R&1Scp+Lqvxd?rslMGF}s0K-42Br5DT(r6nK}w
z=oi<`>*jS!Gqe?H1viT9D9AiCd$T1^tPINP4wz3GH%3JSYeB;iuUh)|m+pbsfA4PT
z0pI8a%gMR;qFA=nTG46<YB{vtI;z`{9&pZ?P^p%dorG49>FrxR2Dq#!Edl9VwV8&+
z>Fi}R?OE`;n|QULjEA&cbeF=m=!>^+HlQ_9{HGW1hot+vKzR<n;5|i$>QXd-A{Ab!
zYMzoKRS4Q+SbGK25T;8ns5C>PQ}2Qj3UXA@p6#>#%S{i$5W%onG}V|-y7u|UB&?J4
zs<A-(l*@yOg^#zv2MQVbB3^Oj0Wd#q7vOGY2f>>EbnU$m)*!$>sDeQU%m+%^EJ*b2
zYj@cd77jZW@ciYvNguco6ul|UBIbJ5DCguJ8B{@wIdGPoCJ3bcu|0Y*9yAUrUGQpV
zO;p#?aj!h3R%5P@d^Q5TR@1qnai*)!Gfd|mk#1jgq?Tv<U{DP=o}5sXxN_a>C(nrj
z-j<}E5YTy%_M5Qx7=ne|CUA9zqM<VWMJD_40?1kIlhMf-sA-_H;CwdAeaRMyYtR<8
z-;T00QCyA&&zUF49|SPME0gA>#fh2G3oP&+I*?{Q%gt*K%(mJ|`z%*?l)z>bdkciB
zPukC5ps9+5g>oV+o7pDgdb->oJni={T?@^+N1BBq7(4G^W`RA3nKVo$IL<lJby4QB
zibVPHo)bE0N9Bqb)ppQ0VY*I{+Q+$c!FWxQg}NVx@;9#Kyiv^ICc4P3%R8x3ZN*0W
zb#O2O4D;L1%@xXnr~JzEDiyy{4VNs{V+7yBW4w+mOnff#STY?2>S%WBhBVeyflA6=
zv^K=jOHTtwHX^NfvAQ=PHT#Vh-Nml+HfY?Rng$w|&i(TwRD;f9D9{<rS&+HVNIwt2
zOd?YW@Xpcx6Zt$;dW!i##|jFKA~`}fNGlfHf*=_}#Ty!vRfoNMlL?JrF=Xs;Z+2Y#
z>4!qhVj!#xOdaCrcb;wLdbQ6j7dbeG08&IucR{LI9H3sf#Z5ZEKfOg)MWG|<u&jUq
zpR-++>8?(^ZElH&Q~SA@0=lB76^t3plx>y&*94814uO_I`5?e60L-c<&hpa-efdOK
zU_7|QGoR^JRG^&!0xC|tKD#lLS03z-fiLxjeff_IP=cA2I%-Rm8^of5?2VQS$Pv^O
zbS!<s3);EI&@8YWP?k;EG`(z&c2C<L+6nHNhDcW@4$cscfPi*$v=Yz^1m%G0v|qE3
zQuoHKGiU*spbpT0Ewe(Mu3$%|rdv4Uq5XFsW)Fb>Ch+Iq+ze#^#L4#e=symHzg`9o
z&;wk<urvdFvqquCfRgON=r@6<S|PoiStU07Dn(IBPF$Vppmufhj0e-xA*`@~S~!zg
zhsDbIn~c)ZQdkwjeweuH5YW-C8{bb}zRP=}%H3`0y_+hS5IWlOA_~K{`26eF{_L7`
zlyZB54f-JI)hD#QF`>dh+|ltSnD+J`3FU9XvIOP=1AVPypSdVt=p2vrCQ~7`SegM@
z6GbYEf_T<G{2>_A9+(yZ?#1j(>MpWnRnB%f_rPvNqjVbV9O!x=O2i_40&ooa1K{|D
zp*!_VGH&wncfbJF0j8%rh;yk5HySo;fmQ@lioxNWf&}_nkZ9g3vkRc#oIx%@ywQ$X
z&BqPRKu9PLU&`FDG=;P9S{{JyfqX$1Z>y|?5EOB3K;iCNbc(A}qO`&FpB2)63brJ_
zdNL2zP|yjp+|`LS06703SmZGeh&R2E>;cB*arGDP0|N$lFTgbJDq@C|gOpdHefZ?d
zkKh;h9zw10ndka4@C{eqWbjpKy_?Pq1R_qrIzespN;gRGho3zbzP5Gg*K|J#VBKYT
zZ09+n>qmlk3JQZ#nPyfXLENNUEU#sznmr(Wu1?9KoaF~m>C$9;-@5hX>mibo#N^iw
z>DCW+;jU_MrXmwZou|Kk`-x$+td76{dTR;9T34h>N7;THT8p4D&VBBCU<@C9CSa0q
zYM_Vb!f_jH({or%wIY17Ew>9;e)PH`_A2HLpnWX@IXuhYwS_aYvbHQYAebn?L>Tu1
zY`Z%3TS9tqGfQP$@E;!j#fh&S0}Zgu6s)o_C<Yd!0#H-%(ChavaFakgy)NLf@zPtO
znA4s4%3NSfv|8l9x@0wN;^Y7eM?_qsH1-%po-CGA!DrVnQ*1$XZge7$#o#;h8DZ4`
zrZP!8f@tP~R4y)<a}q|s{kb=JE`Y>2krg81xxur@XI63zY8}<dgN21?hJp-!U;=}P
z@N!c!6Z4fj!IaCOnHDYcnOK*f=m+QHR7Yr4U%lRd^5#?O6l?xTJ=+DHHKmr@qn}4u
zALvp#u7~Az*wtxtlJQho46Q(Zv>KXVMnxlvXL^jAQ>8-rVY+zy<-4=PTF~Ks`3CUW
z0C(_FP~|R<oBanrE-wy(;$1@)S!@FVaA$zKEwkC>%ij)9Pze>Qa5j>8`A64!SH4LX
z0Y(w$pIljo#`CYz_|M-~SHQ#_vbt!0HpkHCzzXtjw>$?9%um)OZ)TN2n-T+fSVV{D
zOjdyV(Gi_ru5y$Iq)6=X!*;AL>V!bMd<~etbb@D9>uO(RnCNfcancGcC{&vCX7Ucl
zEU#pc`};VJ(NfiM75ClKZ}zk;jQ_v1z4ezJSC%d)C8si#%*@Qpsbpq`n+!^3CfnsU
zm+f+y8OpW{WyW%~y_V2oTb5<ZlBHGM)z#fK{p!t|H-Er<U+j}{Q?BY+YkrvJZ^PMf
zBI2A)aZ|p4!_4GW5KrS}JwZXd65ui@vH|bI=A)gfnW)O6wU`ah--Z=38Ck`RdtJO2
zoOyigY35MD+LC4WAd`h1Mdf#>%c+;X|0BkIpndepPY+qQUwIS?Q|hwO<^O)?!`-kf
z=EZ<BX-d?lRUt6V1uB6;Q;TKHd{GOp8C0&}YJ2?UxH`5iBUbR$k4NYNV1~7&E||)0
zsCu3pWsb5WTtJs!xbV&tWDp2wV?G1H%Gokn5|JmOJIJwy{!sU#_vvk}KnD|0u(oL-
z3Vp0YwHS2{XBS?3Vo@truF7@PDWr9geQ}cw$!&3}*J%;TJ{|<NKnw?9E}~xzJzHd~
zb7LmzCMdTwi!l92WzxX|Ir>QTC1W|@NF=Z4_4UW#Vf59Wj|gH!m$Wv8fV-`)8PdFC
zY>nw&534VzOs#l3z(0{m;ag%b>Ckp=*{pKInSCN1q};R)wU4QG>W*1!h-6UxELbjx
z>&;oZ#rvJvhax%M&mAnYnfu^PH_B&wm_dL;H%|}Oe((uOfgf8DP~XK(Rd5|~jN7;~
zFMRC}o97<!Vu|TU;R%5tCX+2F1LU!143)J89Q#cncqR9L?Xh;T1?10f7%Y6N@)=ct
z%TmpSsS70DlCsg2oA$eJj#CpN81eQ5Q<;A8QIV^>0iU-5?VV<(H-3eYDO&?;LWTRM
z$7RZbdcZhhIJ~1SE{DMLLFLT4R59^|NHD0guD)!m$47OTW?1<8hJK^B*|UL`VUfbE
zS#y9jcXn8Gotr_Mp~rRPj4=xY3vrO#X(WhCBB^qOwgCG1_aorz-e7egz;6!h%za>e
z-`yjowM{zU#7AJQV)^0OMUUDMt{L#L6ba5Wm@7!h2pY_Yy!>DTMw{Mz1td8BF0+oW
zF)lorlP0#}m1QQo3%d!>9;y@>s!169^xPtNx0spDxuQm?)R@m=H|c6K<D<uSma*KS
zKSM!#!JYb8OgpCWQrvR29KCS_k_-Osy}x~UhuD@y&><$G_RK>bJV1Bo;FE=BHT%ll
zDl=Frp$zfJ(Kom#D_(;R5HvwBR6E(a$v{`l9+L6Dyznb7k1Z4EcLZa>0A8Dp3?eVb
z>ev7Qu3<w#5aH_X)+x*?7DxxEEuf862dMD3XUY(q4)g)3RE=Xoq=5Ef_j|5&n}sps
zr(BuCf=frrSD}sThF;zs@EruxK2ggh0VhAQ|M{^bC@Sq*=w%%(y0$XNNz8p$77?qL
zgC$T}Uk^C_C<xBb)szLRP;ZPTi*%Frf57N|SCALJabG1`hDp54NlfxJK&y0;q44FB
zLQ!r>OIN2{?aS;}4V}zNpfAtckPK<gd7bQ{z`m71^VW`SCh@@oJNxpM_OAxEVz7Sv
zGY1G2c&2|CH|fV1YoC&ZWMu;_Y;Q{rbdvVq2!~R3^j`2-m4syjJ9G?i`ZchSsabVh
zF$!N?&__v&T%Gn_$C;20_Tk;PpsB6M%(iIA9X^wob=<<z058mPDPd0;WDx?P-Xk;X
zDg$^m^9Zoy86Meb1XowJhsyM}WKYtu%E|0XfNfTmag|<t1~FRPTiAtrRE#jV@);OV
z0Sc-Sou2=D7<7{<qvCP%*%-QSFN%iy>G{qr-I%h;D$*XT9gYVNurLIn>1$ACC&6+-
zk)Wo9w0aN7A#P697c_w#M56Y;99SwNobNz4{sO8#bn5^#M%LK3=v&4CU`g+VM<+(A
zZcAraS-cIP{eS4ioB<xoP|pW{Im8i@uygd%rx52Kb852=@g1*(p2Pw&*gDbK^a8Lw
zCbHeJcP9+0%jBic-5?Jh2<7#v$hd_WuH?#tVBICEHDqES&_1CDHJ3@9pWnX>Vd}y=
z3KEs?OfEdIl*6qqoPT=oFr+moYXoeg9IT!90CpRjj2bPiSWHzc_h$Ca$Zk2%3!xll
zq5U$*2$PL_b7WFgf%f9W8)-Xvx<UJv=6`wlbNsTwg;!uzZh*$>X_{bz^x-~Q*o|U9
zvmozzV+$&tfnjiIT}vf)KVTiC?!nWzP<w953ItSuxMdYlXk#%><=j&ZBbGKB0B&42
z4<wd?0j!+;%m4Y$Pi#Zc{+-O!QjYAta2&!gFRI<1SpqN=^&sUeXa)Ph_QJzNU7Y2`
zwe5xW*3{0QKVuZ_@_DbZe0MenAgZ#2DGbQJO^!qJQK1k!gR6Z%EM!x}`l?vL%g3}2
z3H1g|V=xuKl+(f0aP?|qnMWVNfJM0p>U4-%i(qVkQ4yS1r+k^pegbg2XMPXXEMo(E
zZc&{KgmV+~Q4Tz@JFI0i@%Za5gIzW&*B*x9_*+pG!!0te(-Tr%o||hDGvmu#Y%1K=
zLLcfPc5qI%D`EKnp|Q@kqJeW41(QAVPLR={<)fd01L|Z(mCp(7uh$~>23vbDAlf-c
zd+WDHfsB1b2)}WUp(wBJySFi7%DcIOIdCF3T*F0M(03P(XrDLlhZ^<u7vMUypTJSe
zWD2_OfHi>1`(_d$=z?pnfi1Q-uBmJpb7hKhudD!oOJe=RfRw+|p??2iEI7dHH+G(S
zTt?uhpJNvcmvOTLdkkR66v^B%sBl6}Vr`{zXP2HDXd8jF^#wckJJp7uS9yR<7#}=b
zs4>|G4xhXm28N(!R^4Nk>+Q&DErjT2)d7slh5JS4wdoI52Eglg^I-?jEw{ek#Nx_h
zwq}jBurGm3Kssg`*46G7-F$C3$V;w*I_tt*#+SJqfV&fD@2+91bTu$+z)I$TyGpHM
z46i(D0lpDz5qISyD62nz2!8Al%e*c~e3eHp6W=v%1$@sxyRB76U4I)UdPqlqhtYMu
z4C2NXQ$UCI?GT0C-#i6N{zfl1Sp5CR`4C$mF2e={L>j=xvcVWbE@TnJO6LT&Ghiw}
z`=kl2?V#2)bMQ%!OFd(D{ywk(moB&c=KCE>%sZN0%-WV=0yb3L%jg&U<`bD|V!FU}
z3KlLt7s!_eIy@Fc<$BoJie$(%s0=Zv5e5UGBo<{Z=;u4YfJ?W1{}2Qd0C1X9k)2!g
zW~u<}xurR|8C0ixz_o8Z)Vf570mHw?224LaAolA|wendJ47#GG4o**N*K~1)k=e3#
z^-l0v_2PjAZo>IJ{p_0zs>88+=ZdZS1EQdhs-DiP3L3cz2AduG^h+uWxcfFXQ+A+Q
zQ1QUQ1Mw$*4Gmz+0wx=j_fNhygdt0Ni6=S^f^C5S-ZUCiEyJwPby0*Hqc@lxjqgsi
zVJ%u%TU4x<K`Cyo{`R3*9~VcucNVnG_;6c-(F7*+9O%?@U^}4E{A_kNm3+{EHqfCG
z6(TWtRIl;|KnuG{isk?P!2R5tohmG-EAe2!de4vVh$egug3I2nXHvK3fLUj$G%Mpd
z+&pu|qyemFGsVZ7BO2xZ`O^WG@+Np_yVbz60MpY)M2VQs&sv+Yy~SI+AYj$QQWE5Z
zg+a09RRj^m+DAQ9ul)Km{eZgCV5SR9i*;7uOxEFEuuJbde)AdK$9!TrFb4rs5xfPk
zwaVn+e1NI!Zznp<Og~<4FPNc%)m~d&hr*&}5<Bi^@G_IZN>^287O7pr23U(=RxyjF
z+v@3R#*TfJEz-a>*D>t@RRqvR#4t2){rA23&~ta+2_$y#5}rtBD9<pk!0C6(Racj>
z&U>_X-}!UJHrd*zYIB<9V8~ydy}L{MCG!rZJ5bAl;wrT-YMB<J&i5G|0~;?W;K|ub
z^ZGbRw2chi0>u|p4&pU}L3En~$b`Lh#S(1w&?<FcnZa5Xoh!1QccR-9dOfbc3^|?)
z#w=!nZucjQR@Qey%jx~|9nV7=KzSg6_7$Kd%Uc-5$ABegCv#n+tOcMTYVB(^v#*x1
z0T!%lJ-p->b}-^&Cpf}Q1!g!%=75JUSQiw8u=2t|EOEZ3vpxE2<^|BDs=2VVCv(o6
z=k@2B8zU`~x<|vHFn3hliW$6_dx~ocV*}A~p4HA55c%B1PJH}!_$nQ><T$rel@7)1
z;DJ;KfZ1b+cGo@#t9{OervVIrXU+DIZxJ(Ro!4Pe{ppF{1dnBKvs)m*hxcb(LW_-c
zD~E)u+|^AB&~})ZT+pcY1AN9DiT}4>f5ou#q)3Za@b&sta9}7`q;UFDz}GUJB@XeN
z)&iQg2a5^`%8Q<W0%Y-ms+OS{BU9`p6J~fulP}vo(@00UKbP6-PuJIim`lK*h(d@&
z?NcdPPVw%{8D6GL4o-<!^o1-1t;FPcLo8PH9sg8K8OGRH3>eHz#OFe2-{#Z~MTbfh
z8Hb2C^&7~Z_Et-Ci&*PUrx+_gb{%$%eITwTG{t<V22zlwYrF@b{o4!kpVcL(;$72N
zmigs*3{w$krE2x!SFuol;%mR7m9KtzRt8A>mZAFPuFxQ6N40KcNVSQO+$A&S7bogd
zjMw@HCY;yH5YzdApcQj4LkI889GGta1C&#tMHproh)j^pjB*`A3m7KBMn&hz%<LaO
z0h{hFXS16IO*wx13~PcBw61JF`w2_Y0wjoFN(_K%wZDe7S2}Ve$e{zuBG(8o-`tl(
z=OR=>d~^Q@20qLk!!h1j2SaxLLm4_V+oE?@Cbpi7pOlC;$`o*O=3;NYKgCDL#CK0P
zvBB(sroxZD0>LdTyQ%!*vAANoS^K{x;oMFZi0%N!l5>)|K6*x5LjNjPX#!$El*6E%
z`^Xa<U^a1ABick;Xw@L##4})LKT*>hT^wsLE~d}4wA9`zW97X2_eWE8dtZ!aXV}qu
zdNCsb^a~`j{nM2}E(zLSE>N}SDHGjb3dU|_Q8OCk+gZZ%1EUIpEg^ZlG>^H_#hcSf
zg6YgsC<P$14MqXbK0OH88EwIu3}bX%m1~??4S<<V={K+LYqQSy`YtgD=hAv2_gZw_
zt&YVtF-PYJnjn~QKp-d{lzf<J*Kyc8ST&W>O2%r>zqxc?{XGm5HLKvPyUS0eYB^U0
zz1<CLe_r4+1fadaka6z;rKS{n_65lS;RCrM8&|JC0Iiviop#1H)ou~oM0?p)$*BGU
zd>jL;IH1X*Ey0goiAZD`UHI57hPUB!jqO6vW*y7RTk>jcW>dZpr|$$i1RDR-$9K}t
zR6&V}VC&qP{9X!_h>@H+PwfN3{bm>rW{!Nr6_5Y;IXK`E2bK{H*2jhqP)`ibo-T<r
z&CWXakQf{7gSGDf$B5}r4O$0j-NtQXsvY|9CK$kcs0WSk%DXQ&XQ1cif=x{Ha3hHk
zU^$?xpQ)f4{i}UzBMwDv`Z_}tgUUb?3<ra){mTmPuof}e2Eoe;zWY#D8H^KjlMXN5
zE@DB(hdunO+I_i_FfqhBm?syp06H4Md<y6Vpr#w_%qu2!5FRWnfQtb*5707*smM&(
zV0SG{Q#XNb2`XkRSDAEOEGw_kph5`sfWPt>^gd7wlU3PSR=~N>{J>MSR|wn>puIhE
zB5FE}m(#ApSAr*=m!l~VuFNEJv+v*9E)Id_rPdO&5wH>T<FhhvIrYLzPEbn@PlNY2
zv!s)Ky1<jtmcY5{0_KvyxSjdSOrGoa430A5-0{c~@6s1llR5&)Jq&<Uhla)^0Dk&)
zE4Zz?(j*7KSTLAMJIj;zi81=>)1tVtM+XmqF&NugeaVpyWrIk}O=z5iWY%#_0@uq=
zHra?um+P2?#_+LIRG$0oo+nNQEJIw*zdZID_ymY|v7ZM6w11Rl&I!UG_z=~>M>5ds
zFS1rYV17~gJP#%kU=X3wX?~x27Kt^so%aS*C)$AO2XX`z1rN_N6M1`Jy{}JQ1B2zv
zp7`o5Cgm2Z6L16BiMp5X`M9f?RWO+RI#`|n_wj!{@a7+qW$2Y=0mWAa;2p7G>ey7O
zO5ky>>8E#P(()1m11yG$)H%Z-V4~GqYoBbOC(=%RtRK>q&e%@qy!$VZxMF(dQ?YT|
zT0}2C4$YZG1%%U`2`k}Q%kJ!40Pj3|<?T%fX6H!U51%l~^f#N(W-<e>HUkY91sOov
zcgKrqFOQGnY%Oi8dpzZc@9tW^cr2rqOHdx=Wwvs`EM?xwhn-G6xcc1(81<Zgr^S`w
z22On`vU4+i`sC-L3t#Sm-0~Zdk#8CMez=*8_VO#Qdtt~;0X0|N0q4GEcEIfNT~+%r
z7AkXWN&7r_F|<sO<CWJLTNmDnJ4iZy4>+*MVyOfztBMQyulw)%`3)GR7RGw3AbI-G
z==BHKo&`$=$n&p)(V5#yLI5f*i_y+9)_x0`mjHf#(I?-K1ceLEVFenmi2*mR%F-7q
zLpulp=6HxUrX!o=^<T5oENeenz-L!k`-&$7z+L8n8fwPqep(M6P!9g_GuHV6$R5-H
zie2aY#$2!qZ-X(A%%y1y<&8zQcY%WlZ&ijS7!H9v%v+lGkM4FKS|byjOuW`)wnI*n
zC|r}>y7?-^;3V@W)f+5$4y-DaV{SM6P`)=CAR`1=fdVZj*4lZEDf;?w7#PrzyI4X;
zx*YNs!>lXU0?<32Ontyv`*LiR{Vc~TZ`p#=`!P21f2*Kgc)o_Krl*)U>VhiSU|@d*
zLO|?Wz$90}^$N6aRL*smLBh6^ejv^J)We!%lrv3n#7930B~bpqNitUiFrE3(Z3ykR
zyYzm#7};w!;^;RWj8Kt2ip+qTbMARD?7p9h!EZ=pbOmf(L3WKYFwY-LgXSV5@RELd
zH+Drgw8nCe+;gg@x(Kg0NlOo78NJvFe&JQR4)9yzDvC3~SWL=MNg=MI@ffCN)4M*|
zbsjl<tP7M=n7&?&5yx~QZx_I`0N#!d^yrIsfhU}OkIQD_E;g$hvkq)Wls77Q7_b0c
zdN~z57{rSJjfbl$V@J5U#EEKGuO%o8a^&mZiJXV<+S&$hC>mxWX~A==9F6$)O{OT&
zK2_Mp8cp&oG8c>X3-$GXec)B*3H_S{qVs_I-F3ycVdPlLO9O1~0vB)U4(wcfJn{4h
zH^)WWeO#t?cXg#gj9a6$9~J9{dgaLowk924zs|dI$v55xs{^HaRW>n1^j7M4<*5f6
z$Dwr$gLyTqdZ?9gTUiUm;JkNi(wK@|6r;eL;psr%iQn=0C>ZWEq|)yKo(KeiYI#+b
z81VfQU`?R$Zyy!Qda47TV-wu671vs*z0U>8JP6={6xC)-JU9d3#v(!VAYe7MSkVD_
zK?`ri)|!~jAVxsu-U&^mko!kW*K8n{vWh}mG)HooyjCMU0IZ+61KdIeP7n?D%)kLq
zc^4Qy&2z}D28uZ!tc*@^{UmrGZ^M-e81)2EZ{Cxa3vIHoFeVJrGZBpLBbAz&y5Zk=
z-iFGhx|y2s#KOkLg#E*|Zk2w@ojf*HjW=V|R6wg$50jSs=*4`D?dQO>kC{W}If5k`
zpLrOKbGbl@<B8iLE<X;&_5t9M;I?#b4=Ay02pnKu0X^ITR|Kfn8l5qvsyxbJETGqz
zEIpq4-HZA9ML@>^?VY~hF`yikSWu(e7&pKo*TIgb@XGTbIz*s#C4DrnZq3NMh;^vs
z*gaORbazRxSa=~vb=!LA6*i;BTwKw9Q65&c7e=(ZAr<`0(_n{O!GNJDmteE2zk;^=
z4(k>-&q@q6FR^L_CL(yTao%JZ)IuYtTzZUKU7rC{dmi6N!~m%Ejmd%N8gb<YkNpu$
z>!*+v<<a{f)EUqO^~UNt1k3R|)grh<fbCNdNi~4v1IA9wsCa<1wWmhVS1db#Q}KS?
zC2P&;WzJ#+Z)Sq;g?*g6H^$=y%=PB8M}i<cH|aN(U|H7CC*ux-Gw*)7mtkO`wUl~P
zs~mIZ9N<T~n#IR%Zr)=Gt_2WMu%v6+zqIm#&bAI*p2d|%?DXHr;H``4dzD5PK0nNj
ztb*txk}6+nEXyPiU{+!u>&VcDIwXVBIe_wEWRaswc5YA!EL<b^=$Ehg4%+Sytuhz*
zLg2!Stdm#{fpIRpn*IjDI+><AHu|An{>+O@_~s=)vD01j=+Gos;Q##n>q~T7lLdX9
zl$h<xIC3F$H^76jbgCEq1^Ae#9hq~fd%<NFq*d_(<yP0<E?@}yKuNA(f+QQi-RrBX
zz#hXBbAxyjTTspVELCm$2M=hEt9WPaqxzi3_0V~}Ab@M2QvjxARLU~<0rVAAfip^-
z%i61jqt6@f2x~cr;i1D!QUe)4b5Mm%6TQ3USF*2@RWu(JAD+6yqlLA@;Knv_^Z}Q(
zg9p^sEOE%eGc_kTKNlLeXmapCNh52sJSrTV(ekU{L4opa@H{V&6IE6)%-fD|Mg1|s
z*_n_5p~2Y@UaW22TF3rqW~ZRu3X*;a1Qnh9K&*=pVY~SwCV#r3k*NnXv$Yn3!f2(@
zhR$(mo(2y}53jR>jO<Ix4$f}b&)kYxTh3#?00|5Z-OhFqK0Go20dSS<;s6~F_=+g6
zy7)S@g`_6%gp6WdI`rUy(~l<CJ2cOaH1cvlWd)f^>Upv8nu3D|`0@eZHr4gTkj%La
z%n14n*xhzOWr-Msd#=r^Vw16Ic2PW?ebHIW;9SsT=Ae0ldU83|vta@`Yhdy8m?{Nm
ztR%h)m|fnCfdJY;yI;9VU0Dx7?)c!#Js?-MQ^0p$9q*SJY)OEnkH5WWe|hjB26wx-
zPengb?1RcJW*E--t93JbuH44N+i+$)0Xuq{)nFzcgUS*!c&=!{y5aP_X1+3*(```l
zEY@STcE9RrqkAho9mX9HXJu>eFS!0FRA6Pra%+`|-ph2!v)|6K8tTu0+O0zB6NCZ4
z0P5)Ln~z}FJZke8{t7O_WsJK3SQ*_Q&jCly(LO2`oCb}_Cm1ePdPZfMstFa>eVD-2
zu7Y^gt?U@(R%>HWlx2UkQZR3rC!W47l?4rOG%l)Dd-lEw|Guq*2Y3M<2$ciEJs=!J
zjdd$iwPN1R)NwWC8T+Fdi&9404Fv3)2<w9|(~d%6@dMfddHn@Ymi7XWBLk?u(YF1@
zZBPJp5onBaG_PdRn$s6*+UiH$G{&?q*G9YaL}uR<uK?_ui045yg0{q>m<BP~Y0fg$
zJ`jKKzyt<Bv<PsE_I}af0agM{B6uE`)5DZ(5t~(etSFIvTc+*#%<idq&q$R8>=dq6
z?F|sAJP9}~VnzfwPhkP9Gv7<eia~(7s|@&xBaKew<1I`3K{`%tnfGHx*=XOj55+vj
zlN;wX)h;-TO%EszY77~h-u9JIRmo;!2-McLwiU5}mxcrw+ew3pJTH;j&|)ySHabDA
zcg|XcXg8k4+z&3!VdZrOkvFQIVp?9s%>?xvJU}1VJK86VkGrL$q^895U3gC=ACq4P
z%L-TmHJQY*2?1;j48<YtZ44yX`(yEp4WOzJ8ehEJ`tMoB`vfxAyBon~4j*PV(}#<7
z=bwjY2>=UNA_sPqH|k_)5^eeQZ=tdr=@p<%1U=3Ow}Jv~2K*u|+}gdMho~(;ZO_>k
zva`^X%?R3tEUV@kh>@Rla$qUPboh^NSc40qwGUe**%!wIoOoG#C&0l2|MkJcE;!1(
z4Gt{;XFCEs`N9Bb4hjej;yp$|%Ed8-`y*YnFEQPgNva0R$}vnUWK_fCz>PaH>Fs{w
zvMbbbSbl=qLXoZ7pS3K6tY~_JwJ`-6&ZcWQiB)GYK7R5GNT4M7*ryN$+E1RY&$=t;
z?q3fHDLcv|O)`(`G06zgexn{LpCV&9VA}UUc5=<E1TM(5&?V8Qf;pyrGCCKEEt3SO
zvG5QDJP4FEPyrq+w$SJr_6(puhh~6I9SYSR&5e9P0=TL5M3t*+ia9YfG_x_fDW*mH
zF_zuW$)~NQucc!`8Yr$@y>6SI#ptk{)*8ZAPuBqXwq1>h%E>viA?)f_awLKM5C~y$
z0zMfcg;$ASw3wn=L5j&aUnDsFap66%p=huQ<|H$o`3ZdepvYMp$e}uJt(-muuDmrD
z&V;Pk7z6jLrr$>Rz-MwMfS7NB2ncgg)9Gl&jQiuN;z0nf-&j}#e*IZ>1L$mPOx2L$
zX7#A9ogUqS0qF6bZC<Wisa)Yz1gZt6W~XYe7R$p=Kc^<k@xQ(Jswg2`I1n_w6ER`o
zyb84z6z))cm8o^`z$l%}%>dti1?K4+Dx;tb`fl~+mm4)T+NVy`#^@Ry?AW9Bo90Pk
zS<w$b;jPzdYRvi}=U4{7fC$dJ{P6_1oxfOw`&yN$85`^!b097m`1y0N!ZSx$s#!eI
zg(bEZ9CL3#kOP?VBb)9>pig2TS8(Ya1`pT|GUw%Isemrr2DS-mUi9#VAPEj0`1z}O
zs99Uq*B(qrhE@owO=6x7v7rLhD!5Q<vRL0hV)gNN!&Y~H4K$BW#0HSD8IKM6GXYdL
zf+mQAs8SuN6S}+dJMTQ&v!YrO20REPa1HczL}aY;GMl=Oe_lZ}Pfg%ngXywN77u6o
zm4X4Kn{7*?g<9MaR;FV-yVlLyEqKCS<n0}-96Z1j_|Z5c6~lgg3cJ+0`!^p$jOa#P
z3C#3??lC*p+ft+QpiE1O|M|*uQ#T%AI9Nddy=M)|G+(0=7unt&K~<pW^RH?jB0BLR
z28@*98)^p7H%Gv1*O!`^aPA;FJeQf_3X0ePv*d_gn9c<QV!yi+yaO~`8kL?YM&Ul+
z-wlQLEE7X0#EA#Mra`>nUNCd3Ie86~9UmRu=IC0e{RW!M8iL>+fI?$dU^EQkc|fg|
zNa6N%TtDFKT5PGk>2}#e`-wT5fJJ)fyXWcoy4i_}HWMgZU8ky_-M@_b42NN540#8a
zYWv+UL*yc(K?B@z+jRL>8>Eg?&#~+0Pp*q4T|WSY&xJT^`VyGlzqDgFA-Zt7-cHmF
z5%t>7|Fvnq8sIm!yRsS_9Km}=TGW+f?pl9-+QkR7X9x_Vpz{w8oBZ_TEYoSD>ilnH
zS`H%!j2!!fIS*X;yyAyvM6I^2URH7Rg5}3Q#NT|gn}y#c$K$yxThaetc=y%&n0sYY
zVKV85PekTs=qDL^ilM7^nmMT3okeTeZ0!~8>RtLVC_7wza__2}@@w!yArlH<^xQ@5
zUfrpMMt1_$AQcQT)Il(#rWfq()EO9cHmZG%<;%2-|0EK8UbWh9=_a-epcABha}IY}
zx<ISioyqErk470Ifp>32qi9STAbOgMpW?bF7R6L(-@}Gv2Kx44umCX&zi`i)->Ez)
zJ$Y}Z_UUvSIrAVRl?FP@b3+PSOCj)<^uk-twyA&>sIUk7?)O{Z$qV~z>N`51siK0$
z>fvR4LEfMRkoLtm-P|8DMf)Y+A-1W#SWx|(MP)yklmo$S(&_0#yv6WDnab(vj7?^x
z`wZI?um9bfp5SI&5)dweqe&1qBgo56(1P5u9iy5FuPgZOSCGqFyYCCd;BbllEkP=S
z<~L+6Z~5axqO++1g^43|&c5nNH#AbdWK6_495#Sj)jgpyae!Oed|i)<bW>jVnjG(;
zc>CAjOSBvUO|yQ|cmK7!4TB)HI8}{VFzG1Uf;!2iuo~1vXx^S#>XE`;&tn35bw<>V
zt-M5NJ2VLzkG<5?&y+ScPIb@KDebH6WZK~0o_z|MPc~S8FH2{xgD!A0M5}Z%U=%&b
zIFH<XG9?uf*kazZ6d001PcXm*RL>pJm<zi2miFs-(3s(XU3CmNJr|+M=|As0g3)2G
z5O81wWDphn)bn6!>tZZjVl*w&Y`}Y4o51J7*#Ug1>R@%pR{s16#5bQ^{SCwbW35s)
zb&YrE<a4knkNdW4+WF}VNMH(d^W9{wW&^~CbtcU!Cu0Ek?TBoKxcC(q9j{}CA=-;{
z87!WB-XK~Pgk8&~>HrupW()K@;C}u^@Wozl88-Qg)IMUJDf-bDYFmGPW=C}fv8ic;
z#x&;YXBY?I&#%2F$}c$G1Z<0Fa<<Q92@96E0v_R{L+fDUd~%+kGgrdF=)MLJS3k#r
zTF{Np64c=IuZ)-D_I~&b%mD;w{}8NwvQ5U|oZ_pGQh7^2>k>2CPrp^s@vR`QzM-Th
zw$XZ!;M`L|mp*BPzO0*0zYMiZA41taeK#HC`{VoDU1E1)3fJ<^1+nW)aUEl9{d^(t
zZ(rP9BG$!b#w<z(YH|cdyr$a;KCalim^$<iVAn>a#MS5qUbJPZvJawXi+RH>0jvle
zYHfZj6#;XK-cRy4|JlI<h0qT96}c6e$WdR29#O$(v-X3t`53I#6{b(Tqv|hIrX4(x
z$8!s_RZHWef@N03-jWQW?!+m6GT<^d0s=<0zoaj|d!3O4EJ17t41d?39{ZyyWD01M
z-iZSV=(Z|Q7H^*>Ht<nUi;QA=b{KEfXrWD+@>&Ap86KRj3WxrvD$^&4Vsyi{81f+o
zK&#uun+r6(_B<Dzm&Yrq6`s^g_M40&kW8mA>wyt_k-UtcjH+_midXQ04TjpsZJ0#H
z^{YiP&^PV}n*ad^4|Jv+UM(G#(T0lj#m|{W9-#EeW;x{S2Dt^=P4oCzZw7&JT>?dH
zfB~B#{{8Vk_FVZL7rPP+cIx(M@GMX-!^(Z;r_*7oI%ot~z9XBot%-ql?axX`_KrQj
zLDi;q5N}nD12>-L)N#<a&zyQ(IZ2fXj6iRLS9=>I7yS#|GEYC|D%z~tQ4xk!?CjfO
z?>foL<bZ;^ma8{urE3KKy}GKFxdlUhlsAhpW9_HM4<5+oVsxk_)OgLow7@37Vgyz!
zz=~N9W!gJ${AL)I?A5NlZ#^(*?^l2w7!o18s?!ci_E5PDBSoBfJU<;8gC2g%bUDm?
z1XMtwvFo^l4BS58OCS1+4$d8#CCPBQN&9#Z%fJUTBUYVPb#h+uwrk1?<d3g_4K^{e
zfNoH8LpJ#M**C!HT2=^`O7HS%V=xB;9MVLBhXpjSQvV)q%RbCAwkjJr%Al~!!a(cu
zf!<;T@0qeV@d-VVV#}Hv>Eo7~J#6c90-4(YBXa3w(ZShH7#N^YM7FQ0wL~WIT;gIc
zO9<f5o5>^agHXxg8oktY;n`X`Q?UEpiMjzw6^NTwxy6M6hkke=-vBuVvar#<GM*{I
z5~=d@1poOiFo5ag!Mdwr)3LAHnO%d_VKa03<EwXXFjUsMQIx99VaDk1(IzH#=9O76
z`oa<P?fYQBG`HAAzu})@?hh3s11RAAzr9~{$3T@Jk@Xn{F-_OCi=y)>)(oJ1@f)Pp
zc6yl|K%$6Z5#TPOxkHqPi|&el@qot2v$g!|9d8u1{rs#PYwC2Ti-5M>KWb$-Uy^L4
z{rtn3Pi9OSZ}G%s{FIeU5<Uu1x1lbhA6Ps49Q1fljdAlBrOOQe#>R=QUV9mU8>U>r
z>zMH8-vL`$<vq2x@<u~pP4dj$U+2q(4ipEQ<t|w_$$;4oTzFR`c&n}*eHf~{J`}}x
z7-Ji;sdg`e?>@ZpdsSR#Uef%-c%{fe0|Y}2aMd;H8MFnRY9MnHIQ12M0F;UVugnBf
zue(^ti%ig35Y4PtAeWI17X82DybS9#a!a3afqDPk3(a(F5U7~h<YfTUVW`W8o(p2u
zFe|jL@8TIgXI=y6ZnkrcA@;?%RK0S!@(>1q_T4S4V|w8F3u3&^z%k)XWejf6wJfHI
zjS4~jpFVt=D-ftxaQYa^N?)A}xz=t_rx)0!VfDgiklF{~7o1gX!JtF?-|`ss$K|_0
zc!GxFQuV{CBK~ybCD#Tnl8;W)d@suzF9go<PV~t2k1u|zdOw$V^`GbZ2iu?k3~4eb
zP%Vx+Q0))a0NQ<<C)C9`MrRllx?{`d#%l2(1&~fp<fsA`6FBp%Yb%*o?hqqigI1J?
z4$cW`9hYAJk5_J&1GM&t3Z5aFx)BDv6|ge}7COys-FV$3%d)N9lI=M+HOlB4j0GK`
zmaVo>dz;8WmIkn6tf0XyJ8&I*Fji%6*X74#&oPK$bjS*q)R6Div-Pjfyb9GGl)Rb1
z8!;n+1rgw?o&Wi(FX$;jH0xcUwkw14A941<11sQ+(~U1zMt;0~ZNI<vEh8|@w2r2Z
z@&%rb8PRswoBr~vH+iOjc@Q*z^Gz9X!NAG8=<Gc1IN>Y}qeEbwAb>B{bl~?Lp)>0F
zAiC)n;0?1tiRT{ATR23wXUmKUw;fgBG6o;49bW2MgVOrvw|<3R$Ib~?{W8u#PjIV1
z7wGIAgR3;{Fgj{o&;SNv#Er`ZRQlAJF(s!$c!9YsFy<43QTpu>@R=Px5dkKm_8^m?
zaAO72>DAL<j^1ohH{V`5dmpa`tdxoLxb%Bbsvzb?jVl+a;yQ_u(NA<guvw}7iFb@y
z0$75!OP9gD<~PBaChNOTJuZF(#LF*&lH<W*L8%P#fB_7s0C8r#*BT$q+-)OcVgK!Y
z;1_;#^IdmHzcJQ5@aH!kx8l}x7_*yKtrR=g%{T+w+@XL;QvmYXR+a3Hfn(MV8@diw
zCL=%2)c`H(Ix<o@wLau3WALG@YoFizL?1%wuIS15ANRiwZF`*#h~NFiB<S29y`&?F
z?lG5P`pE}={9@`Gb@2M_%pgJG(MS2%m>Aj_a*u;zGY-~fTpDwXyRoP?+O9wKzu$f6
zi)B6@H;ny;z81{gcnON1opp^BGjoL*87zl%p5C|ngViwkIfH4>)b>1?(f!SD^{F1v
zc!{h{?OUvV;{Hp^wzc1A(kJPP_wi5ZW#ICIN|7N0<ZNEYY+~{MAFtjfcHvE7kny0g
zv>NSy(5Z#Ma{1K_@NLj7*TWb{0<oQCFR*NDR9orZRH;<gn@RiMW1azLL9nL0K}(@6
z;7m1=Utajg4y54Jry_ZyC6KQS?enxByjD4<g5NkQ2X(&P9Ko`;0);Wxc~jnEexe9m
zu(sU`Wl9DCqbFZw{bez&dAlyvNS%xsukKkDg2P_Cd^?{rj1*ua!qbOMhr6ou6iX8X
z_dApUM!#@Tg0`~{dG|1*QWC|Wy`L|T7Ur<JWKM+_2KzzUn#@O9d0Q?5;6(weZjpg4
z3k4SYEw;<Qf7Y3G@8btnWy=fr$PDm|pI_uL=0@>q0C=KzN*cp)?Crq;mt-adU?PHz
zFeW)&1`IsV&YV#T*gw5bjC*+u2KGF2A7}pb9v2KGfLI)ryr~BWNOTq1R9E2ERmKzP
zFu>EkFiJF*XhcPu402g=%%fV+csxB_O($@Rz@P}#Ze*Zxs`nNn8^zy)xxx$AFfHok
zxSPW|l+i}8Q0@B@prmn9fxGH;+mol>!8jAN7tvkAXJ2p6<Jr913AQ!WQq@wdL(l(n
zFE7RTkNJW*{fPH&L_-0XK+5BB-+X|<nFqnRM8UcHA$cadKD`b6{E^6C@GXr0`Uf@<
zL65GnD%2&DH=$30^5+&gAz%BX4iumPq>62|bKV!DtVK@MF%8WPWVI-!gea!5juO#w
zefR7wC_7oKGx5rGXXv7as$=vfSD?}k+Q`j!?I8e`a>;~22DvQNO2cMGT7($DZ&{XD
z^F%OZmFJIrScO6Fs=FKkXA*5O7QS6vjao1?fUnz}oWZt13v?G(u>}G)RqFlqtEVuU
z>Oc39w=EPpLy)s&{98gS77*Gm0hDovJ$(Fh#*H#jF1=jxzu$WE(d{+fg#8IHhyeH3
zwT(KJ)vy$bm7aCdCrq_Mjo6a$4=?C;?TMqZ3dTfLnow6j+hdGl@wNL^tT6z%$mm!w
zhm=NeuJh}U!E_{74cNzF<lCdF8aij`;G>M2{;#hd16PrhqgkHKG0?hPXRo{mNyiq@
z)m~wxr4T3IFg48e``Z=MAu)$@Orn^jI<7u!xrTu`=->e}@M{l)?d#y~4|#$K9NKmD
zUq1Mwu1`Rt_fo}D%Vh$cpaRWNZhlY`#A@D&-y1PrhtCIXPCaLv^9f`p6I%|6?11nv
zt^p(MwHl;~<1*BQxN;TAL(3aQc~{?LMgxH?Y=G?!FpuNYX)m;UGwTOH^x(OdxqNjx
zH2we5&tU*eiddLr3<X67p2#X>`!|SD_)quFY(S%9fi}?4km$@4K{#s^;KW~Gp~<Zm
zCKld8j(+Us7sc_6BfzNX;05SUbvLn(`j>qDPf#;t7`S+t4h1sncIZG}sDDuVV2tYe
zO2D2AD(0M7Fkhzbm9IFTuDJOagKD_)QZ@ub)-ke}-oe{<6sy2MKX)fs2?*#dS6*e^
zOoCVt0Eac3YE@zaACCFcV_!jPUy#xj&SaYCXGXICjN8`U8XXqK#tH;v@Na9pQ@6|q
zvf5k4h>3Zq7VP>xo<%GNVAiv>Xj{}@@Ib3#J{ikF+%jO2B9i6cc>as|fFkA`z~@k$
zGt-9^YYj@~9+?Wo>oxJlWoo~(y{1BV>IEGC{%x?NE-*kXpM_K=7M+hk<1+F=fzx6R
z6`>vt>B`-7&K4-*A8*S>PT~ypia^VK48!FJT_<pHhh!t%o(JP$Rg?dEgsbk+KJXNf
z%9yk;HwDgP3!+qsFcx`N9V}@eW~k4aS|4k2jjrle>w!uS2rSE@;-N9ShFG+lMPkp(
zrtjJN3}O(@D{rLDGNs%=6Qg3zg<!@4C!SF2>QUq0L#l{4@;vp(m80OwA3xm-{ZKM*
zz$ILKD|k!$Hj6|IdUu#5O&}nShq>N3+kIle?;s6N4=pUt5BK92)}YP|j2}FZn3xfe
z5TL1D0V&A|82tbJUrJ^Q?va8UrzWQdq$VVDl%ArEl2Xz*%18(hjWZHt9<=1t)PUs7
zOtBdDZh|&#6FE8YrDKze!}IZ_ll_as#lwR$L&Nda!_%XShjGn}<a9Mi49G}G$jHdc
zJ9uE%C#&Kp>&>PX(ZIJDcOc9F7RoFMBLf%%laxgsR}Tj_GbUkuGoM*nIx`*)ARX5}
zgIeUp`2k|Qa6~@{<;(*UWhz<1Fk}i>RPjZ9y8o!k?H!DMfjAkfgva3E0o#4rKhPDF
zO=<6}hRr!|%cQ;9nA|_TS@He7tldd-R<$YDl#~l5;I{feQ=nE~utjwjsIjs8z`6fT
zkf04B%6}Z(r?0{h;^up^S6}A!w>y{odGYprs^c<4wLh~U1&+LvYA4VPAR7?73^$d}
z`?p>CwFLxM!5$#k<PG<t!}r?lGTsl-n>{Rp+Vu7vu6Z=MRSe)pcZtbQ1eY<>b?IXn
zfiJKe?P9nmK@-c0_1Oh#RHs2T1x<B>@z|LcJ;4E^i!g`<F-LT`?n{PlB4c(In-bA|
zTa;2NV+$uXSZ-%@G*?XAyJ`#ChoVEoJYYz*qQ3zb2X4Af2H@VJO*n~Emrk5cWX|z&
zMeK_Lo(XVyTcWx`a1k{w%YzV=+hS&dSBHTyWwf^o@b1l80aWk(1e_hr0I%4U3Z@=N
z7V4f^UUGZ{V%U^@le_I%V>LyBfS`aF@M4BSduuMcUTYAT<@tBV!t+ddX~DmJ_PW~L
zBBS9Bnhyr^1@V483AFUK8>2Tr&f}^DK(lC%-+<ubj2p`+1`{x|04|?L`C>EwySu^j
zGQa>XYp3-1yE0;B7@s6pAEVBa&T0%ZnlVn&KBG*>05i@^Bw&yQ2&4{WZ)BYRQU=16
z|1miAp;**z<}mW%;&)oaF5Ian8%l4YKOJol+lsYFod?&`pB9y1GobF(g9pyMDn}i5
ze<^SB-~sg*+eqBXl7ry(f!kUH=wHU^zXMA<PY#wDyJKh!^H<B7A6|&<vfX=jV4NvE
zwihKiGSYtJ5-QuG70CSOufRP(D#@XZmMrH&=-7MHerxSp42_dRcfX{H>2UF>e{}pG
zobvz0WK&{0)`*L%ddRG{uEU5MW?9*3KU}N=w-}dF6wU-5$S|*ka0LlsSsY&1{vadf
zH&Dy40h{e+;k?#x8CXAvp<D*(SY&!Ib6KmS?%T$^$C%zL8mv10s6mzF{yG<!hAqIp
zegv$UAqH5gI+!<r0TgFy4&EvT;X6glG~O{jPUYOu2jvi(&n=6KuYe_=bpvQGqRYpi
z@o5^}Q&;8-$r7Uk+C`}PoamGxcrS?k3ON6qY7HHf*Zv6h-RodX<0Q};;{lw!OXN>a
zL#kzp1^kx%L`K501RBzhLt#i$ncIiOI+4n*+88J!6A_(d22u8MRqinPfOda-c|o5t
zTGS-^(k7%z9kHmY$3SW7$)idrAuSw-ZdrWBr0L{{_Hz;OIK*}V@VazE2a~9K_cOuJ
zT_%}A#5?bUD}O_4TUlS}ohp->eHVsiHW(l!6YLvW26)9tMq;odzsmpO@1OxaU0}TT
zZl7@$JGjke6AzBFH)>ASE-#0<M9ra)XFm~}XPQWz8*=)fulaU>GYv`ffS`8rmZ1To
zWTtVFPpVANK6Xl%@X%x#U+l%7y&XKY#v6P;==5`}1OaV8wncuvXmoyrs520}vrF0a
zJH^5@7Ff+~M@DYOcw6Qq*fS?@Jg)Y^#RK-B>u)d10K5vMqQ1UXfx#?@6=4Ve+qX=W
zLDjKJhv#(Frma)qMbhOM9SE>D0Mnog@8p%$>~XOi(Rp%;u0rXS!Bx7$m<IZcr5I5&
zM3{I@<`^ASQN!ZjWZ|+vw3mu!e=jq2&VmB=t#REt>OIuDVPULVtxEJ`P?B2)>$$S6
zDH}Jj`$916d8#i$jm)ddmV<k__QNK8DZnNGTzj>A1%f@~`h#G(YR43+m6$;Vv@<Qb
ze0zLals94g(<2|`K(q`_@u{uq4_X|^D`%;hDdGqMG?z1eOfV)9`$=R7SVG9c;p%xP
z+7EZcn|hT(F9rcuUuu}5_iVeS+Rh)@@Av&1ISf!eGtD0aZ1D#4ay@7^&6X~xz5e-O
zRv7Pd{1p$*2LA5bK44QtAQQm1Vo!W(Oxj_3-$f2ajpas_{oAXLVuXKJbF&|XGEP4&
zvG4$v_If0@Jf#w1P7YjoeLjV|2C@r%!SVmU9ct-w);7@Y)dJoHva3tqZI-;XV9CyG
zLmzD9Y(XE-7Q>oyf;o-Al!}cpS-HVKeR>~wm`pJ~FGlNlTVP6cZ>U<BxHTYUADF6A
zXy#$fD=H^28sM&L>0p4@X=QbJDnoy1C`PIXc^Nu~ITru|w4bD5XX;lcnDSY{1WMaK
z#d&KX-tM-Io4kSt2;jO+RKx%23k<TvPBUZeqz^Ms{;H1M0^m+qhJcF6W@nizMs4#O
zT%qgrcaHxPWVOm&tzX+&!qNoVi^I%UfPLu@E9j@k#gOyk^U7R&`KhVAB~_A4XeM#n
z#g9#8;$FV#gAr?n^b61}*`O#;9>WLNiQp)0Gjl*j9@@XrU;fga{S;X4-Qdkt`Yw-U
zdT+{sh*R%zJM%1!hUtx7u^#&A87Qu-C+&9^GE}Hb-LwaU%0E`9U0L4E+e@Hib8S<e
z=U#xIBY;K`F@{yn>^6XnpL<F9oH1eCPQI@F1qDsenSS#TI!K2_Jk&T_=Fx`tW+%PF
z=V5e_mXFygolo5X8#kf-)_mDodaP-}`S-o*-13;f;i1a)(1F?<k>GBzx(X#~yYz6;
z7L;J!+6`@+7Y0<cd{hyLanf=}<&X7Wp8HH}!nw4W*4A3%s3&RfaUMPdMn9E-N<o4!
zo~m^aCZri?TD2OB=KVthaWRJsu&wWF3H!@EPt_kh(3;4sh+(Vi9k1Eq2`iv!EuS}_
z0c@~9By)RmiDkitz^Y{t@`f=*U_qb&6+jM$0x$6q7tAbS7BGBYe+njGCIZ|SBNW?m
z?(Wv+f=VuXUL{k2_K|t7%fEt+SI^P{H_UYi+93Ss2(8y~5nNe`_B$L?Q|;I8!f5Kp
zR}LQN`td!e2M^E}d{Hn2TH=W}n00~+MpX>j-6qsJnOOn)TrR-|bn}QZFC=XngJ2CK
z7sAj0PN2dJu)5Y@K6d~%10502HnyuwAPk^T<_v3-W!wcy2Law76%j9}bXqhBu(q^{
zckE%Q2zaG6&;rX)Fv>MK=tEJ)^)dZY0b+2s#L5#K{kYN4oI=_HbE?6AzS95>SY=zA
zLRebcJ1q0UJe5yPV0)*Lukh6N>!OOqJev<m2e=*OEP$;fS5)B!5xf?+Qy{}x?^IFL
zke+8JsR7X|6jor&G=OSwx5~s><Eh%u*=RvE&g0z1AD*a(u=cjzQU}yUWhC5&Gr6u{
z8K=7kLLo7{+;|v>?kZuZ1+y0Ysr}68#X|f3(0=p&Oy~>>v%Ya>fMwSV;w2(MfOWW|
z8*fARTiI9}Ak0Am2DP@(vOlihuAe1E$HsuFSzG!2w&1|5Nxua|)#1?U@M<oX;o&VD
z!t^WCb%N4Dh7qV|ytn~Qw`O`-o3U1aggsPW5D_j`;hYU{D^7*vKz(@xYlU}HS?Mna
zy5l^5zSE~LjeY}|a=R0);>709qAm}m6X!Nn0?c9HLZ4Z+7E%<JvmMYyfLUP<FSq->
z>%IdRuzZF<K_1o?g}dJyz-4RiL4p+IwU~u^x|(GclsZ_mXdDlj*k~W72|W4`hR5zK
z0{7pIJoM?aUxJSu|B`q7;Z86f{QJJ34nP20r(<5|CW9n!6tN9K=d<9sQ)_n*@V#Ji
ztm=l{(F`n{U-XX;anS|pkoHZSY+=koU|7UHh|r_=gI`r$k2M8oYlUR!LUW4#$Mdg=
zotcKWAc998{p})z$^l+Fk9{Nq1SGOG>F9wL49XP?FHW+uZvf90<9-p`55N(Fw=6>@
zxu8x2J;zkm9%ITTM`q(8{dQOmg6SPdGmt?y4MKBV0UHeGvIMof_1_+PPB$=DX{&5W
zYl7&~577T%+%dpC9}*RONi=S2f;$9wFJ`HrjSCOyE+~@Wcu#3+x+63G)F%Nd@!g%%
zxHDkS7f&W)xZARv9hoSDBWme`VL|FlSeaXO3@(?j00P)-1o7HeCiE7kw9A;)&qfSH
zfEXqge=>+Ux!p_`w1V<MHy&E&&<(V~tJ5`Xki7ZLS8m+9ZV6>CZrq2FC&=ycGpYIk
zp32?({`KP{KfK6WuDkyAu_LT@Z2$>e`vcwA2&$@&FBO~Hwu>D!cDG$uK;8W^F{qm=
zm7rElg!V%KUq1mh4;llhw1?JYUB!}hsAan@_E1j=Z>Fy`XYtk^g(`;i#StMcFwo=t
z%YaGm2(nTubEv>TRlf(+{h;8(*2WqLqI=i<m@K1oAS-t-=xrbpr2Tr{iss-gx;GY-
z&f%dgUvR+duWudMsP`}Z`PUe*UIQ;bli-pLh3=sf0d;Q{1H;K2;AXgI=XI;^?^Nl_
zx&Z1cJnB5knB}1$w(`tnFn|ls1u55>aHY<DOoRw_N}w9VP5`8Mb=dSr^&dQ-{R%D&
zd|&|V8s&#Ww6C4DW{q?fY7wk0m*e37O-HZc;`>0?Z}VYp)9buwAjqG}%~zD~G(W-6
z3!;fGQyJNi&r_p8z4i9s>I_B+V88?yU(ORH1)^RzyZrUHN9tr$wB@tJo>?G6Q;2Le
zahgdDl=XT9(LHoz3G0l3aR7yBKh+Tvvgqi>p)<%a%i0%0zz|)%(^-pBGI<(^($kDl
z6jaTMT(G)&XG~!j-2l{<Gc&Df!D<*-F3)+0u?=wG_UPv8cd>9R>hnuaeF+6f0`0&2
zM13#ALq9W@F216CfP%*GKR$Zic=JRUfa`blfM<5`IKWJ!dgFsDZyCD8HYUO_p$bGL
z6J5dejngl=3t&8Xl5NPVV1aX0S~8e|mg<#t<ykQ>8Gzh@>bi-Gcc?)dq|YH4@E~aY
z`0e05b<K5SC59QO!?}g@05=H)f#~rvZzc-R4_Z0#I5>AFVA|-|IQx{Y*BEZR&%f(B
zuknPx2X~~N1#3`q^%=X)n(FxLb04UxS*qIo*f*QbUf>Uw#L!jU!Yii5{I4&+xD(2B
zpq0gC`F{Z8GDCaC0@a3D!Nd{lXSWJ=19RtEfEZAm_W#<Kr#i%>^Tie3Z*&7}QjA<Y
zDSHB}nw}JFm|S@Q+BAsaHwxy!cb|Z%m4|O|1v>ux+()9TmV&{Qe()EH)+=B;eqWt~
z>J`JMd8yv8vmg0F4!bdKK<?Qm+~ztiJy8KYi^*Kg_yCcN@A3PY9A*<oUxVm!0t*AN
zXarq!k9TMcPh^O4#H{mNXiOe~N{cS$5R<k^Q??yF<uG?M)D`Rpv6{!ZPKoIQtcX4D
zb#umW_j72fW?<oxGt%1F!UXQDm@-hDZfX)OYN=^)f$oXT0{ixJ7BgU$3dR%6)kVgc
z3?l(a-h+^qpfv70Aq`B0zIGEj;NN}bc8Iet@)Bcv-W%A&Yz78*|H(&&s>YjCnymcg
z?N7FGB#zr3E#%Tv=h*%KYI0f?gvxU*nbNuapPu0&l-C*PfI8FGv5QYb14d8W2i_4a
zBk-(I^UhWovB|X!fmO0>=gn1nfbQ5cXiiTwkClJ}42Y@CH-FUrbb#(7W}UNZ>)2u7
zW>mpIHPMei=}BCtTTk2uwy3V2DHF^iW-P3_IoSP<E2!T6Hve_o-B21<y|_$)1tVv&
zYL*942lAiwkOSZ+J_r$gVw-yaoH96v+XZ^+4{7aAJ$rIt;!zfy$1s@F!W<LAu__@d
zgT~m|;l2RKLK_QkpkS4XG10gkL1<2Vw`hzKLzvFAv6#*r&w%Vx`y9_O#Yl(Sg}at!
z>0rR7-&9?pT@15{-O7E;JzM57V_6B>eIdu*(h2hI=FN5woO%Y@2q?5!tSkp0wK0+l
zJ9uE*UJP3z!0kZJf4k$0#2xJ$kmy^rPzDtE_5;sp6*4hQ%t5MbD#(Z4?>%|y$%gHW
z^nDu`?O)w}T4P|d40cTWK1+AiJ(znse)yfJYFXhJMS?ihn$IZq7j|DAy)968e<p>g
zk$>!dImnF&G#_c~82!NmHgZJyikoELo1njUJ<62|Xr;TXK|aTyR;~=iXq#)9-FNLv
zOI}Gkv)_be!CYAd34-{baw+r6Wq`9o=%ZCueGUjPC{OrrZRVf+Ef0#lK1C#3!MDE>
zJsaYO=Xc+igMl|_R^?$}Z;mU5%<)kW`v9eCZGU_JH*^p$&XfQmK&fZ$-jNa0iD}G!
z?<giOZj1g>mpBY{8)SUm#9{<Yi_m%aYSyMcfKn|3t_I-302Ut|wd?U2F@vw<)aiO6
zH0`&D#G(!`*N?p`Cd&*+u9#H%Z+AVYef|7C3_N-hz=2{bp9){d%BaD4u_UXe%#|&Y
z1+|CB5qc%^+Xultc38sMwP3toF-ZFYD#&3*ncF${z`wutPoc@tRs9$=3|j|>rp3%b
zWtP~2bXW(|Ywf|)dycSbQf0>OZ+dq)Mvn}(dv?wzsG!croc(RPc~d`gM|A;zrtbh~
z&ztKpYU<jKhxmt>DiX70d-R4|uG9Vk3@^U{W-9gth#^sh^LBzfU1PvF(I4c+<Q$n(
zeUCj8h}l$yrG>PH`CU?9=*g)=%$<V=)|Z`)SB52(+ovySJ$RsR@ZxRSxhoikce33H
zD$7K2*MfE>c-=EEasIZGar-LD*FUxF^-QtN)!t<6o1?Pf;scST6@x*n69#w|1QarC
zd(nr1_WyEuv0%<_ZBAn8wDaCEpu}U3gEJ~@OaMJ2Q2zFeWJL;ih9`4|VK1H#V+Yqi
zIj*`*I0o8}x<qI{OTm()b6T9ifHBZ_kHk=TfkLg-!-S3o@<FTtRU<qzcC#BiEw6=}
z>xl;Y{!Oq;za4CX0O<c<QLPrb>E7Mn9vMnov@^&N@WcP|?7bDWH=bbpe*xye*nfQS
zK19@x+V|Qo=3vN$Was709isPgQ&oD?$pz)HV-=9x7X!(K1MGDGU8Am=?;xf@h&|6w
z`a(x5FW)6YmNLMkjA)o+l9mMhGRiRedBh~Rsf)Cy#la#_DK;Y`ii)0X@P!PkQcFmO
zwJ31LfF2BD3J=j4u1U6Dj6u|H=UA)A3S^eWoOzn@2RcN6GXQCs?%-}ybix)W2P8QA
zr1rO0#FB1t5{=6()s0Y8#@{%?4Cn+g%N9WZO<X3QSvr`jY-wDIm#1##HpgYehA@0p
zejVzYkAm~!RL``xinTxa`8O}CMe&UN^jfI)L#&=TJY9Lw6)K+*7J0pl|H&gs;DGlI
z1K-~2)-tjx7U66atOy+{ga$DrR&!@S2B?T_dw8rxEQum2I<MX-n)Vf4P&1;#^LZA-
z&*ca_UAblDLHb?!uUwx2#W=7M^U+7g7^&pqi6Pz`@LXE4P;DxULlBRsWe<4#`$KnQ
z$mLgoOr2SR(;&7YS^@1tD12P42c&~1smk!{V>gKA{ebL*jd}>4#?zTii41E4sLTy-
zr{|s7oahI3BnBt!+7WP<G%#lB+Ugbysbr&VgprI_;cQvQfbrT?&V_0X0#3dIR&xH;
zG48E2@WyW-M(M3zz`I8bi9vChk@~S8o}AM4Q5M9SJ~X2@&O8dn((yaQWCa?S6+l@4
zm*RWzo#K99DC#K#P5V3CvYl19>DX-7TFKKb&Hcw9fKcX|i|QXLW^@@n1+XpfQ~~>o
zr|0Rnbpw=I+fDFG@9DN{k25B0{y;oPwHB5kgFxS{{nyKXfB?+tUx1MVdgnq0>>xwD
zWlZzI16-G1^R<_!zJ0OQr_a&egOvejUoyWVI`7oNZrt|Gle^!jV~Sy~aun1J6~1p@
z!GK$K+6bW=DwqL)mLhre(0DM(vxiyH$!h5gYbt}_7+5<Dwjw%L!qa)_kOkES0>aMS
zTMI*$122{d0uI}3M=^$i+@C*-&CFD_9pVKe<HC*3fto=PIbgmljYA-=2w=6VY-gI=
ziWQuRQuN)!;A}46-wu||H~_81r=NsiD{Ex7q=F(qf;JH`Z4;{0PrshV830o>eQ-$z
zXZ~nhJS<UHp9WX`9-8(!O_`~4MjCLh&KiW^+*MF2Gmg>H5zE&CQ~FSp<FAFWOPqWY
ztci~26TK32`o(GR4G^HJC^HK>6HMU8;8?FL$GJdeFM=_8bS$u6gagFYSH1hH=w`v-
za<VTp&ZV=OR9(h&%HhM>w@ol0K%t0$4tMUWDh{l%9E_{f29N;P1#osh$Y#+V{KP|G
zL7+kQ8G$GBMaL{OE|6CPm|ToW9JAE}1QdDusT%7L!@7+~;o57$I_L}nlN9LVC4p$r
zicQ=mgt_+XH(~}q{wWyUE3mfH)l8MeFytP6s?Smm*f2y^GYo4VV>1D&Fn~FNsM{sm
zx-v=mtnl^h^kN73?=aM`Sv3<|fy2yBPnkB#^1k(#S?M{(oz7#01Bp>!jiNJjw%oOW
zbwiJ(oT9@WxnDuWxp$$k$bc5g_671ojYFH-$A9wN<QS_;!0H5|3q&4$l`Dt`EysYh
z=%4Hi^R-S>NhveIO+cP0#yrDtYuGTj`wfY<DwntP6QD2&frf(z4j#Dn;ot)F+3!B)
z%w^6C3kTzU>_H5ea=6X>?(@ErrA`-hpLqcsU~|!-B3t*LAAA5?0SiX^u8UP(P;eNc
ze_Zr@2xefWVXeX|@c(?|?T;=!j-gAWMUSh@V1!du>i9sv_U48`j@)><1<LY{Dx(sy
z>YP=>+;OVf-rUx$P`G&D(%oR%{~?pH@Gb2ZOFC6C!^WJkF=O`Rn8nE)@VF^H;in3^
z#lkHyvG6~PEc4u<6oXg}be}H>xcX!@I|a8=9y7!W>}C#regAs)iQhs|RR=8@Brw`o
z>8aoTjvMKi;>p1NCha@@d^@biXG$^V4Rbr_a^TN@xJ#6T^&}{Vt5k!DY)i?&xK<A8
zoO!5{6WZt!ShydURcC+=1Yl640?G6QEVX|fhBC!K3Fryd2(MWODj4IA=)%1yN{s8k
znBD@EBb_TYWvqQ?Vu<M6yI?>>qX=&Z6Z&waY?KWHXv#>-?OYxG+iRb*#BxDQ_<EIX
zP*VoM0A@!Hh-);QR+-Gy226PXh|6P;ff2TPfk`qwEuf~7uTFVgdy!!$hqsJp`JH$I
zwj8y{!Wj&XJqV_~6LI%{F=E_nHsl|1@IV&Sy<h;@dAKLw<d>u1s^zhe#XQfg(#yLB
zF@qaKb8FQtZYj0&_+q&BYvy9+U7SunF9YV1x}6yQAy3<&hZ-6pw0G|2CURpy@&wqw
zzwl%>k7j9a%ch^p1M^P5xeP@)X0r&o4i!`x?@6+K+Sl!hnXA&F@)Bbg-@Nm8n5xXd
zI8UDgrLK^jgik(?bOyE_WSp>o5isdEK<&g+U>QC+ycBC%V1Q`nsS$x-W9h4@Vh3jm
z{P+l19bbK%djPDJ$yUuY+XD4~w0L-1Lp%0PGh6MY*T8gG)XZyN_4dPWSyRasx44}d
zU&7_ISmwJXCqk+D<^xs#7|@$QcZs7a-bM8+7CU&47taRbcK?;mRCR(zF1eGB{a$VZ
zXz~dcg){s?mY_Vgi;y;3O>ck>xSskWc%%%#t3)&8%#3$Ht)!o#K)h}ji0kG7PY^vK
z$l^3r(oIm4LHLYnB}4?z9-sQlXK$>jyEO1zV0os+LNuLsGmk=}n7{PBfGG%!fKszh
zKQANjp7wlj&O7r4m?Mah0WvlU2YBzOxWo$&L-JI?n7OV($)ab*bj#^GMD1f#j@}Jc
z0s8r+Iqe6$<QThQM;q97Ec2+tMYRoM7mn+lm#K4EbY`-aGdyjWo)IMS!az<N{mSQE
zniys)*F+KSSnFj>aQ2c(?c4fzwwKJ{?d`XNz~jUKuKf{Fx3azDLmQ(N7^|&<fmiE|
zIQc#XK>PLEM4^8CK5*^p$g^^xH8=nH`G+&*Fvo)GZWWO@zN+J<e3zLP(;yoz65x$(
z)ibXcIu{tCax1mV4x@08x79Xy!yw~R00KHJ{@3S^-AhG{mBi@6Q@Jd;y{P0p1s0>a
z3WX8st)BbgnOxn<M*^Q=f;n9)^W~fqFM)*(fzi3_VB;xZK&=Sj=bp8NQZ;h<-7M{E
zP(3l?z4}xowq<JnzFj~MMWs||-!tDDg8{(CDVR&Hbb=PG8=zQ$8prlp=2cE4td%s4
zSBwmg@E*C^ue-TwzYxK423TxB-DYqUM1qX8|Njfs;`-f@5b9Q&MG+Wqhd^1oHERYq
zexH96x9mUj!^`ee1Pqbiz^%`q@s5S<<(zn#U<uKX4#t_s=|NK!F5Yk74%m&m!FDe}
zgYrTyRDrJuZ4P(DE)&sq@`*aumw-9eEb3s-y7s@AD`MRTZA;r4*f4&9qc6k2U9dm#
z)J>4!>(3yWkE6N?3cG->KYcMN=9LhU-@PLe{Id4v`q?+NPf{CK)>g#l+mD|dW+Lvs
z;!#oD-~YEyK9UP08WZ>Ehm)b_{wanjubxZpTfJN<UjhRbpg2*viqR!M8H|)OWrD)M
zPQ3-TZOmYd=(KNc2LlRORCQ#l@vKl_zld28!kJxPe<FI$01p*1o;q^R+zUOUTMV^{
zt)j5s0v!_~b>2M6J96b}BhX?zFT5!e*y5%ci`^dz^VrP^ISw8l?k0MCOHu>0Bv9Ig
zhw1A~Il$JH3-aj&JNgw}c<xPAcr;(z{R18YuXW#V--EPv28&$+E25A1pajfB2zUCA
zPaYI)jlr}Ny_W9YXH8}`+gn^|0<DDS)r5p{H^bH;1&(gzhxYCMJ_~F*)Phxk>SicT
z>1Usa<w^##N21E6y)yGu#V{N?{8>~gZKt8Kt8$~-mv8k$i6~+C({h=*VTs8z5IT&x
zCMI?kK{dmpgV!<MmEqk?<+wu?hl4T140`eZQ`~!aM|M?tqph5CR;!d#D^@9$N;zk#
zm2;L#Ijhu?T8Y8f#@JxM1K2PIOc*m53=F~8nDJW9cAIuP+wC-o_sutf`QCf~!2A8~
zKDwoD)i-a}%zEqDza7p#=iYnHK6~$bOI<-K!E+Bm2k0I_RW<|FHGH`G<d6dg7b^8W
zL!MRiSnd7S3~oUgXUV5WR~9TT^@tu`$p<Sh5AR`(ssjN&oZ2erUw(R*Sisw~_?gk$
z{)`6T<sVIHKgI`@j%yL?SOX;Mc?Zb8tfEpr|CMHeRW3AvPvteM(l%U<rhtJ6XdkDY
z-IYeD2>-chrJ+4%4%H?#E`3EIolur}+hNfK!a$6~hMs+vtzFJOlXs7yov3OEKm`sb
zVa7*J#OTHYXIY;FCn6d$=)<0@<C@4>C<)em?PouvlRJG?RB*<v_IV{pU$N4pwxb4W
z7q`J$Dt*FTAk-E>8w06zaLNM4<kd{DDBaX!8(&lfu#6iA%yTCM$CpK>`Tyyu??*rn
z1aU)p#cqOl<7u$@ELs`Y+@dQR9bxGMzKkvdC<b&EW@U%azFO+-$DnnCxD_2EVB88i
z=`6yfq5TLD)aSk{Z4NyWHHdWx(7qc;RPZClV*cQIVWQdy>3RJNn||H}oOM_7yZ3(Z
z6_$?EF9eej;M~bfz134P7s_+6PJ}QA%2&WR=AVE46K;OrAlPd^0&@;wp!xTKsTfIu
z$^Za@RheJ);7Qi(yQqSh^-9nZt*cOWK&N%s0KG+pI@H}`I@`kSF-34cwRi1vI-?+g
zfh+0f91LnAP`Rl<pG~Cj>db3T^8&kZo@#P`{Oi9~Q(+le%${RLaEK08p{S33A#J)`
zKnHiYfq6>1@{&mKg^SU(9x4+3yyIXg1l5h{14K!L(0*S~no)r)3II=}Ql`z+!_b>#
z$TYa~lnwp!1+kE-?CnAo?3RhF=fVuRHqD=FTof>J{ciA0X|##}_m5GT)KL_-7stBi
z!kx<97~}qzg8*(dz<B^^AOB7if0a~def)W_)qR!4J(OLe+UIEYWo*2AseuL6AkE^U
zj<k+lHkk430QO$_75KI^R#zBY1t2;pO^nUj-3WSpC3`Na14mdN7`ecQPo#iXN@Jod
zhT{q>3`T#kf%D8jGibrjd&V4j<_ivY-uYTP?6%!^z6RmDT508kT4%tnfC}jg|3bH{
zk>wY^bQwA@%mW;P)#hUrH&BZl4qAx~fSu)SOs$9eSCA1VqbCB|2ai7f5*ljr{_lr>
z|HWIb$SIA;VDNa+#vr^v2^G@zvO&J_bueCpL7V^zKg~JlDsyKCpz~I#3U1hoKkF=_
z?@B<bVnYi8Wq_Cjz}EZsRCLY7&;ksr>}3oA<B4WdLT1$Ph>x^)L>%6{7s`+{!2jjZ
z$3#zK?Dv4&WBj~XJljA_B7ido-n$cw+g_t$E+%HFbEw?E5vKb6cKcp)l>$bZnXU!6
zF@n)KR+o=e3+jz8f^FN&4_Ztyu5&LybJs1^`wvWpn@{nCe%@*=%+5c3_5tvK<V$aV
z7E-X!lpSPM7ifP>?}h%}k0aiEf=6{F)Pr-Q8Ehq-vI-jR2oNdajc-AYgSe{zLjYig
z)@g5}Sld^QT7+q=9gIF52Kk>bLe6}gd@ni2?j&>p+*}}up)gmGf$t9FEN=tTXXqAt
zSE9$Z4B`nZID6hxQp;FlD)+&FY0#d!095W_v}+WPYXDt)f^!8vc#2im{L+))+|4GC
zssaU2flReW?b*^!C{g|8(gr{M_HL-loC6agkWK)cuRS43{87Fr!V}(qMwGW6jlRgq
zus0-O)l;mD@U(8r0>lSj71Qx-5)@81MGUM9AN<5w8p3lRxUE1pH)}6?m=R?UrLNEh
zj#2U150&#A0ckO3_j4*tf;ouzYBEsJ>OVYoSI3O~gxz`3&;JM=o{3;UcV&}xN>rV5
zOO9JqOhGprc{+f1R;o*6WxTIspJ?-#php7xY753H+@KW&_i1mT?hWi4UtUd)80ZY*
zWLR#)r_JE@U`IpEgj)CF^2P=OcnhS1`LW+vR%U`c5Nj92OTPn)3rxCkXN1~ww8s`X
z6&~^2=Xt&I1h>2lx^MaUuWFzCV3<rXjWp$;j`xIiwgrW{wlX%^DvdSnE-yT&LLvqA
zXJnsoRaf}-o!nmhV+=SA92;33Ev*U>=Q#@lN3w>ryJ_F7_xdl-50iIzh`n<}?;jmj
zS(5-0qihh=w-FAW2?GB9+duX54ijBX!hjYlZx(1sdcd1R+wXRYiw_H2RvS~=d&|rj
zca`)^eecK-)2@Qs8AryA+K2!XsM4;t&nonjGsEK|SKkd%leI^2Z7i%hF30GqD8{XK
zzNHpbSiSt(H<&Ai;i=n3)o+VIWm=^@IGVZVOvBRk)<fz$AeWjYrpG12tKa3Ef-Zsi
zM?1GF7yEf%s)faVk$SX}K>{~EfX09F5X0!DN5DFAJAI`KT-`j6{Ja@qxBlrAhE;uF
zAp2Lx#>B96{d=Mez47?X&$M$!fc><)$2K0C!ib<Rxru7$#-WyTQv~q`u26D7tf*>t
zZhjqlI0)cL<3x@3H;Ph{P@XYiwxO}XX#pieOgO=$2<6qC04CR2Txovp=e!B5J12%f
zsMrWiJbk8MyaGo|80M<>lAu+kk2)Eh3NT^@L}gXu+Sae+9&#Cf{Q4s-dBa%)nGByB
z-}awBIO5g_KK?#^Gfd|Qw5Q-)=x;rvLY~2Q`plXQ{gdCXvS0v3=ALp01L(bn*wt_X
z2Bd@fKm1jZw1oGS=&iK?Llhd<?o%elOljt914P&Ic3&~%s1=N{61QSg(LI%R<FhcR
zOF$FEs_ni=f3Z{)b~D_0F9+1_K?^DHoDGJ&`7P-IAL2R~Ah)5cW35nmHlRcL8X=h2
z8R^_B>rN+~Dp-V2bt0mI$pW~8{&>d^RID-2Lc^Iu%qdfI;Zohmjo&nKiP`gD0GB$-
z@Kb@ryVodalkFpr4th?&P#EfRuVgj?t8_%Q=x+8j)WT_JjZ5F=RDoF2MNTPtj4J}%
zU;GXD-i!B3Gc$Yq`Y)lr@rd@3B{2&3hcJLvc8f|2V$-NubC<NQPLly9Go-mFpooQC
z+t8u8f&x_{mSoJ42vs?U`}@YkAe>I%8tLFV#y!yG16KOnU-+v9X#cSB=~FPWgDE!w
z0vI}%Q;xx(`|B(VL?~!o`|%*P&no0NuBB)>gb66HF|fF~Bw%a(2)wyIPMU?I+kM*a
zSC2E%m^9qnsICL>cfOm~&uh-`P%e5>mDEWtrhnMh&G9;Nwslok>~VVf&HY{1@RcHr
z<Q`D-23P~5O2<MU)Ut^}!z}@v=%x?Yg|-ILULQejU;i$6R|3x|;L_gtod0ToZi(m@
zmCgtHcj?erP>VgFcwT^RxC$Cg0b5vE`s86q1E>wOv=o^{k2EE7mhaxpYZj@LgbFP4
zG_FGxz7@QrGb=_@74wn1CQt<w4dcXVQ5-{yr>wmCCP&SI)>^?BiUN;pT~&Mi^23oZ
zdC?a^hmjExob~CaLZE*9H(<;~9SP8erNwz&;Z!-9T1Hui{TUIhZDzC@OFsc&B6QA!
zxQD>p05{9N*UVrbjq4Vyso3CM<Gs7;z@L2@ET@&VJOTtTn6wYMUdfNEj%8K<=>Ee6
z>Fas>J}5j|puJy~I|78f^F{E=8Aj-0Of$<1kPlK>!0FU7DBNyYqxLa`el@Y(81(N}
z?q;RsLILP~b*k&nK<E84{)lm-BZXyIP;DFC_HjPu`9EPbZviR4aa#dq?2Vs}ac5O9
zLOW6f0Yr_&Uew-CS%}7mkNxwHe+s!%$_)mV3d=>&xv77E`v^2N>c%Ug`L1SnK+~Qb
zn9H@?Atn(wbqn+nvnFcLYnioYHIn<F+^Z6a7Jw@`b&~cqTU<el0c>ZVcLn<3+E0tX
z1#PJf>PS@_^78kUkJTW2F>g@JAx8J;2L+4PSX0uw_`e?f_5IQ;VPHhrOPRU}L{z}c
zbUFgn8o>hhuJ$p~hd??ETRmg+{f9r$Lg{OLJE>^xGO*`m(DQ$j!71=6y?N{eSae5E
z2Bpz0Lil?Ri(I8Uf%nub=MDqK;0jTGn$SFH$#e&&Y6Cs{<I6!1%o#TS$9L7qpf7=R
z_<2Wiwg4yM1*(;%&dUbq-qA~7H-7|nqW7FtcrjoTG;tBk38cM`3=)WO&IVV5+-Hwi
z>vSY=Z`%5DqFvjetaEiAJQl(D(c#t8zs6{Z8>j-`s^b0~@?sg4SzQonO=aQGZ$moW
z1Z=(d6!-~aLTv{$KW|?0JZ`BybWod&@rS4bO(Y-YL$q_hwVyju51D1z)g$ydhuc#m
z8m&+MvL5{A1MFbHkGAkul&1m_D&Q^xy!4}A@HFc#{ko|yvu#Hj*#nG0`!+6_BKC*U
z_CC-)Hgo0~B2<Q#K%7R&xR_KTTvR7)G_<$8_BeM(`{6=nSQaRhk)w916e^$&eD?v)
zyHjl7+$)!n>yJZBi_u=U`jWJl2TX&NO6dgWm80DBrU6>&z{VOqlLOiXRc3;*1GoaZ
z_B~!SPYmD#S`C5H%{kUwveIJ$CT>;F#_qKCW<RkpinUiUkRRTasjh8N9^q!GM1aOl
zwbZS?`6INLpxVo+C3Rmd)P9<#%?I`?iQV7;-C#??@3iJcSubYBp><T+%kj2@g%gNB
zJP8Ic1_iyG`UX#~eH$bsoiAW2$Ag$oEUf^OFlVGROj_WLM_{VC1$FD@m;GC6npS&`
z^&vfs_Kqm9T(O<T0$STvG39E11ECKJX$ezN_sX9$avNsA`YyVTKmhF>J-jH;Hqw_6
zVuCWiS_q!a00MYp19({j$RnKhjJ$o$KyVeJrd4n9j;m@e3&Ez|Axgf8G7+EL&r1P`
zMMI)6GOzG#^qd*=r*GiHL?Qdo1ojM9+w||Bd02YFSTqGM`~)(eDf!`(U>`i<z6Q}<
z4|X2@Fzqd|P?pAwfoI%!yq8f6l!N*}blkg-fN9?597zCY*)i5P#?!ah>>029JMTpp
z4T|PC00b}(fkNii_}I`CL=;!WGN6Og@$-KF-U7zWMwf?lnASEf)!|PH+h`sXhz-1)
zn>5?Xnf95(#QeeCknQ1MZLwkjUokNYlR@cRl3)oUK-6Fe)`51;=fl4gBV(c>QFP(g
zA0JMG#v;H(7zS~Dhbkzd`~LQKf6hME%|MWjfpMrk<4kBiIsWTI4@y_5>5bD5*=)`a
zz;;3GmF|1TEEf2F*5>f|=7M;SUQXWk@4x&N8L%h<oPLVq%|X=-E(56a;RMz}+#(%$
z+zYkn{*)Q*7ZJ)~I#P$}19gRQt^JTMWV99r%q?}&3i_)d<E=kmK2`yawqQ&f%W)kb
z<;E8ttDMh~eoxMg9-Ls`5r|WB0|B0_C6%Ic3}CM|%nm>W-7>R0R7@2U%TQ{@z<AW!
z&H`hlfA@%H!2s7|@&vnazit#|^x!rF0|Y49Wk9PHpB++vkdo^>|L&t=>oU49vSS&I
zDd}RX-0M00Mok!^oEU{%F1LCq8Vs0bIivwt`NyZVXTn+N2GBVB0Iqe(7tBqqG!;6d
zj3)fD_vdPMO>~{Re8GQ{I$gGm0M7!^QG&kYeeIuG8AaEBqHHCQgMQxX7}hUl*JY5(
ztcHI4JGk~r12)d+W9{w|!gJU7bY5NLz2^UT>=|(2_JA4-J5#aIUkn;kLwg}t81G%z
zAB-xCTnj(q+CmtoERW70*UCiC2uPJ`(3Jb2{nKM#4c9(w%H0MmV_7HjkQ3a%HkQl5
zuJQl*S3ibg2Jvc@oGW(<7`(&=_wJJ)g@E&NA3OloK5PTe7`DECFQkBhd3a_etYWdK
zO!@#Q%S&x<GAQCeedW@V)>fDYt*LLl^ts0?hjTb99(x-`&>rq-0Oxi9StnqHqK|a%
zL8+6*0MigS7IE`&`VU~}0NLZOJn*M4eF@rC5Q84L`2&&Qj6^{DcN0i;6tu}3zX0C_
zWzjA40C183*3-cq;C2J={X`{lJsK>Pf~gC1VXia+Rv<6r@uoMK&w3<q&7plj`x~$e
zwA_GT#^xT61oj!0K&TrxKXv2N-#Li^yc(pgE4Ino;|yjGP@2cQ+l{+C51~d6Ub`bl
zdCjMqQ@gasZ3Ggg>Cu`G?f~!VWRV6cdAUIohir%i+yZK2jHHN(h1v+3lBUkNGL`($
zl$W!rz0lxN%jqq^&s$yWm_ZJ>g4`<9wARJOyAr!n(yP5%Hn<yeUM*bjEgk<o1K08C
zmsNm42b8Ozz4Cdn{NXP|(jE0%V6BG?R3?h1FeJ7QN7x<!0iicl8OYPVFhYzxk{hy~
z8JGsaY~{Yw11do26t(*e%0cVPuyyZqPv8CC?cR<~VnPF}tVaS~p`JT60a^{p8>x8V
z3205LV9uX@y$T}pa70UC&I0fSDbKH-elSydxFQxrZIx|M)I48jEdjKD0%79u<ZZJC
zu@3jni*i!LOjh^wWU+zwzW%Fr2%hi1sM>4aAGkIkt|4ZrnrDsLsa_5hTUHCC{jz0Z
z9b{Po((8{Kr;`1lrEq*EeS2oAh6bOF?AojL%trSN=)hn5H~v7VbCW!I|0Gy38S`dQ
z6^!Bn`VL^!TWd`0d+u`~{>P9;P%$WMVS{M}bTUysdUBi_!Ya@O(h+mxi_QkLX2Qt<
z|JrpYIu>9NWJUw?$|)**cf_i_`vEG~DHw>4g%X_2MK^(x_nR#>R&`$q-glEx!V98U
zf`+#1mSZ5*g}Oqy7=Rl;3t~Dh2Mom}<xejJElg*>{uLNvUjGW$nFNYrG;+a-gSoj3
zWWbG8EYD>aoI{?!hpvp_x_QMgP;JI#aKRqa&4~lK$o)vLO;A;4D>!E}2Lj^N9;*_`
zF-Ou9`ocF;w71{3(m%Wgpz25h7s3q{c#5EF0VW5Ze^!Ii^18;j+=1B(%t>l1V4;;c
zokOl=tZ;0J;F+_?LYWE+8DPq3iBJWBw$}a2_REU#fKy)z0^<%dTFydkP6=$iAOmPm
z`njt7S>=Wrv4OL`l<?|2qX)M;oGu3@N0RBrkn$|>E*jTCf<5JVmRy0^vm%=AKmG7k
zbM5KRFv3^`@{iR%DF!W^vje(CC~u-+UwOU{nsxLTH{zJNLpKA_mQ7nKWTiC0$CLJs
zJSG>U-NpUY{!s<;lkcP)LU3h3nL%WVN^fatZ<~a=B~5cr?x1KU2(Hp%8C1c52CEq0
zxseB6#)ym-3!}dol-OJh?hK;E&5e7y1`;qTn8Cm(_w}WF0t~4v<Ez^-+)beLonMFo
z-WC{ZWHvCifpigv1+`Gh>KSRnJj;~W(+cPVOGiPwx>*hJxNOi+g^Um$Vt~{>l9Y)b
z2;3E;Dh5%Z%om^$RAxvTO6}kx1%VZ=nG69sXrdY{?MONfJKUkPTb+k5L3V%+>!pV-
z0k&;LEmiqYPM-@IQUX;CTced<;!X7oBNc;cC3};{T(gHg0MT088HrR|xy@|`m`^}B
z==vAg$OO@`SN*&XFSOP$V@o{~1|YUUfTdex6HDIs09ywjdnCq7Ke!s12Q@-BYBx>2
zl^;THO@JB~wO^E__ko<cCU8bEz+@N`l`a7oL%B0xzRy0+b#8$;M?lq>a%m}1hL$O~
zj77(u$gGIhJ|6<kGc`>4GT4kDhKSofn1>k*aId&33RhXnZeUDAd%p{mEwUgd4>bwt
zQenm|<%DOb^r`6JnKwY%%XmnR<-ImBtp}XJBpBz_7T>r}H$b^O%ZLFuY(RSt6%W7u
z**G4{J2VjAlAvO#OYM#JRg%(4N9YK*_<5`78(iPCM}CPm;v1eaurR-aH9ZPMZ@%}0
z+9~NRQ#GKfaobHS;CrCO!}Jq~pgDUa-g0pn%uV1zRqPC$KElFum=e4K(fI5=S}MHu
zxns0|T1HnJNFCGJIduuGd=L=HkOH<rOr4as?_&M|QNbzTF#$EB5n=%6#tV8b`DAh%
zSjU)3_GsZvB%OB%>(<}@;G2+P8^*?CMh+td@Xw#~S>lc`wShU1g|6=eg<3bkFP1jk
z{1&8u5za+(5xgdyAn2o)nTde*1wASb#WGkRvZ5A})qrWzMP(5LFjzZ3y$k#Th})>9
z6aW=S=IMb*k0D#IsN<C)!JEoodr%r7t&mAO3n~H<=#amE@N?i?HgK7FNB=j)fiXM)
z-UbBzA_AkC9IK$${({lpIAwE%@Br=ao&M{0#TuH)Ltp?Sqz~j91hz~c+M-{45<)Oq
zfBK_OT`;segC&5LA~za68NOUgCA~XisqolmuGBR~s=)BzT?U9{`Qp4nj4Cn%=zoUL
zOdS}brNbH(A4BIljrW((ck{U?jBfhF%&P(!Z8jsv3pgkZ#IQR_8wF1dn~vcV!<qL1
zxHB)N78{##b6JA4Zz*@zzAz>3=J#PxGgaA}r+hNFi!L2m87jbGX)G{zaLc2Qz|8H;
z7Qy~VN3f-cdD2+M@B{GoUs1MV^XMU9D|e4mHjRUMiD`A@<$FZ~_w$|xpXOEwvX3A+
zT2RNM1<JQ9M!+Fxd2vY^4I*Y!Eu!eWH_rh5#Rpoo@33mx&^vsf`I@8jHk9dYC-Bad
zRKtjvd(QSonuE*Hrq>ylI<lBOZHCL>VQPj@Uu8TEfOMpG^OBvm#UjgrtD3J5Q$R54
z{?<cW^*pGK@dnI_pl`Tc>{2fDWC&D&fMn2Hzci_I2yTDrw}1TMld53mJpWr6RA5%A
zm~EXdA5G;QnSNKnAY<&aM|<?v_s%e=G%J%QEl*kti4~&EM0c&3ycoD0wo5MR+WTwl
zjnv-U#~1~)*AQh;+MoZeRvTXYhQCXXvGAgDcq^AB=w@B@;(ldmKlD8Z?X6$<N+)<I
zdu}|ax1ngCfGWAzUVSllXjFTf!zJY+be?{ReOp#*ApP1~qFf1`$yS#L2JpV4_IEmi
z#ZAyUN|!|!gQc(}(VM&#ZYvMEnppP*b5l%*CN2P=o4Ip3HrK91wW-@Ej3dtK$oN)u
zxoBAHBV>GNs!Jr(zA5$CqYW~Zm1Bk;PWNzEjz&Xp`ZuG&ECUQo1x`8@lmrqm2z7AB
zbnHP>No&)7f;qCCbr;}mL||Mf7v+`}?PY~T4vD=EpE93pUYl1-$x$Kv+SeyHayTKc
zEI;o|7DhUe=W2=VXzcNM=?<t-pu)1SIZpVN^9QFJxcyd852%iT>M`M?lKI+q(G)N)
zyTVJeVj*9Bki~Rv=6Ed!lKB(Dg9Y565}zLKjlqrPH}3^op}T;Xv~Dl$r$ZSSJW{~G
z1^Tb(7APP5L^(<O^BWAyuFjKZ2SBfJn$N7_ZIv|WF1_0jqt5HnxVW$o+UDWQQqsWC
zDg`lnc;lukdi=ayA#$~cl&jF(wUX=1bpXxLjo_nO)=Sa`4_nS{>7xfoz{U*S540H>
zT3oq>u}}xPC5}#Du&sa<gLJfU@tpEpG)J@A0V>3`B=ALh6lCRDJ-GuB)2-k-_A-ii
zWwm^03}p^}?T*9o#S2il@_4l*&cM^$I#~&+f+31J3x2s*-V{VtnU;z{J3K8D>ecV5
zaBJfZ%r?b?Yj1yvgu=bFUmFdeAh{wYQAKw(Lu>{VHptq-wMI4VwGN&>*ShiQ`<INh
z5N)Xz)<HTExW!Ruzxfy17evmLMe=k4o)VyQt2jQ1!47a8`^ln%JMW76r@Nkv$Vr3#
zw~zjL(TiUbqn~#=6fTO6I`b%vR%BPxQWQf(Z3%ZhA5^@TzXt*E7$XyYe!93lt%q*r
zLlBVR=e-wi5A%YlX=?(^doEF%k?M3ZFtXYxN?8j}T3!Se7IVvx?ag5APe16XjV=w(
z<ct{wS^jFl&<`Vf7(cnz@D0W{5LVCUvno(QD%dNZ<<tQNC$OEw<?LN#VFWxxfTt-n
z)NEpOn?c+q;GMfgPKW@n*EJ|@+s2u^GR_~k7$;Kg)|j-8D|xzC{tWEW+M(FMwdXze
z$*er+8~{{O7dVM{@e#<z_+ZuKhI@e$U3wyA060T6&<0kTn4}wz*Nl1PXl?p4T#e=+
zy+@QNughB@2dQZ<KFUx&<YEod<LBMMLx49p<wZJvhTEl;41aTBXlzshn!tew?zr7O
zVgtYa$l}@p*L!loeb@KGJTya+3wS7l=kzfGs*VK)!-b(j|3PjZYaDP?>oUs&{_dH-
z5d(N6>+AaA%2meVF=%Ux_X*p?EVKIcJJXVS2GtzVW(f)DCu4rBu^^~~gO=sovvC3m
zFl^3(R|THE*C)+`lWFASH3q4@+0y-(hth)r&R^ifi2!Ox9EuN~)`vqe>5VtP57o}N
zH;vlA7(rPNWR=lAP6G*4sH~}&E<>xlQLNx^Jjy-k0@)Kr6@|iRd!Xs#p&VWTo6{`}
zEFJ4AV_|spTVNC8>N%0=Pz}nB0OKK%+z^1yo)Q&Yc_2Z1?JrhF@=H%gv}=C~bK;f*
zMpZvzYT1aZ0mqv=PyC0WuvGHoOoQU#kkh|8F$lC<B|~g?-vyl8!uhcp009M8Xa%Zf
zf)*gW3MyO+D%`lMo@D-Ht}F{qAg%<s_8Bmq9I%>BucjTuP!VN@lZtd#6$LtzXV8gl
zY91g!ol&h#ZCQ@=sUaC??e%~~sd(Z%7i9?hrre^n-%yOAU66j`w_yu(^NSC2LP=!X
zRsZm_KZ8_9^5raWBtmUFOB_%L>U2q+(U&G>gO-L3rD#REM)y?BgfH9@39!51B6Q9r
z&}KL%6u2;|xeqGv;rCp@oul93pz~<p>q&sVRy*XwlXyhG!~^J%7w_O=GRai<jzI-N
zK$~952m5F+!U64DgcdGnTiE0J(sxG146dGx=)>Ef0u|N^O({(rurdU;G1ZsR(A~V`
z=Uu@Cy1w$maORfa?K^dqC8b-<WFtY$S0U`lbtOtY-%P^HP}cGC-B4}Pv_fRegQ+>6
zq?e;n2U;Fa&4lP~0h<O1dJiG5KiW3dRycP04McrvPh~_&U~e<DnnY=W$GmiB4YNNx
z*t7*v8&IA+V5A1T@)#J0wt={);sCIaZZHy;B)EBxI-v^SaF};MgVSzCr(h_mo11T{
z0BZs*gGx9Xpcmw279Bk5azf_i^udN|Yif;4Znvs{p3*;~1)%7OhoAgbwAxCQ9V`#b
zX`Y$$z8c5bsK(1b5Z$GWg`m5IAv78olhaIv-m~v#hH`q9O)Uz&vvVEb0R7JR+J5tA
z;DbyUK_3KX;OFh5+o06w4S!zc`TO*Cw06H>bmj#6hC+CO)TQ70Tmu@t^ffRIDkv;8
zl-L$l+q0bJ#G3xjeccD9ohW*BFZj88+jEPc(E+|=Od}wWThVXIRq;NHmiAp|um1wt
zHt70K<G=;ZBDQKSi87}5+`NZll>>CEuB@JZdFngAzT)4zC3Z{LwIAUfGH9f_tl)Ra
z7WLswS4IX9b%C)3Y>4RH+K1994Oi`JmX3XAA{Zk>pjW><oXXeGe%>m>(|M<vVmC0x
zipl~yo~o8V&)eO<-us;+2%tWPX%!DTV)A8ree@NG@%=Sb5t&y@3<q|)r-O6+=qBB-
z<I~4y#-zdAIg2Z3&`)Wg6}=<KpG$I+wik6K4+ATQ^XB#gBOMHF*V7vcbT4BHxN)m9
z2qF?B*ykApoE#7XN-4bdV+i~;9GrnDU?N!GzDpVrnYmEL|MdGWOZzN8lE!ZTk=hTv
z{*R+)jF)E|(<NhR;|#IeK(BQ($932XshHmTJ|teibJk<}sI$sy4>y3<F@OR(EYg|t
z_Pze1fvbe9nl+*Er>{LG9ooP0oq3NxtfA6#p6_uHX6DQhDE_^_v9kC?&8nb3n$Uh=
zRQpHj0d#N^I=Qn!tmfO7MmWEgj78d4xHW<~rmI8H4=TsFpaXqSnP@{I4pj_F%l|4X
zp529UOsYyye6#lPE@+%ioMmdH98$1hRK>_0?dIdS>DTXvqB6^q<AFXV*IaSAE7t+|
zc{eYK=4|S$hNc|X1hu+2rtNaQ`|^!1p%K6Y_Sobk1ayk;M^AyLj)CzG4X`|VFuMTO
zJHV+i8QfNGw_Jd#c6YBGx~dKNHt16z9VMb&4b&}AvPLpC=biV&q^7%hbyM1tElp0)
zm`y;a7aLum3x#ehtxkU4fA^U?E=2fw<K3G^Us!ZZxrIZIVtGi?jm%}>{cnLi_qj4Q
z*{SMsXK#~QHf+YzzXIkvDPyTU2>lIP${SZM_i#?YX|k7GP@!<a&)cas-ZM(Nb>13R
z`y6*tJLL3__cuR_W}QA}iQ74;y^4Z@H$n0miC(>)(HEgqtuq2y9s@wr&XL?*z_VpL
zfK`mW29OJbrF%hno8GCW6$Bini)<9ZGu48er3s!7>h;*W^49n5=CYRYj_UsF-`2h#
z)4q!*b@{A<g3BUqSvFTzK9U%~a`e)7WB@pe#~&VDoq}LY0q=fY<Um#H;su6FxwOGA
zs*q4MBke-j;N-lzH^-;PV#Mk97gWhHL088`;i^5_>6u(HfOm(zcQ?<S!+EZO2LI{v
zziADF0wjo_6+~kXXc_dvlbmcrXF7dl09|0fM}GM2`Q0lM&;$pe6AOA{$~b98<7)2_
z^8`@$dle;lBk9bYU~a&$3{bhys%n@R!3RLKtW;y52#`+)m^u%%F3z-(*|)~X1`dnC
zxRKdB<k@Gy1P431dRAGHM%`ZJx(RAESMkXM{&byBqTxt-%OOTY1?P!WNI|1_fFOoL
zZ6ug+nGT`@IHhb5z_l)q8l*8p@2jQyx*VX<H<Tz{LEPS1(7v>F&ik@}`%qUYx<^mQ
zh^zlbh~DW0&3&eMm8VfxYlcdP01TwMgJ=dJ=f-emwJ=PAX<uA6b}$VUxU7y~kd3FY
z{%G$*(B8<+*#l8dEYmFUG_aTgdn!DWD&54Ob5(Jq?b@=wn*=rDm4}(hEzAWLud~=F
zPKor*clEzO&ss022d<IZ&x=l;=|m272~0i`4O~TEJxj~+Ruj{|#TCpG)RsUia9YF?
z20FDjgcOMxeEapM84^qafy#5e4J9tvB~{ur^g?@Iho3hO0z$)L8P;L50h<sJ)pVj@
zP3^s>0c~#dyWaupV~hZUpmF7Xs5W{yVmqpA{`xNyGf&k1u(S!{ee8ifH>a&;Z2n04
z=vMljGs+dy_DpO7D=NU76`)E+8a-pJ2Wu+N;RNZf4kMQkWC;#}c==&4ppiWW?!$=D
z&pSno?)lMB7>NDh+>gBHpU#0`h!`TqzyW^|qg7#}(2~8tGGjjenONWY66Ch%CG;3`
z>G{uyA$>xma7J`nM%S#mJ6#BwTDj^0+fIin0$fGMIvA64dmfDQ*Ky*SRmVh8x%u%!
zdJl&J_5N29u4K^(LG=y!`&AsQ{l<Om6~;Grgs~L|0!AuC3QviC;X7-bS=;q{e65h$
zms&97bZlM9uV5Af+?^i1gS(nm2)11pl^Z*%&W|=b-cr*XA46kPZ6{usOnS9)3&#@w
z^?~m}#!W7QYo9cx2Lz1}FKe$<rEO+mZTIv3rzf6Bb%DxIsFp6_D<Zg`Z$1U4!fu~W
zM*wHWdAdY_#WF!lN@J9bdq~;>F*JF|Yrg@z4H;;Ah;8dKWK6j<=6Nuu&TWndF%WJ;
zEXTHjcoCMKi8DWBeqw-Osu)biAMbfk*N=1!O72z0@98&MaUt6Cu?*=Ku^^)M?aw+F
z@p_B?)uj-;N+Kvb7|hLlaQe^Zuie!P!z!2br_VeFKK0&%2iL!l4~_doH}N{a#gf1w
z2+O4d-r9hA2Cj~_BnDe^VoZXVoGxhJV%^SYR5{Pb9pKEj`0)(sO$=S2o-WFK`H`|Q
z$OX#~=Tun*77IE8F-|X>FiHhg<+>Wl;sG#Fb5;6O>b-T_oRsN}FMzwZflZ`=HJ=t!
zXP%h17Z+zR=E{QA!*vPL>IT*YI-!>sdrhDu5YQg!%CUmZGFOKEB<N^v``WJ>Z~Opi
zmUM#O{6;Agm}OV6Sk9;cd`?7a|3)Dycq;=ub@ZYMA|zVObU&~dKArOZceCt)HhK^M
zGu_49E~Lw-JoiJ+J`AM%PZ6)S2kJ^D*c7NQ1q`swv#R#;7BP6~#sgV4OU5SGz5MJ0
zrPVywqKo#?*ailGDn1PRVU_w*<!I>8e#iJ~G2Pu&#(i-$=zUDkI6CKl{raxI5^FLi
zs#eJJaQzz%i*&wn6g0uB4~2`u4FNa?V63WHY41{HxC3>2d}a|WEJJ?!L$KzHDm+5?
z`~xU=7D{WbaRv%A0N_N2va}bj?&`|0^w&r)I5Q++Q*T1)Jvn&xt47^|_j~{TXJ|1K
z2lsTdDA(?UY{=8C^nbwUK(QDOxjOEG;39KcV75TMO!s9nT?M&3ER4@{2PW<7JA$AV
zhW+6iD)O%VJm)m*>8l+B*aQgp^d7J=Q2%MY-Y)7}MZ3jx3A}x!oWY{w+ApDU!^)M@
zZ9Gq%-hT+q-N^C?Fts_!tHWR=AVFtT`sJ?Wqkn(m?$h@tbZlnV$MQZbhzc7NrMjx`
zm9HNaNQ3u-V^P7&L9gD;V`f3!AfQJ?h7V*?w`(NCCy#<LbQupdiC&$-&68Y_>%!>!
zPg|j!Mc}*l_@b>Y7y>o7S^7^H8LXo8>_xFL|8%qzFxr{zp}Gyq@iBdqeAXno&-FW)
zyPP7B3-X9)ao(+ph2}QfxnPN6M)RJ$5K4bLu?BI6flPWijgcR91h(5z2F^8p_!yXA
z3}Q~yuwCxaMIVziC$>KQxw8lKfBWVS59vJZo3&s3t4*kk$DqJtEm)4YOd6_Q`7JoB
z-f)|mZ;*XAVFpq_K^4sA@_Z|&2jncg@eK&ho~!!XM}7>BzwQr00F9u7^6(>wnJ$C5
znBfM{ebMvN)(*21v{99?&Xd}YX*lh}cG-9SU5*5}@FEaT$<i(0ayvwIG3eG5mIhA7
z0-|2^caJ^@F$Q9?>)<tX#Ax3&J{ZVgU@QaOogx`+^be3$X9(k5nCje1>Tca&?VO81
z`}WMtN!}~~FAzO0;Bi362$(BJG{h8N(7r}UI(U{G6^c+977WB(e|0wK0M9wuEk9u=
z7U7}s8Oph6XMpI1;pFYDfN2ITcdZ@-)bXr3;>?%JdKTOm2UXRatSSX~qIO1$`=bg>
zMHO_~NX73wQ<(z^q^k2utBvz6^{i&%@Zc-F{R;9^SpWrmra9RzYg=L678oFu*XdrM
z&y|sJacRyKq8d~}SMY?pBhNj+Df22U1e2o|SK}dpLrwsQ(0*#FNc$EF%c$7EHwVCK
zK*3B8!KYt@RC7Dftr6LS_Kg*53sWLpblzQpLAESgN#AG_v=iw7uWMq!vXE_k^o&@G
zA#&TRt|oCEy`ZS)?vtL3k^V%{ic{2_E}&f%*mC-6d>+k-eMY9c>8cF9jeq>h7w<_E
zTeANcvt+e`*`c;to!=lM?8S@M;W_}L2<EupU%vgD9Y!}ZU>fB9+E1iIxH{K@v}Pz7
zV@A4HFlu55PWpq5TVs85Rb6NJ1FT#B_~?_;5Z<!6CCXT%n{HPZK-*$ZezZmhw68gO
z=K&cR6$Kf$55`#>ictwPxKJIox4!VnXQUNm)2;upvVYWaoZXekT^!R#s_824!W`4z
z5zkm<xCJv#KYSc4ClBn%0#-qX?z~M$PJ-rJSu-!q?Scc0iZPy(r<gt30srvVfvpg=
z)PZ+?CG!7z>a+LF#9Y4qa7|1IEDfOUDW4eWT+C#p6vT-ziJ*HbL8s3NJaxHr33o5$
zZtq9fqd@G%41nfHu&kr5uthO`c&|wA6%TXTllE@LWK&e)=^tkotGKL6XRrnC!WJ_{
zP&wzCp17Khh7+B``zp{H-gQA$!TGGcm%js(OAVO6Cs-+{LWLNQ0eptk>Y&fl(`S)=
zf|k3o{EA%-e%@{jNgW1T8OzPldcK%)SUg|-OKEf+y3SQEmT}GjJC!@!=^+r?^&plW
z?I%kbwzHQOlQ}y<GZ)=8P=5XAklUck2b_g=On&QSsPxz;kAW=}7r%KAWNaqbI0(qf
zv;;%++t$EY)+b(g>e`*7IU9UJVFkFNlBu-?jYg1;X`UG1t_TWrjpu9wHN5*h@F^#r
zt>rWr7(_s!Iv(nKcY$qIbb(vAhE-6rFIX^Z*jl0$oa+Qu)t*qWMsBrpFn1`Vb{`y2
zp=dw7Eqxm1%fs5w7BXZ{e-+2idtvm=AInkuA3g+y2bn>esyf%7#03WRz5f7Adz1hE
z`_Dsif=s&<&T2ac444oBZh7H;a=l&k!qcKR^f9jXD#5ra!LqK+aWy+^b7=z!jPZH%
z5cqI$+;CLerC?@--Psq>VwA8LmpX$1O3=YJv4pQ}LMa1vfbu~e+{o9z1jadPr415@
zS%3XLZHwqX`T&T?<W$GC_v*6T0h}Sd#0zj+eL&oTvoIOZdgZHNOQ2Ng;vNKZk=jLK
z!<RV`fXiLhzI;mBI=5t`=9w75z^;e|5nUhuq-P^}d10kk+C}UR0da34#0Fk=VqwB@
zc241RzQBF9+Fvx5KLICE2P&N}bLwshRKd2xmumm#V-J0G4e|)&%KcCo(bNh()vdwJ
zoh}C6`(6O}I%vFAxu4}FBp!@gvIqir?R<|mCkR7<n83NR6KT1K;ITSXxhaUxPz4%2
zo$(pbosi=Ou7h6YDzZSiZ+xE9*!1$`=zuyVvbybFaSE(L%L4@9JCa=YMFrpN>-fjt
zG9Bnvo;ehhT>~~#s{QN?wA;`<y7}&d)^v3``nZ+LEn;M3g18~kQ7_%ilXQd1L4u5B
z16{z9Ix*6xo1rkmvd<b``)NDp%vm+EssMEc#$XU=JE%6so%sk9vhJw4h=oso+*hCr
zlyy7^ZV*Fp1*BFPV}ezT79U&O?D3P!62xoReU6522bjG8GY`kP<_r+0E@=I?JAMuA
z)d#^;t(A@>h%LHjQrgZ}E<>4>hR#`AfE5v0N$xbHNNWm0d#f8HlL{CUAv|o6;rh{2
zV_g=8D-dO$7h29@<Yv9{%`j;QZ#k^s5m&sx=qNyQZM5ilgQLBw05O_Qe~*Wo*PO$;
zCP-n;VB*JuN*L~foehzHxbNbTZiK?z2->HQiy1CJwOJ&Oji9?bU3)%y6w)Ggoin5a
zMQM;bDsMRfpq2Uf+WoFC-N#Kl{WkM8`$-s8m$;R?fyv-p$Os59fRc6(q+zVgV%UwB
zFsTHS|9JPk%PcH@-tM}wh7mdks?|F9WDYPn&!f2YqU9fc`LiV#>576d7|OvnwXcJM
zGQY~*4_{2G^8DaVXaOTkNuX;i*O1OaO3%FhYs-=<zeh|7v-#yeOI2=ZT{u4QaaTSq
z3#^Wt)Be=}qxxL-$UP%{RdS$p8+aE;a%0laH2@a|bh0R1f0$VkUYR@`$_3<u!UDm7
zc;9+az?~S{SrgKxcY$5dtB-2;0A>xN@CcM-OIJZ!Ri>t2`RUKm0kAG6dNyju47~mu
zF{WIC<oSJ{yQ7+(Dg<!@0PUNVMHT+iy=Pm{wJ#7}%oJ<6N-R#mtDS+gDIMUvNwrGr
zgI__KQ!X;9>_vi;&5;4^5QA%?gEOLm7L_tkHP>+7g5cnsCTIiNzaNQ-nO6Yfv787{
zqLKmXqFC}EtXG*~Iy^+XCDvH`bfo&Q+5$QBk!wj)EbEnET>Hgk72F3|$&52rsC|8e
z_T|B7Wr-!9u|UrVM*UTcp^5Kq)qZqS`*-#Lo@q>7s6hq?{M#4525rOcr+2<7J<2hS
zYQ67%6=t<VzB~ft!Ud|eLfh<^&4pkF3l0+?NrD^qv&1vYX?y+$W3v!^7y`Ii46L^p
zPpJVcilwvxFMWx_2SHc-W5Aoni$A#+l4-4d__T5e8W6_N-qbz{sX{=xETm$X-JnXI
zle-KsXOBe%Z)Sa%=9Uh`ZEUou-PXH#9LrXDw)S>di`sjdgWJG(!RTnP5#?B@9{==(
z?_@&?7?~VC{o-GOpFV~{^#TFfT|rH4f__v{&zbPVF=Jo?749mtIo6iK%mo%e?|xyJ
z87L^!wWScKmD(qb)IA67s;eH^@Bb7^xIXTz(ujuhqT1pYo<xgbJazqEe-;@XkpoZ}
z6#|vOEZ!GDoIP;*y#sa7OFGGcDmrND`h646Jq6{XpSfx8Vc1J9L2KiPv7)`wuR5F>
zWu)|1R)Paur(m$GT$Gp%RxtZ4w2$q^xyQknhPTbF&2HPatW&F+x;~A+RhsgQ@y2*f
zEi23I-QTe0wp(pevs1RI+@76j8x4c)ZMkjMwRzh@e14wUQc;0#fl#L@tK8d|npw8m
z;-{_ii)-bH|NgsY65|(V%M%9*+l}oTwbq5jrmI`lo~s?bGgp^qEVGG~6=i!Rd#f9(
zxJ>-s>dM-7$zFNl)a?4SwFElNxry=G5ZiKjVt$S(+iWx?;-Y0+vvVcgbq(q$w8|3~
zY_^S(+}z#W-JIS0ob|2wT$9CO$u;KX=H+E$ME3Unnr&(?du=;OKSiB&duD5K!?w7-
z7SB^oO|S3R$`cdg)rmLGoOo?JN1IyC%=&6BI&w`p`MKJe&?xELTA$sSv2IngFV1YO
zZ?DhUo_##Ne{p+p9rE(p%Ho<ezH?=2Ek3V|2iwP&vDIqb-(BCD-L8<%GfmFw%q&mW
ze#Yc*;<GclLpFA{Ry23a&g5EGtgF^F+cxHIlDW^$l-L$+E7l6lrPV7oQC1e|ff7u3
zzC3X+d)7KPwX<SN#QZp+l|`j!<8$rjfHs4uvaN5)X?m=ir{}9^np%$^+L>Cr?G)M&
zo<ctm`mB9l{+koK<xJY-&$#YC>rKR{?#^4nU~g&zt~81(F{-3iR^Pk6zVe@mgxe1(
z(+$?TDVwz-&uFq_8_n5y#$Ho?NkLIbL1Ct`xWs5I%e`%+c4Yhd?Bd-1f9A+DBV|AS
zYhGz@|BwB$x~jXTy}hJveP)N@*IZYzv$Hr`(ok4aTxT&BWf#=tTe1rZ^32)Q`GtjG
zmb`|7LSsQep;=q#t;ceyur#l2+oslLtpAm>ojKOg^7*6k|C#^gSkxh;gI>uh?3EBz
zwmfkcb?$#ta7||lZr$Sc#>&(_eALT)jJIv=SXp0j4ppA>&WzT|^I*4iF1Kg#sx{t}
zmzO&*5^stxh|e2~&+T4cx1nK-@6K&rn_G`JDZOr_%ve-ssVQzK%(oQO<(Uf#=<u`t
zuK0f+feSP&uGp+wxeY4_(mLzR`mFWzwvcBoI(t{(lf1g;=G9IG%B=~Uk4hHMY;tg-
zZiEsEir2pCQ{6Q)crP9G^f^0-cer5fTlz1bHH*Fu!KXhy*l7>tyrrtAJZO_3o@|6O
z3jiH*QhZ1Y3;c+WBF&)kL$G8J&=axYHLxM?torb27<R=D-s|Qrx<5px%Z@aLc{9Gc
za3Kx86^pCSC={0f?UI}h-cQRxyg>s7dcbK4jE_w<>XxlmtX_ignZtO&i|&~s>mpN+
zMCyr2J%dFxp(h3eWMoz0OKzt)AOW9c7bCZ$D!eG3i}xz!LvZ%{cOAWdXL&Ky5V9n)
zZL<7yFf}1QVCtZkfiaNRfH8*GK6c`DcoB!uzPDw?`{9fTRlIB8sQxn*#upV9<`>4Z
z>(pkB*R=RG-DWB-IA_Z%z~c)4J2S4(RD8P)aenUDe2eAWH42P{=gwD<XR_RWKGdZ1
zwxaxV*C;41K6k!Cllk2I3iAq(oVTu{Fu&;BIHS>2<j_~={a5T@oaMX?k9+5i%`-W$
z<vFnBJFpcvusPV@JO}H@Gdnoe!8-CBtRvsSI`SQ?Bj3R~@*S)LdwI?;h|u#k2kXdp
zu#S8O>&SPo4#e}hYalkx+Z?Q;z`;5Y_vepwu#N%;>nL!rjsgekC~&Y2<k7k77CKl*
zp@Vf4I#@@cgLM=-SVy6Qbrd>SN1=mtAg9mGxyZpfiX5z?$iX^_9IT_r!8(c@tfR=m
zI*J^u12y9OoE@yg>|h;c2kS69SclocI?N8%VRo<%vx9Y*9jpU2@!UGh4%UHsdTv~?
zgLM=;SVysgb>N<I{(KJBQS4wH#SYd{>|h<m4%T6DunvoZbyys%!{T5a76<FFI9P|p
z!8&jsJm&+8gLPQWufv2p@9oE$jOW*3GM-<D$#{MpCgb^an2hJwVKSayhsk(;9VX-X
zb(o9})?sq64m>cQTbs$jI!q4MVREnzN6$Iu{b$ZU|EyvzJog-tUu=P2ZhdCRx6r4z
z+KLK{xaZwwGv^~VZnqVjeFm{@O)aiix9~@nveOqA`Tw84qL9}NQ(HE9Oe`{6^7v_n
tl={nfr^4x@BP{>FOfI)`%X%eVn^P`dbM@F((Sp-%o3+`9msM)k{|{4oPQw5I

literal 0
HcmV?d00001


From eb2981fb4d247dd9c32e74aa5edcfaf33bcae0cc Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 20 Jul 2019 06:54:56 +0000
Subject: [PATCH 104/194] resubmit "Unknown" job

---
 src/ClusterManager/job_manager.py | 34 +++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 0d4cf91b3..6b524f1fa 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -42,7 +42,21 @@
 from job_role import JobRole
 
 
+def all_pods_not_existing(job_id):
+    job_deployer = JobDeployer()
+    job_roles = JobRole.get_job_roles(job_id)
+    statuses = [job_role.status() for job_role in job_roles]
+    logging.info("Job: {}, status: {}".format(job_id, statuses))
+    return all([status == "NotFound" for status in statuses])
+
+
 def SubmitJob(job):
+    # check if existing any pod with label: run=job_id
+    assert("jobId" in job)
+    if not all_pods_not_existing(job["jobId"]):
+        logging.warning("Waiting until previously pods are cleaned up! Job {}".format(job["jobId"]))
+        return
+
     ret = {}
     dataHandler = DataHandler()
 
@@ -203,7 +217,8 @@ def UpdateJobStatus(job):
                     k8sUtils.kubectl_delete(jobDescriptionPath)
             else:
                 logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"], retries)
-                SubmitJob(job)
+                killJob(job, "queued")
+                # SubmitJob(job)
 
     if result.strip() != "Unknown" and job["jobId"] in UnusualJobs:
         del UnusualJobs[job["jobId"]]
@@ -324,12 +339,15 @@ def TakeJobActions(jobs):
     logging.info("TakeJobActions : global resources : %s" % (globalResInfo.CategoryToCountMap))
 
     for sji in jobsInfo:
-        if sji["job"]["jobStatus"] == "queued" and sji["allowed"] == True:
-            SubmitJob(sji["job"])
-            logging.info("TakeJobActions : submitting job : %s : %s : %s" % (sji["jobParams"]["jobName"], sji["jobParams"]["jobId"], sji["sortKey"]))
-        elif sji["jobParams"]["preemptionAllowed"] and (sji["job"]["jobStatus"] == "scheduling" or sji["job"]["jobStatus"] == "running") and sji["allowed"] == False:
-            KillJob(sji["job"], "queued")
-            logging.info("TakeJobActions : pre-empting job : %s : %s : %s" % (sji["jobParams"]["jobName"], sji["jobParams"]["jobId"], sji["sortKey"]))
+        try:
+            if sji["job"]["jobStatus"] == "queued" and sji["allowed"] == True:
+                SubmitJob(sji["job"])
+                logging.info("TakeJobActions : submitting job : %s : %s : %s" % (sji["jobParams"]["jobName"], sji["jobParams"]["jobId"], sji["sortKey"]))
+            elif sji["jobParams"]["preemptionAllowed"] and (sji["job"]["jobStatus"] == "scheduling" or sji["job"]["jobStatus"] == "running") and sji["allowed"] == False:
+                KillJob(sji["job"], "queued")
+                logging.info("TakeJobActions : pre-empting job : %s : %s : %s" % (sji["jobParams"]["jobName"], sji["jobParams"]["jobId"], sji["sortKey"]))
+        except Exception as e:
+            logging.error("Process job failed {}".format(sji["job"]), exc_info=True)
 
     logging.info("TakeJobActions : job desired actions taken")
 
@@ -365,7 +383,7 @@ def Run():
                         elif job["jobStatus"] == "unapproved":
                             ApproveJob(job)
                     except Exception as e:
-                        logging.info(e)
+                        logging.info(e, exc_info=True)
             except Exception as e:
                 logging.exception("process pending job failed")
             finally:

From 8ccdbeb1223608d608e66ef1952db1146f851fe4 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 20 Jul 2019 12:32:24 +0000
Subject: [PATCH 105/194] refactoring

---
 src/ClusterManager/job_manager.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 6b524f1fa..37e419dfb 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -132,31 +132,31 @@ def SubmitJob(job):
     return ret
 
 
-def KillJob(job, desiredState="killed"):
+def KillJob(job_id, desiredState="killed"):
     dataHandler = DataHandler()
-    result, detail = k8sUtils.GetJobStatus(job["jobId"])
-    dataHandler.UpdateJobTextField(job["jobId"], "jobStatusDetail", base64.b64encode(json.dumps(detail)))
-    logging.info("Killing job %s, with status %s, %s" % (job["jobId"], result, detail))
+    result, detail = k8sUtils.GetJobStatus(job_id)
+    dataHandler.UpdateJobTextField(job_id, "jobStatusDetail", base64.b64encode(json.dumps(detail)))
+    logging.info("Killing job %s, with status %s, %s" % (job_id, result, detail))
 
     job_deployer = JobDeployer()
-    errors = job_deployer.delete_job(job["jobId"])
+    errors = job_deployer.delete_job(job_id)
 
     if len(errors) == 0:
-        dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", desiredState)
-        dataHandler.UpdateJobTextField(job["jobId"], "lastUpdated", datetime.datetime.now().isoformat())
+        dataHandler.UpdateJobTextField(job_id, "jobStatus", desiredState)
+        dataHandler.UpdateJobTextField(job_id, "lastUpdated", datetime.datetime.now().isoformat())
         dataHandler.Close()
         return True
     else:
-        dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
-        dataHandler.UpdateJobTextField(job["jobId"], "lastUpdated", datetime.datetime.now().isoformat())
+        dataHandler.UpdateJobTextField(job_id, "jobStatus", "error")
+        dataHandler.UpdateJobTextField(job_id, "lastUpdated", datetime.datetime.now().isoformat())
         dataHandler.Close()
         logging.error("Kill job failed with errors: {}".format(errors))
         return False
 
 
-def ApproveJob(job):
+def ApproveJob(job_id):
     dataHandler = DataHandler()
-    dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "queued")
+    dataHandler.UpdateJobTextField(job_id, "jobStatus", "queued")
     dataHandler.Close()
     return True
 
@@ -344,7 +344,7 @@ def TakeJobActions(jobs):
                 SubmitJob(sji["job"])
                 logging.info("TakeJobActions : submitting job : %s : %s : %s" % (sji["jobParams"]["jobName"], sji["jobParams"]["jobId"], sji["sortKey"]))
             elif sji["jobParams"]["preemptionAllowed"] and (sji["job"]["jobStatus"] == "scheduling" or sji["job"]["jobStatus"] == "running") and sji["allowed"] == False:
-                KillJob(sji["job"], "queued")
+                KillJob(sji["job"]["jobId"], "queued")
                 logging.info("TakeJobActions : pre-empting job : %s : %s : %s" % (sji["jobParams"]["jobName"], sji["jobParams"]["jobId"], sji["sortKey"]))
         except Exception as e:
             logging.error("Process job failed {}".format(sji["job"]), exc_info=True)
@@ -375,13 +375,13 @@ def Run():
                     try:
                         logging.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"]))
                         if job["jobStatus"] == "killing":
-                            KillJob(job, "killed")
+                            KillJob(job["jobId"], "killed")
                         elif job["jobStatus"] == "pausing":
-                            KillJob(job, "paused")
+                            KillJob(job["jobId"], "paused")
                         elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
                             UpdateJobStatus(job)
                         elif job["jobStatus"] == "unapproved":
-                            ApproveJob(job)
+                            ApproveJob(job["jobId"])
                     except Exception as e:
                         logging.info(e, exc_info=True)
             except Exception as e:

From 02a6d43ee00d724d0efd6d61aa95bf3a4ac7cdd9 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 20 Jul 2019 13:03:38 +0000
Subject: [PATCH 106/194] no need anymore, all job pod "restartPolicy: Never"

---
 src/ClusterManager/job_role.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
index d54354325..d9299a3b3 100644
--- a/src/ClusterManager/job_role.py
+++ b/src/ClusterManager/job_role.py
@@ -40,10 +40,6 @@ def status(self):
         pod = pods[0]
         phase = pod.status.phase
 
-        # if container was restarted, return Error
-        if pod.status.container_statuses[0].restart_count > 0:
-            return "Error"
-
         # !!! Pod is running, doesn't mean "Role" is ready and running.
         if(phase == "Running"):
             # Found that phase won't turn into "Unkonwn" even when we get 'unknown' from kubectl

From 0e8cf0a292269e61c3750bef11853cf7568b946f Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 20 Jul 2019 13:04:02 +0000
Subject: [PATCH 107/194] fix typo

---
 src/ClusterManager/job_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 37e419dfb..6f67851ec 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -217,7 +217,7 @@ def UpdateJobStatus(job):
                     k8sUtils.kubectl_delete(jobDescriptionPath)
             else:
                 logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"], retries)
-                killJob(job, "queued")
+                KillJob(job["jobId"], "queued")
                 # SubmitJob(job)
 
     if result.strip() != "Unknown" and job["jobId"] in UnusualJobs:

From e9d1389f32111541d96304d28b2116fc05c232dc Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sat, 20 Jul 2019 13:40:17 +0000
Subject: [PATCH 108/194] two potential problems

---
 src/ClusterManager/job_manager.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 6f67851ec..ff56da11e 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -206,6 +206,10 @@ def UpdateJobStatus(job):
         if job["jobId"] not in UnusualJobs:
             logging.warning("!!! Job status ---Unknown---, job: {}".format(job["jobId"]))
             UnusualJobs[job["jobId"]] = datetime.datetime.now()
+        # TODO
+        # 1) May need to reduce the timeout.
+        #     It takes minutes before pod turns into "Unkonw", we may don't need to wait here.
+        # 2) If node resume before we resubmit the job, the job will end in status 'failed'.
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
             del UnusualJobs[job["jobId"]]
             retries = dataHandler.AddandGetJobRetries(job["jobId"])

From 93b75bac4d61ce8444e1c172be253f36540c6d1a Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 05:54:10 +0000
Subject: [PATCH 109/194] profile data handler in jobmanager

---
 src/ClusterManager/cluster_manager.py  |  52 +++++-
 src/ClusterManager/command_manager.py  |   7 +
 src/ClusterManager/endpoint_manager.py |   7 +
 src/ClusterManager/job_manager.py      |   9 +-
 src/ClusterManager/joblog_manager.py   |   7 +
 src/ClusterManager/node_manager.py     |   6 +
 src/ClusterManager/requirements.txt    |   2 +
 src/ClusterManager/user_manager.py     |   6 +
 src/utils/MySQLDataHandler.py          | 191 ++++++++-------------
 src/utils/SQLDataHandler.py            | 229 +++++++------------------
 10 files changed, 220 insertions(+), 296 deletions(-)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index 845bd1662..fa63f8f09 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -5,10 +5,37 @@
 import logging.config
 import sys
 import time
+import argparse
+import threading
+
+from prometheus_client.twisted import MetricsResource
+
+from twisted.web.server import Site
+from twisted.web.resource import Resource
+from twisted.internet import reactor
 
 logger = logging.getLogger(__name__)
 
 
+class HealthResource(Resource):
+    def render_GET(self, request):
+        request.setHeader("Content-Type", "text/html; charset=utf-8")
+        return "<html>Ok</html>".encode("utf-8")
+
+def exporter_thread(port):
+    root = Resource()
+    root.putChild(b"metrics", MetricsResource())
+    root.putChild(b"healthz", HealthResource())
+    factory = Site(root)
+    reactor.listenTCP(port, factory)
+    reactor.run()
+
+def setup_exporter_thread(port):
+    t = threading.Thread(target=exporter_thread, args=(port,),
+            name="exporter")
+    t.start()
+    return t
+
 def create_log(logdir="/var/log/dlworkspace"):
     if not os.path.exists(logdir):
         os.system("mkdir -p " + logdir)
@@ -18,17 +45,17 @@ def create_log(logdir="/var/log/dlworkspace"):
     logging.config.dictConfig(logging_config)
 
 
-def Run():
+def Run(args):
     create_log()
 
     cwd = os.path.dirname(__file__)
     cmds = [
-        ["python", os.path.join(cwd, "job_manager.py")],
-        ["python", os.path.join(cwd, "user_manager.py")],
-        ["python", os.path.join(cwd, "node_manager.py")],
-        ["python", os.path.join(cwd, "joblog_manager.py")],
-        ["python", os.path.join(cwd, "command_manager.py")],
-        ["python", os.path.join(cwd, "endpoint_manager.py")],
+        ["python", os.path.join(cwd, "job_manager.py"), args.j],
+        ["python", os.path.join(cwd, "user_manager.py"), args.u],
+        ["python", os.path.join(cwd, "node_manager.py"), args.n],
+        ["python", os.path.join(cwd, "joblog_manager.py"), args.l],
+        ["python", os.path.join(cwd, "command_manager.py"), args.c],
+        ["python", os.path.join(cwd, "endpoint_manager.py"), args.e],
     ]
 
     FNULL = open(os.devnull, "w")
@@ -49,4 +76,13 @@ def Run():
 
 
 if __name__ == "__main__":
-    sys.exit(Run())
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-j", help="port of job_manager", type=int, default=9200)
+    parser.add_argument("-u", help="port of user_manager", type=int, default=9201)
+    parser.add_argument("-n", help="port of node_manager", type=int, default=9202)
+    parser.add_argument("-l", help="port of joblog_manager", type=int, default=9203)
+    parser.add_argument("-c", help="port of command_manager", type=int, default=9204)
+    parser.add_argument("-e", help="port of endpoint_manager", type=int, default=9205)
+    args = parser.parse_args()
+
+    sys.exit(Run(args))
diff --git a/src/ClusterManager/command_manager.py b/src/ClusterManager/command_manager.py
index 9ad9b5956..fd117835b 100755
--- a/src/ClusterManager/command_manager.py
+++ b/src/ClusterManager/command_manager.py
@@ -31,6 +31,8 @@
 
 import logging
 
+from cluster_manager import setup_exporter_thread
+
 logger = logging.getLogger(__name__)
 
 def RunCommand(command):
@@ -66,4 +68,9 @@ def Run():
         time.sleep(1)
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", help="port of exporter", type=int, default=9204)
+    args = parser.parse_args()
+    setup_exporter_thread(args.port)
+
     Run()
diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 727e7996a..d95931969 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -13,6 +13,8 @@
 import yaml
 import logging.config
 
+import argparse
+from cluster_manager import setup_exporter_thread
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
 import k8sUtils
@@ -250,4 +252,9 @@ def Run():
         time.sleep(1)
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", help="port of exporter", type=int, default=9205)
+    args = parser.parse_args()
+    setup_exporter_thread(args.port)
+
     Run()
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index ff56da11e..e2be00315 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -9,7 +9,6 @@
 import copy
 import traceback
 
-
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../storage"))
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),"../utils"))
 
@@ -41,6 +40,8 @@
 from job_deployer import JobDeployer
 from job_role import JobRole
 
+from cluster_manager import setup_exporter_thread
+
 
 def all_pods_not_existing(job_id):
     job_deployer = JobDeployer()
@@ -360,7 +361,6 @@ def Run():
     create_log()
 
     while True:
-
         try:
             config["racks"] = k8sUtils.get_node_labels("rack")
             config["skus"] = k8sUtils.get_node_labels("sku")
@@ -399,4 +399,9 @@ def Run():
 
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", help="port of exporter", type=int, default=9200)
+    args = parser.parse_args()
+    setup_exporter_thread(args.port)
+
     Run()
diff --git a/src/ClusterManager/joblog_manager.py b/src/ClusterManager/joblog_manager.py
index 5e55e74d1..bdc383e62 100755
--- a/src/ClusterManager/joblog_manager.py
+++ b/src/ClusterManager/joblog_manager.py
@@ -32,6 +32,8 @@
 from config import config, GetStoragePath
 from DataHandler import DataHandler
 
+from cluster_manager import setup_exporter_thread
+
 logger = logging.getLogger(__name__)
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -159,4 +161,9 @@ def Run():
         time.sleep(1)
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", help="port of exporter", type=int, default=9203)
+    args = parser.parse_args()
+    setup_exporter_thread(args.port)
+
     Run()
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index 36c3b5b5c..0849a6038 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -39,6 +39,7 @@
 from config import config
 from DataHandler import DataHandler
 
+from cluster_manager import setup_exporter_thread
 
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -252,4 +253,9 @@ def Run():
         time.sleep(30)
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", help="port of exporter", type=int, default=9202)
+    args = parser.parse_args()
+    setup_exporter_thread(args.port)
+
     Run()
diff --git a/src/ClusterManager/requirements.txt b/src/ClusterManager/requirements.txt
index 111149103..f1363b2db 100644
--- a/src/ClusterManager/requirements.txt
+++ b/src/ClusterManager/requirements.txt
@@ -1,3 +1,5 @@
 marshmallow==2.19.5
 kubernetes==9.0.0
 PyYAML>=5.1.1
+prometheus-client==0.7.1
+twisted==19.2.1
diff --git a/src/ClusterManager/user_manager.py b/src/ClusterManager/user_manager.py
index 13968f365..fc89f0b18 100755
--- a/src/ClusterManager/user_manager.py
+++ b/src/ClusterManager/user_manager.py
@@ -34,6 +34,7 @@
 from config import config
 from DataHandler import DataHandler
 
+from cluster_manager import setup_exporter_thread
 
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -91,4 +92,9 @@ def Run():
 
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", help="port of exporter", type=int, default=9201)
+    args = parser.parse_args()
+    setup_exporter_thread(args.port)
+
     Run()
diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index 380138c79..f22c6a141 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -1,9 +1,9 @@
-# from config import config
 import mysql.connector
 import json
 import base64
 import os
 import logging
+import functools
 
 import timeit
 
@@ -12,12 +12,35 @@
 from config import config
 from config import global_vars
 
+from prometheus_client import Histogram
+
 logger = logging.getLogger(__name__)
 
+data_handler_fn_histogram = Histogram("datahandler_fn_latency_seconds",
+        "latency for executing data handler function (seconds)",
+        buckets=(.05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0,
+            7.5, 10.0, 12.5, 15.0, 17.5, 20.0, float("inf")),
+        labelnames=("fn_name",))
+
+db_connect_histogram = Histogram("db_connect_latency_seconds",
+        "latency for connecting to db (seconds)",
+        buckets=(.05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, float("inf")))
+
 
-class DataHandler:
+def record(fn):
+    @functools.wraps(fn)
+    def wrapped(*args, **kwargs):
+        start = timeit.default_timer()
+        try:
+            return fn(*args, **kwargs)
+        finally:
+            elapsed += timeit.default_timer() - start
+            logger.info("DataHandler: %s, time elapsed %.2fs", fn.__name__, elapsed)
+            data_handler_fn_histogram.labels(fn.__name__).observe(elapsed)
+    return wrapped
 
 
+class DataHandler(object):
     def __init__(self):
         start_time = timeit.default_timer()
         self.database = "DLWSCluster-%s" % config["clusterId"]
@@ -32,9 +55,9 @@ def __init__(self):
         username = config["mysql"]["username"]
         password = config["mysql"]["password"]
 
-
-        self.conn = mysql.connector.connect(user=username, password=password,
-                                            host=server, database=self.database)
+        with db_connect_histogram.time()
+            self.conn = mysql.connector.connect(user=username, password=password,
+                                                host=server, database=self.database)
 
         self.CreateDatabase()
         self.CreateTable()
@@ -43,8 +66,6 @@ def __init__(self):
         elapsed = timeit.default_timer() - start_time
         logger.info("DataHandler initialization, time elapsed %f s", elapsed)
 
-
-
     def CreateDatabase(self):
         if "initSQLDB" not in global_vars or not global_vars["initSQLDB"]:
             logger.info("===========init SQL database===============")
@@ -218,41 +239,34 @@ def CreateTable(self):
             self.conn.commit()
             cursor.close()
 
-
+    @record
     def AddStorage(self, vcName, url, storageType, metadata, defaultMountPath):
         try:
-            start_time = timeit.default_timer()
             sql = "INSERT INTO `"+self.storagetablename+"` (storageType, url, metadata, vcName, defaultMountPath) VALUES (%s,%s,%s,%s,%s)"
             cursor = self.conn.cursor()
             cursor.execute(sql, (storageType, url, metadata, vcName, defaultMountPath))
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: AddStorage to DB: url : %s, vcName: %s , time elapsed %f s", url, vcName, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
-
+    @record
     def DeleteStorage(self, vcName, url):
         try:
-            start_time = timeit.default_timer()
             sql = "DELETE FROM `%s` WHERE url = '%s' and vcName = '%s'" % (self.storagetablename, url, vcName)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: DeleteStorage: url:%s, vcName:%s, time elapsed %f s", url, vcName, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
-
+    @record
     def ListStorages(self, vcName):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `storageType`,`url`,`metadata`,`vcName`,`defaultMountPath` FROM `%s` WHERE vcName = '%s' " % (self.storagetablename, vcName)
         ret = []
@@ -271,45 +285,39 @@ def ListStorages(self, vcName):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: ListStorages time elapsed %f s", elapsed)
         return ret
 
 
+    @record
     def UpdateStorage(self, vcName, url, storageType, metadata, defaultMountPath):
         try:
-            start_time = timeit.default_timer()
             sql = """update `%s` set storageType = '%s', metadata = '%s', defaultMountPath = '%s' where vcName = '%s' and url = '%s' """ % (self.storagetablename, storageType, metadata, defaultMountPath, vcName, url)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: UpdateStorage: vcName: %s, url: %s, time elapsed %f s", vcName, url, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def AddVC(self, vcName, quota, metadata):
         try:
-            start_time = timeit.default_timer()
             sql = "INSERT INTO `"+self.vctablename+"` (vcName, quota, metadata) VALUES (%s,%s,%s)"
             cursor = self.conn.cursor()
             cursor.execute(sql, (vcName, quota, metadata))
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: AddVC to DB: vcName: %s , time elapsed %f s", vcName, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def ListVCs(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `vcName`,`quota`,`metadata` FROM `%s`" % (self.vctablename)
         ret = []
@@ -326,45 +334,39 @@ def ListVCs(self):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: ListVCs time elapsed %f s", elapsed)
         return ret
 
 
+    @record
     def DeleteVC(self, vcName):
         try:
-            start_time = timeit.default_timer()
             sql = "DELETE FROM `%s` WHERE vcName = '%s'" % (self.vctablename, vcName)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: DeleteVC: vcName: %s , time elapsed %f s", vcName, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def UpdateVC(self, vcName, quota, metadata):
         try:
-            start_time = timeit.default_timer()
             sql = """update `%s` set quota = '%s', metadata = '%s' where vcName = '%s' """ % (self.vctablename, quota, metadata, vcName)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: UpdateVC: vcName: %s , time elapsed %f s", vcName, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def GetIdentityInfo(self, identityName):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `identityName`,`uid`,`gid`,`groups` FROM `%s` where `identityName` = '%s'" % (self.identitytablename, identityName)
         ret = []
@@ -379,17 +381,14 @@ def GetIdentityInfo(self, identityName):
                 ret.append(record)
         except Exception as e:
             logger.error('GetIdentityInfo Exception: %s', str(e))
-            pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: GetIdentityInfo time elapsed %f s", elapsed)
         return ret
 
 
+    @record
     def UpdateIdentityInfo(self, identityName, uid, gid, groups):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
 
             if (isinstance(groups, list)):
@@ -404,16 +403,14 @@ def UpdateIdentityInfo(self, identityName, uid, gid, groups):
 
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: UpdateIdentityInfo %s to database , time elapsed %f s", identityName, elapsed)
             return True
         except Exception as e:
             logger.error('UpdateIdentityInfo Exception: %s', str(e))
             return False
 
 
+    @record
     def GetAceCount(self, identityName, resource):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT count(ALL id) as c FROM `%s` where `identityName` = '%s' and `resource` = '%s'" % (self.acltablename,identityName, resource)
         cursor.execute(query)
@@ -422,14 +419,12 @@ def GetAceCount(self, identityName, resource):
             ret = c[0]
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: GetAceCount time elapsed %f s", elapsed)
         return ret
 
 
+    @record
     def UpdateAce(self, identityName, identityId, resource, permissions, isDeny):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
             existingAceCount = self.GetAceCount(identityName, resource)
             logger.info(existingAceCount)
@@ -443,34 +438,30 @@ def UpdateAce(self, identityName, identityId, resource, permissions, isDeny):
 
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: UpdateAce %s - %s to database , time elapsed %f s", identityName, resource, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def UpdateAclIdentityId(self, identityName, identityId):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
             sql = """update `%s` set identityId = '%s' where `identityName` = '%s' """ % (self.acltablename, identityId, identityName)
             cursor.execute(sql)
 
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: UpdateAclIdentityId %s - %s to database , time elapsed %f s", identityName, identityId, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def DeleteResourceAcl(self, resource):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
 
             sql = "DELETE FROM `%s` WHERE `resource` = '%s'" % (self.acltablename, resource)
@@ -478,17 +469,15 @@ def DeleteResourceAcl(self, resource):
 
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: DeleteResourceAcl %s, time elapsed %f s", resource, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def DeleteAce(self, identityName, resource):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
 
             sql = "DELETE FROM `%s` WHERE `identityName` = '%s' and `resource` = '%s'" % (self.acltablename, identityName, resource)
@@ -496,16 +485,14 @@ def DeleteAce(self, identityName, resource):
 
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: DeleteAce %s : %s time elapsed %f s", resource, identityName, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def GetAcl(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `identityName`,`identityId`,`resource`,`permissions`,`isDeny` FROM `%s`" % (self.acltablename)
         ret = []
@@ -524,13 +511,11 @@ def GetAcl(self):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: GetAcl time elapsed %f s", elapsed)
         return ret
 
 
+    @record
     def GetResourceAcl(self, resource):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `identityName`,`identityId`,`resource`,`permissions`,`isDeny` FROM `%s` where `resource` = '%s'" % (self.acltablename, resource)
         ret = []
@@ -548,30 +533,26 @@ def GetResourceAcl(self, resource):
             logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: GetResourceAcl time elapsed %f s", elapsed)
         return ret
 
 
+    @record
     def AddJob(self, jobParams):
         try:
-            start_time = timeit.default_timer()
             sql = "INSERT INTO `"+self.jobtablename+"` (jobId, familyToken, isParent, jobName, userName, vcName, jobType,jobParams ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)"
             cursor = self.conn.cursor()
             jobParam = base64.b64encode(json.dumps(jobParams))
             cursor.execute(sql, (jobParams["jobId"], jobParams["familyToken"], jobParams["isParent"], jobParams["jobName"], jobParams["userName"], jobParams["vcName"], jobParams["jobType"],jobParam))
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: added job %s to database, time elapsed %f s", jobParams["jobId"], elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
 
+    @record
     def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or")):
-        start_time = timeit.default_timer()
         ret = []
         cursor = self.conn.cursor()
         try:
@@ -591,13 +572,12 @@ def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or"
             if num is not None:
                 query += " limit %s " % str(num)
 
-            start_time1 = timeit.default_timer()
             cursor.execute(query)
-            elapsed1 = timeit.default_timer() - start_time1
-            start_time2 = timeit.default_timer()
+
+            fetch_start_time = timeit.default_timer()
             data = cursor.fetchall()
-            elapsed2 = timeit.default_timer() - start_time2
-            logger.info("(fetchall time: %f)", elapsed2)
+            fetch_elapsed = timeit.default_timer() - fetch_start_time
+            logger.info("(fetchall time: %f)", fetch_elapsed)
             for (jobId,jobName,userName, vcName, jobStatus,jobStatusDetail, jobType, jobDescriptionPath, jobDescription, jobTime, endpoints, jobParams,errorMsg, jobMeta) in data:
                 record = {}
                 record["jobId"] = jobId
@@ -619,12 +599,10 @@ def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or"
             logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get job list for user %s , time elapsed %f s (SQL query time: %f)", userName, elapsed, elapsed1)
         return ret
 
+    @record
     def GetJob(self, **kwargs):
-        start_time = timeit.default_timer()
         valid_keys = ["jobId", "familyToken", "isParent", "jobName", "userName", "vcName", "jobStatus", "jobType", "jobTime"]
         if len(kwargs) != 1: return []
         key, expected = kwargs.popitem()
@@ -638,27 +616,23 @@ def GetJob(self, **kwargs):
         ret = [dict(zip(columns, row)) for row in cursor.fetchall()]
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get job details with query %s=%s , time elapsed %f s", key, expected, elapsed)
         return ret
 
+    @record
     def AddCommand(self, jobId, command):
         try:
-            start_time = timeit.default_timer()
             sql = "INSERT INTO `"+self.commandtablename+"` (jobId, command) VALUES (%s,%s)"
             cursor = self.conn.cursor()
             cursor.execute(sql, (jobId, command))
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: add command to database, jobId: %s , time elapsed %f s", jobId, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
+    @record
     def GetPendingCommands(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `id`, `jobId`, `command` FROM `%s` WHERE `status` = 'pending' order by `time`" % (self.commandtablename)
         cursor.execute(query)
@@ -671,27 +645,23 @@ def GetPendingCommands(self):
             ret.append(record)
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get pending command , time elapsed %f s", elapsed)
         return ret
 
+    @record
     def FinishCommand(self, commandId):
         try:
-            start_time = timeit.default_timer()
             sql = """update `%s` set status = 'run' where `id` = '%s' """ % (self.commandtablename, commandId)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: set command %s as finished , time elapsed %f s", commandId, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
+    @record
     def GetCommands(self, jobId):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `time`, `command`, `status`, `output` FROM `%s` WHERE `jobId` = '%s' order by `time`" % (self.commandtablename, jobId)
         cursor.execute(query)
@@ -705,8 +675,6 @@ def GetCommands(self, jobId):
             ret.append(record)
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get command list for job %s , time elapsed %f s", jobId, elapsed)
         return ret
 
     def load_json(self, raw_str):
@@ -717,9 +685,9 @@ def load_json(self, raw_str):
         except:
             return {}
 
+    @record
     def GetPendingEndpoints(self):
         try:
-            start_time = timeit.default_timer()
             jobs = self.GetJob(jobStatus="running")
 
             # [ {endpoint1:{},endpoint2:{}}, {endpoint3:{}, ... }, ... ]
@@ -728,16 +696,14 @@ def GetPendingEndpoints(self):
             # endpoint["status"] == "pending"
             pendingEndpoints = {k: v for d in endpoints for k, v in d.items() if v["status"] == "pending"}
 
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: get pending endpoints %d, time elapsed %f s", len(pendingEndpoints), elapsed)
             return pendingEndpoints
         except Exception as e:
             logger.exception("Query pending endpoints failed!")
             return {}
 
+    @record
     def GetDeadEndpoints(self):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
             # TODO we need job["lastUpdated"] for filtering
             query = "SELECT `endpoints` FROM jobs WHERE `jobStatus` <> 'running' order by `jobTime` DESC"
@@ -748,16 +714,14 @@ def GetDeadEndpoints(self):
                 dead_endpoints.update(endpoint_list)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: get dead endpoints %d , time elapsed %f s", len(dead_endpoints), elapsed)
             return dead_endpoints
         except Exception as e:
             logger.exception("Query dead endpoints failed!")
             return {}
 
+    @record
     def UpdateEndpoint(self, endpoint):
         try:
-            start_time = timeit.default_timer()
             job_id = endpoint["jobId"]
             job = self.GetJob(jobId=job_id)[0]
             job_endpoints = self.load_json(job["endpoints"])
@@ -770,15 +734,13 @@ def UpdateEndpoint(self, endpoint):
             cursor.execute(sql, (json.dumps(job_endpoints), job_id))
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: update endpoints to database, endpointId: %s , time elapsed %f s", endpoint["id"], elapsed)
             return True
         except Exception as e:
             logger.exception("Update endpoints failed!")
             return False
 
+    @record
     def GetPendingJobs(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `jobId`,`jobName`,`userName`, `vcName`, `jobStatus`, `jobType`, `jobDescriptionPath`, `jobDescription`, `jobTime`, `endpoints`, `jobParams`,`errorMsg` ,`jobMeta` FROM `%s` where `jobStatus` <> 'error' and `jobStatus` <> 'failed' and `jobStatus` <> 'finished' and `jobStatus` <> 'killed' order by `jobTime` DESC" % (self.jobtablename)
         cursor.execute(query)
@@ -801,44 +763,37 @@ def GetPendingJobs(self):
             ret.append(record)
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get pending jobs %d, time elapsed %f s", len(ret), elapsed)
         return ret
 
 
-
+    @record
     def SetJobError(self, jobId, errorMsg):
         try:
-            start_time = timeit.default_timer()
             sql = """update `%s` set jobStatus = 'error', `errorMsg` = '%s' where `jobId` = '%s' """ % (self.jobtablename, errorMsg, jobId)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: set job %s error status in database, time elapsed %f s", jobId, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
+    @record
     def UpdateJobTextField(self, jobId, field, value):
         try:
-            start_time = timeit.default_timer()
             sql = "update `%s` set `%s` = '%s' where `jobId` = '%s' " % (self.jobtablename, field, value, jobId)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: update job %s, field %s to %s, time elapsed %f s", jobId, field, value, elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
+    @record
     def GetJobTextField(self, jobId, field):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `jobId`, `%s` FROM `%s` where `jobId` = '%s' " % (field, self.jobtablename, jobId)
         ret = None
@@ -851,12 +806,10 @@ def GetJobTextField(self, jobId, field):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get filed %s of job %s , time elapsed %f s", field, jobId, elapsed)
         return ret
 
+    @record
     def AddandGetJobRetries(self, jobId):
-        start_time = timeit.default_timer()
         sql = """update `%s` set `retries` = `retries` + 1 where `jobId` = '%s' """ % (self.jobtablename, jobId)
         cursor = self.conn.cursor()
         cursor.execute(sql)
@@ -872,29 +825,25 @@ def AddandGetJobRetries(self, jobId):
             ret = value
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get and update retries for job %s , time elapsed %f s", jobId, elapsed)
         return ret
 
+    @record
     def UpdateClusterStatus(self, clusterStatus):
         try:
             status = base64.b64encode(json.dumps(clusterStatus))
 
-            start_time = timeit.default_timer()
             sql = "INSERT INTO `%s` (status) VALUES ('%s')" % (self.clusterstatustablename, status)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info("DataHandler: update cluster status, time elapsed %f s", elapsed)
             return True
         except Exception as e:
             logger.error('Exception: %s', str(e))
             return False
 
+    @record
     def GetClusterStatus(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `time`, `status` FROM `%s` order by `time` DESC limit 1" % (self.clusterstatustablename)
         ret = None
@@ -908,13 +857,11 @@ def GetClusterStatus(self):
             logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get cluster status , time elapsed %f s", elapsed)
         return ret, time
 
 
+    @record
     def GetUsers(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT `identityName`,`uid` FROM `%s`" % (self.identitytablename)
         ret = []
@@ -926,10 +873,9 @@ def GetUsers(self):
             logger.error('Exception: %s', str(e))
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info("DataHandler: get users, time elapsed %f s", elapsed)
         return ret
 
+    @record
     def GetActiveJobsCount(self):
         cursor = self.conn.cursor()
         query = "SELECT count(ALL id) as c FROM `%s` where `jobStatus` <> 'error' and `jobStatus` <> 'failed' and `jobStatus` <> 'finished' and `jobStatus` <> 'killed' " % (self.jobtablename)
@@ -942,6 +888,7 @@ def GetActiveJobsCount(self):
 
         return ret
 
+    @record
     def GetALLJobsCount(self):
         cursor = self.conn.cursor()
         query = "SELECT count(ALL id) as c FROM `%s`" % (self.jobtablename)
diff --git a/src/utils/SQLDataHandler.py b/src/utils/SQLDataHandler.py
index d60c806d2..f352ebea4 100755
--- a/src/utils/SQLDataHandler.py
+++ b/src/utils/SQLDataHandler.py
@@ -12,6 +12,7 @@
 
 from config import config
 from config import global_vars
+from MySQLDataHandler import record
 
 logger = logging.getLogger(__name__)
 
@@ -20,7 +21,7 @@
 sql_live_connect_num = 25
 
 
-class SQLConnManager:
+class SQLConnManager(object):
 
     @staticmethod
     def Connect():
@@ -142,7 +143,7 @@ def ReturnConnection(conn):
                     global_vars["sql_lock"].release()
         return None
 
-class DataHandler:
+class DataHandler(object):
     def __init__(self):
         start_time = timeit.default_timer()
         self.CreateDatabase()
@@ -150,7 +151,7 @@ def __init__(self):
         logger.debug ("********************** created a new Data Handler *******************")
         self.conn = SQLConnManager.GetConnection()
         logger.debug ("Get database connection %s" % str(self.conn))
-        
+
         #print "Connecting to server ..."
         self.jobtablename = "jobs-%s" %  config["clusterId"]
         self.acltablename = "acl-%s" %  config["clusterId"]
@@ -164,8 +165,6 @@ def __init__(self):
         elapsed = timeit.default_timer() - start_time
         logger.debug ("DataHandler initialization, time elapsed %f s" % elapsed)
 
-
-
     def CreateDatabase(self):
         if "initSQLDB" not in global_vars or not global_vars["initSQLDB"]:
             logger.info("===========init SQL database===============")
@@ -327,7 +326,7 @@ def CreateTable(self):
             self.conn.commit()
             cursor.close()
 
-            
+
             sql = """
             if not exists (select * from sysobjects where name='%s' and xtype='U')
                 CREATE TABLE [dbo].[%s]
@@ -349,41 +348,34 @@ def CreateTable(self):
             self.conn.commit()
             cursor.close()
 
-
+    @record
     def AddStorage(self, vcName, url, storageType, metadata, defaultMountPath):
         try:
-            start_time = timeit.default_timer()
             sql = "INSERT INTO [%s] (storageType, url, metadata, vcName, defaultMountPath) VALUES (?,?,?,?,?)""" % self.storagetablename
             cursor = self.conn.cursor()
             cursor.execute(sql, (storageType, url, metadata, vcName, defaultMountPath))
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: AddStorage to DB: url : %s, vcName: %s , time elapsed %f s" % (url, vcName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def DeleteStorage(self, vcName, url):
         try:
-            start_time = timeit.default_timer()
             sql = "DELETE FROM [%s] WHERE [url] = '%s' and [vcName] = '%s'" % (self.storagetablename, url, vcName)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteStorage: url:%s, vcName:%s, time elapsed %f s" % (url, vcName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def ListStorages(self, vcName):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [vcName],[url],[storageType],[metadata],[defaultMountPath] FROM [%s] WHERE [vcName] = '%s' " % (self.storagetablename, vcName)
         ret = []
@@ -402,45 +394,36 @@ def ListStorages(self, vcName):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: ListStorages time elapsed %f s" % (elapsed))
         return ret 
 
-
+    @record
     def UpdateStorage(self, vcName, url, storageType, metadata, defaultMountPath):
         try:
-            start_time = timeit.default_timer()
             sql = """update [%s] set storageType = '%s', metadata = '%s', defaultMountPath = '%s' where [vcName] = '%s' and [url] = '%s' """ % (self.storagetablename, storageType, metadata, defaultMountPath, vcName, url)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateStorage: vcName: %s, url: %s, time elapsed %f s" % (vcName, url, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def AddVC(self, vcName, quota, metadata):
         try:
-            start_time = timeit.default_timer()
             sql = """INSERT INTO [%s] (vcName, quota, metadata) VALUES (?,?,?)""" % self.vctablename
             cursor = self.conn.cursor()
             cursor.execute(sql, vcName, quota, metadata)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: AddVC to DB: vcName: %s , time elapsed %f s" % (vcName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def ListVCs(self, vcName):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [vcName],[quota],[metadata] FROM [%s]" % (self.vctablename)
         ret = []
@@ -457,45 +440,36 @@ def ListVCs(self, vcName):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: ListVCs time elapsed %f s" % ( elapsed))
-        return ret   
-
+        return ret
 
+    @record
     def DeleteVC(self, vcName):
         try:
-            start_time = timeit.default_timer()
             sql = "DELETE FROM [%s] WHERE [vcName] = '%s'" % (self.vctablename, vcName)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteVC: vcName: %s , time elapsed %f s" % (vcName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def UpdateVC(self, vcName, quota, metadata):
         try:
-            start_time = timeit.default_timer()
             sql = """update [%s] set quota = '%s', metadata = '%s'  where [vcName] = '%s'""" % (self.vctablename, quota, metadata, vcName)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateVC: vcName: %s , time elapsed %f s" % (vcName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetIdentityInfo(self, identityName):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [identityName],[uid],[gid],[groups] FROM [%s] where [identityName] = '%s'" % (self.identitytablename, identityName)
         ret = []
@@ -513,35 +487,29 @@ def GetIdentityInfo(self, identityName):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetIdentityInfo time elapsed %f s" % (elapsed))
-        return ret 
-
+        return ret
 
+    @record
     def UpdateIdentityInfo(self, identityName, uid, gid, groups):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
-            
+
             if len(self.GetIdentityInfo(identityName)) == 0:
                 sql = """INSERT INTO [%s] (identityName,uid,gid,groups) VALUES (?,?,?,?)""" % self.identitytablename
                 cursor.execute(sql, identityName, uid, gid, json.dumps(groups))
             else:
                 sql = """update [%s] set uid = '%s', gid = '%s', groups = '%s' where [identityName] = '%s' """ % (self.identitytablename, uid, gid, groups, identityName)
                 cursor.execute(sql)
-            
+
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateIdentityInfo %s to database , time elapsed %f s" % (identityName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetAceCount(self, identityId, resource):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT count(ALL id) as c FROM [%s] where [identityId] = '%s' and [resource] = '%s'" % (self.acltablename,identityId, resource)
         cursor.execute(query)
@@ -550,88 +518,73 @@ def GetAceCount(self, identityId, resource):
             ret = c[0]
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetAceCount time elapsed %f s" % ( elapsed))
-        return ret 
-
+        return ret
 
+    @record
     def UpdateAce(self, identityName, identityId, resource, permissions, isDeny):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
-          
+
             if self.GetAceCount(identityId, resource) == 0:
                 sql = """INSERT INTO [%s] (identityName,identityId,resource,permissions,isDeny) VALUES (?,?,?,?,?)""" % self.acltablename
                 cursor.execute(sql, identityName, identityId, resource, permissions, isDeny)
             else:
                 sql = """update [%s] set permissions = '%s' where [identityName] = '%s' and [resource] = '%s' """ % (self.acltablename, permissions, identityName, resource)
                 cursor.execute(sql)
-            
+
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateAce %s - %s to database , time elapsed %f s" % (identityName, resource, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def UpdateAclIdentityId(self, identityName, identityId):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
             sql = """update [%s] set identityName = '%s' where [identityName] = '%s' """ % (self.acltablename, identityId, identityName)
             cursor.execute(sql)
-            
+
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: UpdateAclIdentityId %s - %s to database , time elapsed %f s" % (identityName, identityId, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-    
+    @record
     def DeleteResourceAcl(self, resource):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
-        
+
             sql = "DELETE FROM [%s] WHERE [resource] = '%s'" % (self.acltablename, resource)
             cursor = self.conn.cursor()
-            
+
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteResourceAcl %s, time elapsed %f s" % (resource, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def DeleteAce(self, identityName, resource):
         try:
-            start_time = timeit.default_timer()
             cursor = self.conn.cursor()
-        
+
             sql = "DELETE FROM [%s] WHERE [identityName] = '%s' and [resource] = '%s'" % (self.acltablename, identityName, resource)
             cursor = self.conn.cursor()
-            
+
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: DeleteAce %s : %s, time elapsed %f s" % (resource, identityName, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetAcl(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [identityName],[identityId],[resource],[permissions],[isDeny] FROM [%s]" % (self.acltablename)
         ret = []
@@ -650,13 +603,10 @@ def GetAcl(self):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetAcl time elapsed %f s" % ( elapsed))
-        return ret   
-
+        return ret
 
+    @record
     def GetResourceAcl(self, resource):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [identityName],[identityId],[resource],[permissions],[isDeny] FROM [%s] where [resource] = '%s'" % (self.acltablename, resource)
         ret = []
@@ -675,30 +625,24 @@ def GetResourceAcl(self, resource):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: GetResourceAcl time elapsed %f s" % ( elapsed))
-        return ret          
-
+        return ret
 
+    @record
     def AddJob(self, jobParams):
         try:
-            start_time = timeit.default_timer()
             sql = """INSERT INTO [%s] (jobId, familyToken, isParent, jobName, userName, vcName, jobType,jobParams ) VALUES (?,?,?,?,?,?,?)""" % self.jobtablename
             cursor = self.conn.cursor()
             jobParam = base64.b64encode(json.dumps(jobParams))
             cursor.execute(sql, jobParams["jobId"], jobParams["familyToken"], jobParams["isParent"], jobParams["jobName"], jobParams["userName"], jobParams["vcName"], jobParams["jobType"],jobParam)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: added job %s to database, time elapsed %f s" % (jobParams["jobId"],elapsed))
             return True
         except Exception as e:
            logger.error('Exception: '+ str(e))
            return False
 
-
+    @record
     def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or")):
-        start_time = timeit.default_timer()
         ret = []
         cursor = self.conn.cursor()
         try:
@@ -715,16 +659,14 @@ def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or"
                 else:
                     status_list = [ " [jobStatus] %s '%s' " % (op[0],s) for s in status.split(',')]
                     status_statement = (" "+op[1]+" ").join(status_list)
-                    query += " and ( %s ) " % status_statement                       
+                    query += " and ( %s ) " % status_statement
 
             query += " order by [jobTime] Desc"
-            start_time1 = timeit.default_timer()
             cursor.execute(query)
-            elapsed1 = timeit.default_timer() - start_time1
-            start_time2 = timeit.default_timer()
+            fetch_start = timeit.default_timer()
             data = cursor.fetchall()
-            elapsed2 = timeit.default_timer() - start_time2
-            logger.info ("(fetchall time: %f)" % (elapsed2))
+            fetch_time = timeit.default_timer() - fetch_start
+            logger.info ("(fetchall time: %f)" % (fetch_time))
             for (jobId,jobName,userName, vcName,jobStatus,jobStatusDetail, jobType, jobDescriptionPath, jobDescription, jobTime, endpoints, jobParams,errorMsg, jobMeta) in data:
                 record = {}
                 record["jobId"] = jobId
@@ -744,16 +686,12 @@ def GetJobList(self, userName, vcName, num = None, status = None, op = ("=","or"
                 ret.append(record)
         except Exception as e:
             logger.error('Exception: '+ str(e))
-            pass                
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get job list for user %s , time elapsed %f s (SQL query time: %f)" % (userName, elapsed, elapsed1))
         return ret
 
-
+    @record
     def GetJob(self, **kwargs):
-        start_time = timeit.default_timer()
         valid_keys = ["jobId", "familyToken", "isParent", "jobName", "userName", "vcName", "jobStatus", "jobType", "jobTime"]
         if len(kwargs) != 1: return []
         key, expected = kwargs.popitem()
@@ -767,29 +705,23 @@ def GetJob(self, **kwargs):
         ret = [dict(zip(columns, row)) for row in cursor.fetchall()]
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get job details with query %s=%s , time elapsed %f s" % (key, expected, elapsed))
         return ret
 
-
+    @record
     def AddCommand(self,jobId,command):
         try:
-            start_time = timeit.default_timer()
             sql = """INSERT INTO [%s] (jobId, command) VALUES (?,?)""" % self.commandtablename
             cursor = self.conn.cursor()
             cursor.execute(sql, jobId, command)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: add command to database, jobId: %s , time elapsed %f s" % (jobId, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetPendingCommands(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [id], [jobId], [command] FROM [%s] WHERE [status] = 'pending' order by [time]" % (self.commandtablename)
         cursor.execute(query)
@@ -802,29 +734,23 @@ def GetPendingCommands(self):
             ret.append(record)
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get pending command , time elapsed %f s" % (elapsed))
-        return ret    
-
+        return ret
 
+    @record
     def FinishCommand(self,commandId):
         try:
-            start_time = timeit.default_timer()
             sql = """update [%s] set status = 'run' where [id] = '%s' """ % (self.commandtablename, commandId)
             cursor = self.conn.cursor()
             cursor.execute(sql)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: set command %s as finished , time elapsed %f s" % (commandId, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetCommands(self, jobId):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [time], [command], [status], [output] FROM [%s] WHERE [jobId] = '%s' order by [time]" % (self.commandtablename, jobId)
         cursor.execute(query)
@@ -838,13 +764,10 @@ def GetCommands(self, jobId):
             ret.append(record)
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get command list for job %s , time elapsed %f s" % (jobId, elapsed))
-        return ret    
-
+        return ret
 
+    @record
     def GetPendingJobs(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [jobId],[jobName],[userName], [vcName], [jobStatus], [jobType], [jobDescriptionPath], [jobDescription], [jobTime], [endpoints], [jobParams],[errorMsg] ,[jobMeta] FROM [%s] where [jobStatus] <> 'error' and [jobStatus] <> 'failed' and [jobStatus] <> 'finished' and [jobStatus] <> 'killed' order by [jobTime] DESC" % (self.jobtablename)
         cursor.execute(query)
@@ -867,45 +790,36 @@ def GetPendingJobs(self):
             ret.append(record)
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get pending jobs , time elapsed %f s" % (elapsed))
-        return ret        
-
+        return ret
 
+    @record
     def SetJobError(self,jobId,errorMsg):
         try:
-            start_time = timeit.default_timer()
             sql = """update [%s] set jobStatus = 'error', [errorMsg] = ? where [jobId] = '%s' """ % (self.jobtablename,jobId)
             cursor = self.conn.cursor()
             cursor.execute(sql,errorMsg)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: set job %s error status in database, time elapsed %f s" % (jobId, elapsed))
             return True
         except Exception as e:
             logger.error('Exception: '+ str(e))
-            return False        
-
+            return False
 
+    @record
     def UpdateJobTextField(self,jobId,field,value):
         try:
-            start_time = timeit.default_timer()
             sql = """update [%s] set [%s] = ? where [jobId] = '%s' """ % (self.jobtablename,field, jobId)
             cursor = self.conn.cursor()
             cursor.execute(sql,value)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: update job %s, field %s , time elapsed %f s" % (jobId, field, elapsed))
             return True
         except Exception, e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetJobTextField(self,jobId,field):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [jobId], [%s] FROM [%s] where [jobId] = '%s' " % (field, self.jobtablename,jobId)
         ret = None
@@ -918,12 +832,10 @@ def GetJobTextField(self,jobId,field):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get filed %s of job %s , time elapsed %f s" % (field, jobId, elapsed))
         return ret
 
+    @record
     def AddandGetJobRetries(self,jobId):
-        start_time = timeit.default_timer()
         sql = """update [%s] set [retries] = [retries] + 1 where [jobId] = '%s' """ % (self.jobtablename, jobId)
         cursor = self.conn.cursor()
         cursor.execute(sql)
@@ -939,30 +851,24 @@ def AddandGetJobRetries(self,jobId):
             ret = value
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get and update retries for job %s , time elapsed %f s" % (jobId, elapsed))
         return ret
 
-
+    @record
     def UpdateClusterStatus(self,clusterStatus):
         try:
-            start_time = timeit.default_timer()
             sql = """INSERT INTO [%s] (status) VALUES (?)""" % self.clusterstatustablename
             cursor = self.conn.cursor()
             status = base64.b64encode(json.dumps(clusterStatus))
             cursor.execute(sql,status)
             self.conn.commit()
             cursor.close()
-            elapsed = timeit.default_timer() - start_time
-            logger.info ("DataHandler: update cluster status, time elapsed %f s" % (elapsed))
             return True
         except Exception, e:
             logger.error('Exception: '+ str(e))
             return False
 
-
+    @record
     def GetClusterStatus(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT TOP 1 [time], [status] FROM [%s] order by [time] DESC" % (self.clusterstatustablename)
         ret = None
@@ -977,13 +883,10 @@ def GetClusterStatus(self):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get cluster status , time elapsed %f s" % (elapsed))
         return ret, time
 
-
+    @record
     def GetUsers(self):
-        start_time = timeit.default_timer()
         cursor = self.conn.cursor()
         query = "SELECT [identityName],[uid] FROM [%s]" % (self.identitytablename)
         ret = []
@@ -996,11 +899,9 @@ def GetUsers(self):
             pass
         self.conn.commit()
         cursor.close()
-        elapsed = timeit.default_timer() - start_time
-        logger.info ("DataHandler: get users, time elapsed %f s" % ( elapsed))
         return ret
 
-
+    @record
     def GetActiveJobsCount(self):
         cursor = self.conn.cursor()
         query = "SELECT count(ALL id) as c FROM [%s] where [jobStatus] <> 'error' and [jobStatus] <> 'failed' and [jobStatus] <> 'finished' and [jobStatus] <> 'killed' " % (self.jobtablename)
@@ -1011,8 +912,9 @@ def GetActiveJobsCount(self):
         self.conn.commit()
         cursor.close()
 
-        return ret        
+        return ret
 
+    @record
     def GetALLJobsCount(self):
         cursor = self.conn.cursor()
         query = "SELECT count(ALL id) as c FROM [%s]" % (self.jobtablename)
@@ -1023,7 +925,7 @@ def GetALLJobsCount(self):
         self.conn.commit()
         cursor.close()
 
-        return ret    
+        return ret
 
     def __del__(self):
         logger.debug("********************** deleted a DataHandler instance *******************")
@@ -1039,7 +941,7 @@ def Close(self):
     CREATE_TABLE = False
     CREATE_DB = True
     dataHandler = DataHandler()
-    
+
     if TEST_INSERT_JOB:
         jobParams = {}
         jobParams["id"] = "dist-tf-00001"
@@ -1047,9 +949,8 @@ def Close(self):
         jobParams["user-id"] = "hongzl"
         jobParams["job-meta-path"] = "/dlws/jobfiles/***"
         jobParams["job-meta"] = "ADSCASDcAE!EDASCASDFD"
-        
+
         dataHandler.AddJob(jobParams)
-    
 
     if CREATE_TABLE:
         dataHandler.CreateTable()

From 29357c36cc6906f064fc84b3f2d3cab434fa59cf Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 06:46:19 +0000
Subject: [PATCH 110/194] fix some bugs

---
 src/ClusterManager/cluster_manager.py | 14 +++++++-------
 src/utils/MySQLDataHandler.py         |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index fa63f8f09..2fdab5851 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -28,7 +28,7 @@ def exporter_thread(port):
     root.putChild(b"healthz", HealthResource())
     factory = Site(root)
     reactor.listenTCP(port, factory)
-    reactor.run()
+    reactor.run(installSignalHandlers=False)
 
 def setup_exporter_thread(port):
     t = threading.Thread(target=exporter_thread, args=(port,),
@@ -50,12 +50,12 @@ def Run(args):
 
     cwd = os.path.dirname(__file__)
     cmds = [
-        ["python", os.path.join(cwd, "job_manager.py"), args.j],
-        ["python", os.path.join(cwd, "user_manager.py"), args.u],
-        ["python", os.path.join(cwd, "node_manager.py"), args.n],
-        ["python", os.path.join(cwd, "joblog_manager.py"), args.l],
-        ["python", os.path.join(cwd, "command_manager.py"), args.c],
-        ["python", os.path.join(cwd, "endpoint_manager.py"), args.e],
+        ["python", os.path.join(cwd, "job_manager.py"), "--port", str(args.j)],
+        ["python", os.path.join(cwd, "user_manager.py"), "--port", str(args.u)],
+        ["python", os.path.join(cwd, "node_manager.py"), "--port", str(args.n)],
+        ["python", os.path.join(cwd, "joblog_manager.py"), "--port", str(args.l)],
+        ["python", os.path.join(cwd, "command_manager.py"), "--port", str(args.c)],
+        ["python", os.path.join(cwd, "endpoint_manager.py"), "--port", str(args.e)],
     ]
 
     FNULL = open(os.devnull, "w")
diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index f22c6a141..42b532c70 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -55,7 +55,7 @@ def __init__(self):
         username = config["mysql"]["username"]
         password = config["mysql"]["password"]
 
-        with db_connect_histogram.time()
+        with db_connect_histogram.time():
             self.conn = mysql.connector.connect(user=username, password=password,
                                                 host=server, database=self.database)
 

From 4042459fe2df14401a4ea41af5ddd8618a537c96 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 06:46:33 +0000
Subject: [PATCH 111/194] config exporter port

---
 .../services/jobmanager/jobmanager.yaml       | 38 ++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml b/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
index caa7ae9d6..48aa16ad1 100755
--- a/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
+++ b/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
@@ -13,6 +13,9 @@ spec:
       labels:
         jobmanager-node: pod
         app: jobmanager
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
     spec:
       {% if cnf["dnsPolicy"] %}    
       dnsPolicy: {{cnf["dnsPolicy"]}}
@@ -39,7 +42,40 @@ spec:
         - mountPath: {{cnf["storage-mount-path"]}}/jobfiles
           name: dlwsdatajobfiles
         - mountPath: /var/log/dlworkspace
-          name: log                                                    
+          name: log
+        ports:
+        - containerPort: 9200
+          hostPort: 9200
+          name: job_manager_exporter
+          protocol: TCP
+        - containerPort: 9201
+          hostPort: 9201
+          name: user_manager_exporter
+          protocol: TCP
+        - containerPort: 9202
+          hostPort: 9202
+          name: node_manager_exporter
+          protocol: TCP
+        - containerPort: 9203
+          hostPort: 9203
+          name: joblog_manager_exporter
+          protocol: TCP
+        - containerPort: 9204
+          hostPort: 9204
+          name: command_manager_exporter
+          protocol: TCP
+        - containerPort: 9205
+          hostPort: 9205
+          name: endpoint_manager_exporter
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          initialDelaySeconds: 3
+          periodSeconds: 30
+          successThreshold: 1
+          tcpSocket:
+            port: 9200
+          timeoutSeconds: 10
       volumes:
       - name: certs
         hostPath:

From 3c558a7d3ed5151974806e35e79e28a24012f141 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 06:58:20 +0000
Subject: [PATCH 112/194] shorter port name

---
 .../services/jobmanager/jobmanager.yaml              | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml b/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
index 48aa16ad1..54ccf8c67 100755
--- a/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
+++ b/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
@@ -46,27 +46,27 @@ spec:
         ports:
         - containerPort: 9200
           hostPort: 9200
-          name: job_manager_exporter
+          name: job-mgr
           protocol: TCP
         - containerPort: 9201
           hostPort: 9201
-          name: user_manager_exporter
+          name: user-mgr
           protocol: TCP
         - containerPort: 9202
           hostPort: 9202
-          name: node_manager_exporter
+          name: node-mgr
           protocol: TCP
         - containerPort: 9203
           hostPort: 9203
-          name: joblog_manager_exporter
+          name: joblog-mgr
           protocol: TCP
         - containerPort: 9204
           hostPort: 9204
-          name: command_manager_exporter
+          name: cmd-mgr
           protocol: TCP
         - containerPort: 9205
           hostPort: 9205
-          name: endpoint_manager_exporter
+          name: endpoint-mgr
           protocol: TCP
         readinessProbe:
           failureThreshold: 3

From fb3a9d7ab4c09b08a21e243c0b47c757077777ed Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 07:16:42 +0000
Subject: [PATCH 113/194] fix some bug

---
 src/utils/MySQLDataHandler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index 42b532c70..7ced12ca8 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -34,7 +34,7 @@ def wrapped(*args, **kwargs):
         try:
             return fn(*args, **kwargs)
         finally:
-            elapsed += timeit.default_timer() - start
+            elapsed = timeit.default_timer() - start
             logger.info("DataHandler: %s, time elapsed %.2fs", fn.__name__, elapsed)
             data_handler_fn_histogram.labels(fn.__name__).observe(elapsed)
     return wrapped

From b6540b0c86ccea280a537d850c54a25050f94c4c Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 09:04:00 +0000
Subject: [PATCH 114/194] expose in restfulapi

---
 .../services/restfulapi/restfulapi.yaml       | 22 +++++++++++++------
 src/RestAPI/dlwsrestapi.py                    |  8 +++++++
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/src/ClusterBootstrap/services/restfulapi/restfulapi.yaml b/src/ClusterBootstrap/services/restfulapi/restfulapi.yaml
index e3d089c83..d1e5a6ce4 100755
--- a/src/ClusterBootstrap/services/restfulapi/restfulapi.yaml
+++ b/src/ClusterBootstrap/services/restfulapi/restfulapi.yaml
@@ -4,7 +4,7 @@ metadata:
   name: restfulapi
   namespace: default
   labels:
-     run: dlwsrestfulapi  
+     run: dlwsrestfulapi
 spec:
   selector:
     matchLabels:
@@ -15,13 +15,17 @@ spec:
       labels:
         restfulapi-node: pod
         app: restfulapi
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "5000"
     spec:
-      {% if cnf["dnsPolicy"] %}    
+      {% if cnf["dnsPolicy"] %}
       dnsPolicy: {{cnf["dnsPolicy"]}}
       {% endif %}
       nodeSelector:
         restfulapi: active
-      hostNetwork: true        
+      hostNetwork: true
       containers:
       - name: restfulapi
         image: {{cnf["worker-dockerregistry"]}}{{cnf["dockerprefix"]}}{{cnf["restfulapi"]}}:{{cnf["dockertag"]}}
@@ -31,6 +35,10 @@ spec:
           name: apiconfig
         - mountPath: /var/log/apache2
           name: log
+        ports:
+        - containerPort: 5000
+          hostPort: 5000
+          name: main
         {% if False %}
         {% for volume in cnf["mountpoints"] %}
         {% if cnf["mountpoints"][volume]["mountpoints"] is string and cnf["mountpoints"][volume]["mountpoints"]!="" %}
@@ -42,7 +50,7 @@ spec:
           name: {{mp}}
         {% endfor %}
         {% endif %}
-        {% endfor %}          
+        {% endfor %}
         {% endif %}
       volumes:
       - name: apiconfig
@@ -60,14 +68,14 @@ spec:
       {% else %}
       {% for mp in cnf["mountpoints"][volume]["mountpoints"] %}
       - name: {{mp}}
-        hostPath: 
+        hostPath:
           path: {{cnf["storage-mount-path"]}}/{{mp}}
       {% endfor %}
       {% endif %}
-      {% endfor %}          
+      {% endfor %}
       {% endif %}
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
       - key: node-role.kubernetes.io/master
-        effect: NoSchedule        
+        effect: NoSchedule
diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index df34e6600..0a66a53a2 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -28,6 +28,10 @@
 import traceback
 import threading
 
+import prometheus_client
+
+CONTENT_TYPE_LATEST = str("text/plain; version=0.0.4; charset=utf-8")
+
 dir_path = os.path.dirname(os.path.realpath(__file__))
 with open(os.path.join(dir_path, 'logging.yaml'), 'r') as f:
     logging_config = yaml.load(f)
@@ -1205,6 +1209,10 @@ def endpoint_exist(endpoint_id):
 ##
 api.add_resource(Endpoint, '/endpoints')
 
+@app.route("/metrics/")
+def metrics():
+    return Response(prometheus_client.generate_latest(), mimetype=CONTENT_TYPE_LATEST)
+
 if __name__ == '__main__':
     app.run(debug=False,host="0.0.0.0",threaded=True)
 

From 7fa12c4547ec3cff159182c166f11a60d20bed6f Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 22 Jul 2019 09:19:06 +0000
Subject: [PATCH 115/194] fix bug

---
 src/RestAPI/dlwsrestapi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index 0a66a53a2..77477c549 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -2,7 +2,7 @@
 import json
 import os
 
-from flask import Flask
+from flask import Flask, Response
 from flask_restful import reqparse, abort, Api, Resource
 from flask import request, jsonify
 import base64
@@ -1209,7 +1209,7 @@ def endpoint_exist(endpoint_id):
 ##
 api.add_resource(Endpoint, '/endpoints')
 
-@app.route("/metrics/")
+@app.route("/metrics")
 def metrics():
     return Response(prometheus_client.generate_latest(), mimetype=CONTENT_TYPE_LATEST)
 

From d4e223929e382f4dd8e30ff81b6088bfa2d0178c Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Mon, 22 Jul 2019 17:08:19 -0700
Subject: [PATCH 116/194] Change title of every table and adjust the orders
 (#427)

---
 .../WebPortal/Views/Home/JobSubmission.cshtml |  7 ++--
 .../WebPortal/Views/Home/ViewCluster.cshtml   | 32 ++++++++++---------
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 604360d79..971d429f1 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -282,7 +282,7 @@
 
         });
 
-        $scope.$watch('current.jobtrainingtype', function (value, oldValue) {
+        $scope.$watch('current.jobtrainingtype', function (value) {
             if (value === 'PSDistJob') {
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
@@ -334,16 +334,13 @@
             var selected = $filter('filter')($scope.joblist, { Value: $scope.curtemplateValue });
             var showName = "None";
             if ($scope.curtemplateValue > 0 && selected.length) {
-                if ($scope.lastTemplateValue && $scope.lastTemplateValue == $scope.curtemplateValue) {
-
+                if ($scope.lastTemplateValue && $scope.lastTemplateValue == $scope.curtemplateValue) { 
                 }
                 else {
-                    $scope.lastTemplateValue = $scope.curtemplateValue;
                     $scope.current = $scope.$eval(selected[0].Json);
                     $scope.loadTemplate();
                     $scope.setMounts();
                     $scope.checkCurrent();
-                    //console.log($scope.current);
 
                     if (!$scope.current.hasOwnProperty("runningasroot"))
                         $scope.current.runningasroot = false;
diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml
index 2951cc326..328a1ef54 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml
@@ -166,7 +166,7 @@
 
 </script>
 
-<h1>Cluster Status:</h1>
+<h1>Team Virtual Cluster Status:</h1>
 
 
 <table id="cluster_status" class="table table-bordered table-condensed table-hover table-striped">
@@ -183,8 +183,22 @@
     <tbody></tbody>
 </table>
 
+<h1>
+    Team VC User Status: 
+</h1>
+
+<table id="user_status" class="table table-bordered table-condensed table-hover table-striped">
+    <thead>
+        <tr>
+            <th>User Name</th>
+            <th>Used GPU</th>
+        </tr>
+    </thead>
+    <tbody></tbody>
+</table>
 
-<h1>Cluster Usage:</h1>
+
+<h1>Physical Cluster Usage:</h1>
 
 
 <table id="cluster_usage" class="table">
@@ -205,22 +219,10 @@
 
 
 
-<h1>
-    User Status:
-</h1>
 
-<table id="user_status" class="table table-bordered table-condensed table-hover table-striped">
-    <thead>
-        <tr>
-            <th>User Name</th>
-            <th>Used GPU</th>
-        </tr>
-    </thead>
-    <tbody></tbody>
-</table>
 
 <h1>
-    Node Status:
+    Physical Cluster Node Status:
 </h1>
 
 

From ee72f95d3b67e2ef4c2a38b59b78914b4ae175cd Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 23 Jul 2019 05:28:17 +0000
Subject: [PATCH 117/194] fix typo

---
 src/ClusterManager/job_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index e2be00315..b21f07acc 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -209,7 +209,7 @@ def UpdateJobStatus(job):
             UnusualJobs[job["jobId"]] = datetime.datetime.now()
         # TODO
         # 1) May need to reduce the timeout.
-        #     It takes minutes before pod turns into "Unkonw", we may don't need to wait here.
+        #     It takes minutes before pod turns into "Unknown", we may don't need to wait so long.
         # 2) If node resume before we resubmit the job, the job will end in status 'failed'.
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
             del UnusualJobs[job["jobId"]]

From 7dc05561ca5f749d7b6d214f85f4ef9d8ee827c8 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 23 Jul 2019 09:20:56 +0000
Subject: [PATCH 118/194] When node back after "lost", the pod may turn into
 "NotFound".

---
 src/ClusterManager/job_deployer.py |  2 ++
 src/ClusterManager/job_manager.py  | 11 +++++++----
 src/ClusterManager/job_role.py     |  3 +++
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index df93c0340..e7cf76839 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -1,6 +1,7 @@
 import yaml
 import os
 import logging
+import logging.config
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 from kubernetes.stream import stream
@@ -98,6 +99,7 @@ def get_pods(self, field_selector="", label_selector=""):
             field_selector=field_selector,
             label_selector=label_selector,
         )
+        logging.debug("Get pods: {}".format(api_response))
         return api_response.items
 
     def get_services_by_label(self, label_selector):
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index b21f07acc..fdee7f47c 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -203,14 +203,14 @@ def UpdateJobStatus(job):
         if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
             k8sUtils.kubectl_delete(jobDescriptionPath)
 
-    elif result == "Unknown":
+    elif result == "Unknown" or result == "NotFound":
         if job["jobId"] not in UnusualJobs:
-            logging.warning("!!! Job status ---Unknown---, job: {}".format(job["jobId"]))
+            logging.warning("!!! Job status ---{}---, job: {}".format(result, job["jobId"]))
             UnusualJobs[job["jobId"]] = datetime.datetime.now()
         # TODO
         # 1) May need to reduce the timeout.
         #     It takes minutes before pod turns into "Unknown", we may don't need to wait so long.
-        # 2) If node resume before we resubmit the job, the job will end in status 'failed'.
+        # 2) If node resume before we resubmit the job, the job will end in status 'NotFound'.
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
             del UnusualJobs[job["jobId"]]
             retries = dataHandler.AddandGetJobRetries(job["jobId"])
@@ -225,7 +225,7 @@ def UpdateJobStatus(job):
                 KillJob(job["jobId"], "queued")
                 # SubmitJob(job)
 
-    if result.strip() != "Unknown" and job["jobId"] in UnusualJobs:
+    if result != "Unknown" and result != "NotFound" and job["jobId"] in UnusualJobs:
         del UnusualJobs[job["jobId"]]
 
     dataHandler.Close()
@@ -236,6 +236,9 @@ def check_job_status(job_id):
     job_deployer = JobDeployer()
     job_roles = JobRole.get_job_roles(job_id)
 
+    if len(job_roles) < 1:
+        return "NotFound"
+
     # role status in ["NotFound", "Pending", "Running", "Succeeded", "Failed", "Unknown"]
     # TODO ??? when ps/master role "Succeeded", return Succeeded
     for job_role in job_roles:
diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
index d9299a3b3..09a466547 100644
--- a/src/ClusterManager/job_role.py
+++ b/src/ClusterManager/job_role.py
@@ -1,3 +1,5 @@
+import logging
+import logging.config
 from job_deployer import JobDeployer
 
 
@@ -33,6 +35,7 @@ def status(self):
         # pod-phase: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
         deployer = JobDeployer()
         pods = deployer.get_pods(field_selector="metadata.name={}".format(self.pod_name))
+        logging.debug("Pods: {}".format(pods))
         if(len(pods) < 1):
             return "NotFound"
 

From 2d8fa8e9b67936dc51fe87012a24f31abf6cebbe Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 23 Jul 2019 11:16:06 +0000
Subject: [PATCH 119/194] add useful link

---
 src/ClusterManager/job_role.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ClusterManager/job_role.py b/src/ClusterManager/job_role.py
index 09a466547..f4e61e9f8 100644
--- a/src/ClusterManager/job_role.py
+++ b/src/ClusterManager/job_role.py
@@ -33,6 +33,7 @@ def status(self):
             CONTAINER_READY -> WORKER_READY -> JOB_READY (then the job finally in "Running" status.)
         """
         # pod-phase: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+        # node condition: https://kubernetes.io/docs/concepts/architecture/nodes/#condition
         deployer = JobDeployer()
         pods = deployer.get_pods(field_selector="metadata.name={}".format(self.pod_name))
         logging.debug("Pods: {}".format(pods))

From 42d80321a3452f072eeb592e28b5306fa37e2d2c Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 23 Jul 2019 05:05:12 +0000
Subject: [PATCH 120/194] add performance dashboard

---
 .../grafana-config/perf-dashboard.json        | 238 ++++++++++++++++++
 1 file changed, 238 insertions(+)
 create mode 100644 src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json

diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json b/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
new file mode 100644
index 000000000..7b3ebad7a
--- /dev/null
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
@@ -0,0 +1,238 @@
+{
+  "dashboard": {
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "gnetId": null,
+    "graphTooltip": 0,
+    "hideControls": false,
+    "id": null,
+    "links": [],
+    "rows": [
+      {
+        "collapse": false,
+        "height": "250px",
+        "panels": [
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 1,
+            "legend": {
+              "avg": false,
+              "current": false,
+              "max": false,
+              "min": false,
+              "show": true,
+              "total": false,
+              "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "null",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "histogram_quantile(0.9, sum(rate(datahandler_fn_latency_seconds_bucket[5m])) by (le, fn_name))",
+                "format": "time_series",
+                "intervalFactor": 2,
+                "legendFormat": "{{'{{'}} fn_name {{'}}'}}",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Datahandler 90th percentile latecy per function",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "s",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": "0",
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": false
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      },
+      {
+        "collapse": false,
+        "height": 250,
+        "panels": [
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 2,
+            "legend": {
+              "avg": false,
+              "current": false,
+              "max": false,
+              "min": false,
+              "show": true,
+              "total": false,
+              "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "null",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "histogram_quantile(0.9, sum(rate(db_connect_latency_seconds_bucket[5m])) by (le))",
+                "format": "time_series",
+                "hide": false,
+                "intervalFactor": 2,
+                "legendFormat": "Connection latency",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "90th percentile DB connection latency",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "s",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": "0",
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      }
+    ],
+    "schemaVersion": 14,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now-6h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ]
+    },
+    "timezone": "browser",
+    "title": "Performance dashboard",
+    "version": 0
+  }
+}

From 3678e77759378d9f1e205fe396f539eb29a4f648 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 23 Jul 2019 05:30:35 +0000
Subject: [PATCH 121/194] add manager histogram

---
 src/ClusterManager/cluster_manager.py  |  6 +++
 src/ClusterManager/command_manager.py  | 25 ++++++-----
 src/ClusterManager/endpoint_manager.py | 13 +++---
 src/ClusterManager/job_manager.py      | 61 ++++++++++++++------------
 src/ClusterManager/joblog_manager.py   | 11 ++---
 src/ClusterManager/node_manager.py     | 11 ++---
 src/ClusterManager/user_manager.py     | 11 ++---
 7 files changed, 77 insertions(+), 61 deletions(-)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index 2fdab5851..83d2ad81f 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -9,6 +9,7 @@
 import threading
 
 from prometheus_client.twisted import MetricsResource
+from prometheus_client import Histogram
 
 from twisted.web.server import Site
 from twisted.web.resource import Resource
@@ -16,6 +17,11 @@
 
 logger = logging.getLogger(__name__)
 
+manager_iteration_histogram = Histogram("manager_iteration_latency_seconds",
+        "latency for manager to iterate",
+        buckets=(2.5, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, float("inf")),
+        labelnames=("name",))
+
 
 class HealthResource(Resource):
     def render_GET(self, request):
diff --git a/src/ClusterManager/command_manager.py b/src/ClusterManager/command_manager.py
index fd117835b..452eedd96 100755
--- a/src/ClusterManager/command_manager.py
+++ b/src/ClusterManager/command_manager.py
@@ -31,7 +31,7 @@
 
 import logging
 
-from cluster_manager import setup_exporter_thread
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram
 
 logger = logging.getLogger(__name__)
 
@@ -54,17 +54,18 @@ def create_log(logdir = '/var/log/dlworkspace'):
 def Run():
     create_log()
     while True:
-        try:
-            dataHandler = DataHandler()
-            pendingCommands = dataHandler.GetPendingCommands()
-            for command in pendingCommands:
-                try:
-                    logger.info("Processing command: %s", command["id"])
-                    RunCommand(command)
-                except Exception as e:
-                    logger.exception("run command failed")
-        except Exception as e:
-            logger.exception("getting command failed")
+        with manager_iteration_histogram.labels("command_manager").time():
+            try:
+                dataHandler = DataHandler()
+                pendingCommands = dataHandler.GetPendingCommands()
+                for command in pendingCommands:
+                    try:
+                        logger.info("Processing command: %s", command["id"])
+                        RunCommand(command)
+                    except Exception as e:
+                        logger.exception("run command failed")
+            except Exception as e:
+                logger.exception("getting command failed")
         time.sleep(1)
 
 if __name__ == '__main__':
diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index d95931969..8e3d964a8 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -14,7 +14,7 @@
 import logging.config
 
 import argparse
-from cluster_manager import setup_exporter_thread
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
 import k8sUtils
@@ -243,12 +243,13 @@ def Run():
     create_log()
 
     while True:
-        # start endpoints
-        start_endpoints()
-        time.sleep(1)
+        with manager_iteration_histogram.labels("endpoint_manager").time():
+            # start endpoints
+            start_endpoints()
+            time.sleep(1)
 
-        # clean up endpoints for jobs which is NOT running
-        cleanup_endpoints()
+            # clean up endpoints for jobs which is NOT running
+            cleanup_endpoints()
         time.sleep(1)
 
 if __name__ == '__main__':
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 96544f5f0..e7572c1ee 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -41,7 +41,7 @@
 from job_deployer import JobDeployer
 from job_role import JobRole
 
-from cluster_manager import setup_exporter_thread
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram
 
 
 def all_pods_not_existing(job_id):
@@ -370,45 +370,50 @@ def TakeJobActions(jobs):
 
 
 def Run():
-    notifier = Notifier(config.get("job-manager"))
+    notifier = notify.Notifier(config.get("job-manager"))
     notifier.start()
     create_log()
 
 
     while True:
-        try:
-            config["racks"] = k8sUtils.get_node_labels("rack")
-            config["skus"] = k8sUtils.get_node_labels("sku")
-        except Exception as e:
-            logging.exception("get node labels failed")
+        with manager_iteration_histogram.labels("job_manager").time():
+            try:
+                config["racks"] = k8sUtils.get_node_labels("rack")
+                config["skus"] = k8sUtils.get_node_labels("sku")
+            except Exception as e:
+                logging.exception("get node labels failed")
 
-        try:
-            dataHandler = DataHandler()
             try:
                 pendingJobs = dataHandler.GetPendingJobs()
                 TakeJobActions(pendingJobs)
 
                 pendingJobs = dataHandler.GetPendingJobs()
-                logging.info("Updating status for %d jobs" % len(pendingJobs))
-                for job in pendingJobs:
-                    try:
-                        logging.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"]))
-                        if job["jobStatus"] == "killing":
-                            KillJob(job["jobId"], "killed")
-                        elif job["jobStatus"] == "pausing":
-                            KillJob(job["jobId"], "paused")
-                        elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
-                            UpdateJobStatus(job, notifier)
-                        elif job["jobStatus"] == "unapproved":
-                            ApproveJob(job["jobId"])
-                    except Exception as e:
-                        logging.info(e, exc_info=True)
+                dataHandler = DataHandler()
+                try:
+                    pendingJobs = dataHandler.GetPendingJobs()
+                    TakeJobActions(pendingJobs)
+
+                    pendingJobs = dataHandler.GetPendingJobs()
+                    logging.info("Updating status for %d jobs" % len(pendingJobs))
+                    for job in pendingJobs:
+                        try:
+                            logging.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"]))
+                            if job["jobStatus"] == "killing":
+                                KillJob(job["jobId"], "killed")
+                            elif job["jobStatus"] == "pausing":
+                                KillJob(job["jobId"], "paused")
+                            elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
+                                UpdateJobStatus(job, notifier)
+                            elif job["jobStatus"] == "unapproved":
+                                ApproveJob(job["jobId"])
+                        except Exception as e:
+                            logging.info(e, exc_info=True)
+                except Exception as e:
+                    logging.exception("process pending job failed")
+                finally:
+                    dataHandler.Close()
             except Exception as e:
-                logging.exception("process pending job failed")
-            finally:
-                dataHandler.Close()
-        except Exception as e:
-            logging.exception("close data handler failed")
+                logging.exception("close data handler failed")
 
         time.sleep(1)
 
diff --git a/src/ClusterManager/joblog_manager.py b/src/ClusterManager/joblog_manager.py
index bdc383e62..9f86641f8 100755
--- a/src/ClusterManager/joblog_manager.py
+++ b/src/ClusterManager/joblog_manager.py
@@ -32,7 +32,7 @@
 from config import config, GetStoragePath
 from DataHandler import DataHandler
 
-from cluster_manager import setup_exporter_thread
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram
 
 logger = logging.getLogger(__name__)
 
@@ -154,10 +154,11 @@ def Run():
     logging.info("start to update job logs ...")
 
     while True:
-        try:
-            update_job_logs()
-        except Exception as e:
-            logger.exception("update job logs failed")
+        with manager_iteration_histogram.labels("joblog_manager").time():
+            try:
+                update_job_logs()
+            except Exception as e:
+                logger.exception("update job logs failed")
         time.sleep(1)
 
 if __name__ == '__main__':
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index 0849a6038..2c4df9dcf 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -39,7 +39,7 @@
 from config import config
 from DataHandler import DataHandler
 
-from cluster_manager import setup_exporter_thread
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram
 
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -246,10 +246,11 @@ def Run():
     logging.info("start to update nodes usage information ...")
     config["cluster_status"] = None
     while True:
-        try:
-            get_cluster_status()
-        except Exception as e:
-            logging.exception("get cluster status failed")
+        with manager_iteration_histogram.labels("node_manager").time():
+            try:
+                get_cluster_status()
+            except Exception as e:
+                logging.exception("get cluster status failed")
         time.sleep(30)
 
 if __name__ == '__main__':
diff --git a/src/ClusterManager/user_manager.py b/src/ClusterManager/user_manager.py
index fc89f0b18..2e01da521 100755
--- a/src/ClusterManager/user_manager.py
+++ b/src/ClusterManager/user_manager.py
@@ -34,7 +34,7 @@
 from config import config
 from DataHandler import DataHandler
 
-from cluster_manager import setup_exporter_thread
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram
 
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -84,10 +84,11 @@ def Run():
     create_log()
     logging.info("start to update user directory...")
     while True:
-        try:
-            set_user_directory()
-        except Exception as e:
-            logging.exception("set user directory failed")
+        with manager_iteration_histogram.labels("user_manager").time():
+            try:
+                set_user_directory()
+            except Exception as e:
+                logging.exception("set user directory failed")
         time.sleep(1)
 
 

From 9e877bab8e7ddf713d547430e5a5e91d6b74f66a Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 23 Jul 2019 05:30:46 +0000
Subject: [PATCH 122/194] perf job deployer

---
 src/ClusterManager/job_deployer.py | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index e7cf76839..43ed23910 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -2,11 +2,31 @@
 import os
 import logging
 import logging.config
+import timeit
+
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 from kubernetes.stream import stream
 from kubernetes.stream.ws_client import ERROR_CHANNEL, STDERR_CHANNEL, STDOUT_CHANNEL
 
+from prometheus_client import Histogram
+
+job_deployer_fn_histogram = Histogram("job_deployer_fn_latency_seconds",
+        "latency for executing job deployer (seconds)",
+        buckets=(.05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0,
+            7.5, 10.0, 12.5, 15.0, 17.5, 20.0, float("inf")),
+        labelnames=("fn_name",))
+
+def record(fn):
+    @functools.wraps(fn)
+    def wrapped(*args, **kwargs):
+        start = timeit.default_timer()
+        try:
+            return fn(*args, **kwargs)
+        finally:
+            elapsed = timeit.default_timer() - start
+            job_deployer_fn_histogram.labels(fn.__name__).observe(elapsed)
+    return wrapped
 
 class JobDeployer:
 
@@ -17,6 +37,7 @@ def __init__(self):
         self.namespace = "default"
         self.pretty = "pretty_example"
 
+    @record
     def create_pod(self, body, dry_run=None):
         api_response = self.v1.create_namespaced_pod(
             namespace=self.namespace,
@@ -26,6 +47,7 @@ def create_pod(self, body, dry_run=None):
         )
         return api_response
 
+    @record
     def delete_pod(self, name, dry_run=None):
         api_response = self.v1.delete_namespaced_pod(
             name=name,
@@ -36,6 +58,7 @@ def delete_pod(self, name, dry_run=None):
         )
         return api_response
 
+    @record
     def create_service(self, body, dry_run=None):
         api_response = self.v1.create_namespaced_service(
             namespace=self.namespace,
@@ -45,6 +68,7 @@ def create_service(self, body, dry_run=None):
         )
         return api_response
 
+    @record
     def delete_service(self, name, dry_run=None):
         api_response = self.v1.delete_namespaced_service(
             name=name,
@@ -55,6 +79,7 @@ def delete_service(self, name, dry_run=None):
         )
         return api_response
 
+    @record
     def cleanup_pods(self, pod_names):
         errors = []
         for pod_name in pod_names:
@@ -68,6 +93,7 @@ def cleanup_pods(self, pod_names):
                 errors.append({"message": message, "exception": e})
         return errors
 
+    @record
     def cleanup_services(self, services):
         errors = []
         for service in services:
@@ -81,6 +107,7 @@ def cleanup_services(self, services):
                 errors.append({"message": message, "exception": e})
         return errors
 
+    @record
     def create_pods(self, pods):
         # TODO instead of delete, we could check update existiong ones. During refactoring, keeping the old way.
         pod_names = [pod["metadata"]["name"] for pod in pods]
@@ -92,6 +119,7 @@ def create_pods(self, pods):
             logging.info("Create pod succeed: %s" % created_pod.metadata.name)
         return created
 
+    @record
     def get_pods(self, field_selector="", label_selector=""):
         api_response = self.v1.list_namespaced_pod(
             namespace=self.namespace,
@@ -102,6 +130,7 @@ def get_pods(self, field_selector="", label_selector=""):
         logging.debug("Get pods: {}".format(api_response))
         return api_response.items
 
+    @record
     def get_services_by_label(self, label_selector):
         api_response = self.v1.list_namespaced_service(
             namespace=self.namespace,
@@ -110,6 +139,7 @@ def get_services_by_label(self, label_selector):
         )
         return api_response.items
 
+    @record
     def delete_job(self, job_id):
         label_selector = "run={}".format(job_id)
 
@@ -125,6 +155,7 @@ def delete_job(self, job_id):
         errors = pod_errors + service_errors
         return errors
 
+    @record
     def pod_exec(self, pod_name, exec_command, timeout=60):
         """work as the command (with timeout): kubectl exec 'pod_name' 'exec_command'"""
         try:

From f79ad2fc08969c47c8c55e3a8962b79b13a990fd Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 24 Jul 2019 03:28:43 +0000
Subject: [PATCH 123/194] add missing import

---
 src/ClusterManager/job_deployer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 43ed23910..4ec8186c4 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -3,6 +3,7 @@
 import logging
 import logging.config
 import timeit
+import functools
 
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException

From ad4062981c94c1314c0cc3de367f639b193d4d1a Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 24 Jul 2019 03:46:55 +0000
Subject: [PATCH 124/194] fix conflict error

---
 src/ClusterManager/job_manager.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index e7572c1ee..c43161393 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -374,7 +374,6 @@ def Run():
     notifier.start()
     create_log()
 
-
     while True:
         with manager_iteration_histogram.labels("job_manager").time():
             try:
@@ -384,10 +383,6 @@ def Run():
                 logging.exception("get node labels failed")
 
             try:
-                pendingJobs = dataHandler.GetPendingJobs()
-                TakeJobActions(pendingJobs)
-
-                pendingJobs = dataHandler.GetPendingJobs()
                 dataHandler = DataHandler()
                 try:
                     pendingJobs = dataHandler.GetPendingJobs()

From 99f57b43a050c78c9ddcbcad6393b8a37c102372 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 24 Jul 2019 06:21:15 +0000
Subject: [PATCH 125/194] add reaper

---
 src/ClusterBootstrap/params.py                |  5 ++
 .../services/monitor/alert-manager.yaml       | 32 +++++++-
 .../services/monitor/alerting/jobs.rules      |  5 ++
 src/RestAPI/dlwsrestapi.py                    |  2 +
 src/docker-images/reaper/Dockerfile           |  7 ++
 src/docker-images/reaper/main.py              | 81 +++++++++++++++++++
 6 files changed, 131 insertions(+), 1 deletion(-)
 create mode 100644 src/docker-images/reaper/Dockerfile
 create mode 100644 src/docker-images/reaper/main.py

diff --git a/src/ClusterBootstrap/params.py b/src/ClusterBootstrap/params.py
index fa0bfd4f8..09986420b 100755
--- a/src/ClusterBootstrap/params.py
+++ b/src/ClusterBootstrap/params.py
@@ -31,6 +31,11 @@
         # If want to deploy with alert-manager, should config
         # configured with True, and fill appropriate value to:
         # smtp_url, smtp_from, smtp_auth_username, smtp_auth_password and receiver
+        "reaper": {
+            "dry-run": True,
+            "port": "9500",
+            "restful-url": "http://localhost:5000",
+        }
     },
 
     "mysql_port": "3306",
diff --git a/src/ClusterBootstrap/services/monitor/alert-manager.yaml b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
index 819756e14..016ad3dcb 100644
--- a/src/ClusterBootstrap/services/monitor/alert-manager.yaml
+++ b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
@@ -24,7 +24,7 @@ spec:
       hostNetwork: true
       containers:
       - name: alert-manager
-        image: prom/alertmanager:v0.15.1
+        image: prom/alertmanager:v0.18.0
         args:
           - '--config.file=/etc/alertmanager/config.yml'
           - '--storage.path=/alertmanager'
@@ -40,6 +40,23 @@ spec:
           mountPath: /alertmanager
         - name: templates-volume
           mountPath: /etc/alertmanager/template
+      {% if cnf["alert-manager"]["reaper"] %}
+      - name: reaper
+        image: {{cnf["worker-dockerregistry"]}}{{cnf["dockerprefix"]}}reaper:{{cnf["dockertag"]}}
+        command:
+          - 'python'
+          - '/reaper/main.py'
+          - '--port'
+          - '{{ cnf["alert-manager"]["reaper"]["port"] }}'
+          - '--restful_url'
+          - '{{ cnf["alert-manager"]["reaper"]["restful-url"] }}'
+          {% if cnf["alert-manager"]["reaper"]["dry-run"] %}
+          - '--dry_run'
+          {% endif %}
+        ports:
+        - name: alert-manager
+          containerPort: {{ cnf["alert-manager"]["reaper"]["port"] }}
+      {% endif %}
       volumes:
       - name: config-volume
         configMap:
@@ -93,6 +110,11 @@ data:
         match_re:
           type: user_alert
           alertname: "job-state-changed"
+      - receiver: reaper
+        group_by: [alertname, user_email, job_name]
+        group_wait: 0s
+        match_re:
+          type: reaper
     receivers:
     - name: "alert-email"
       email_configs:
@@ -129,3 +151,11 @@ data:
             {% endif %}
             subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
 {% endif %}
+    - name: "reaper"
+      {% if cnf["alert-manager"]["reaper"] %}
+      webhook_configs:
+        - send_resolved: False
+          url: 'http://localhost:{{ cnf["alert-manager"]["reaper"]["port"] }}/kill'
+          http_config:
+            bearer_token: 'shinigami'
+      {% endif %}
diff --git a/src/ClusterBootstrap/services/monitor/alerting/jobs.rules b/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
index 976263107..b7fbb8e25 100644
--- a/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
+++ b/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
@@ -6,3 +6,8 @@ groups:
         for: 4h
         labels:
           type: user_alert
+      - alert: kill-idle-jobs
+        expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
+        for: 8h
+        labels:
+          type: reaper
diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index 77477c549..5fb607c82 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -393,6 +393,8 @@ def get(self):
         result = JobRestAPIUtils.KillJob(userName, jobId)
         ret = {}
         if result:
+            # NOTE "Success" prefix is used in reaper, please also update reaper code
+            # if need to change it.
             ret["result"] = "Success, the job is scheduled to be terminated."
         else:
             ret["result"] = "Cannot Kill the job. Job ID:" + jobId
diff --git a/src/docker-images/reaper/Dockerfile b/src/docker-images/reaper/Dockerfile
new file mode 100644
index 000000000..7ff9dd641
--- /dev/null
+++ b/src/docker-images/reaper/Dockerfile
@@ -0,0 +1,7 @@
+FROM python:3.7
+
+RUN pip3 install requests flask
+
+WORKDIR /reaper
+
+COPY * /reaper/
diff --git a/src/docker-images/reaper/main.py b/src/docker-images/reaper/main.py
new file mode 100644
index 000000000..6b6460414
--- /dev/null
+++ b/src/docker-images/reaper/main.py
@@ -0,0 +1,81 @@
+#!/usr/bin/python
+
+import urllib.parse
+import argparse
+import requests
+import logging
+import faulthandler
+import signal
+import json
+
+import flask
+from flask import Flask
+from flask import request
+
+logger = logging.getLogger(__name__)
+
+app = Flask(__name__)
+
+@app.route("/kill", methods=["POST"])
+def kill():
+    args = request.args
+    auth = request.headers.get("Authorization")
+    if auth != "Bearer shinigami":
+        logger.warning("get unauthorized call")
+        return "Unauthorized", 401
+    try:
+        body = json.loads(request.data.decode("utf-8"))
+
+        for alert in body["alerts"]:
+            if alert.get("status") == "resolved":
+                continue
+            logger.info("processing alert of %s", alert)
+            if not dry_run:
+                job_name = alert["labels"]["job_name"]
+                username = alert["labels"]["user_email"]
+                params = {"jobId": job_name, "userName": username}
+                args = urllib.parse.urlencode(params)
+                url = restful_url + "/KillJob?" + args
+
+                response = requests.get(url, timeout=10)
+                response.raise_for_status()
+                result = response.json().get("result")
+                if result is not None and result.startswith("Success"):
+                    logger.info("killing %s success", params)
+                else:
+                    logger.warning("killing %s failed", params)
+            else:
+                logger.info("reaper in dry_run mode, will not kill %s", alert)
+        return "Ok", 200
+    except Exception as e:
+        logger.exception("caught exception while processing kill, data is %s",
+                requests.data)
+        raise e
+
+def register_stack_trace_dump():
+    faulthandler.register(signal.SIGTRAP, all_threads=True, chain=False)
+
+def main(args):
+    app.run(host="0.0.0.0", port=args.port, debug=False, use_reloader=False)
+
+if __name__ == "__main__":
+    register_stack_trace_dump()
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s@%(thread)d - %(message)s",
+            level=logging.INFO)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", "-p", default=9500, type=int,
+            help="port to listen, default 9500")
+    parser.add_argument("--restful_url", "-r", required=True,
+            help="restful api url, e.g. http://localhost:5000")
+    parser.add_argument("--dry_run", "-d", action="store_true",
+            help="if dry_run, the reaper will do nothing")
+    args = parser.parse_args()
+
+    global dry_run
+    global restful_url
+
+    dry_run = args.dry_run
+    restful_url = args.restful_url
+
+    main(args)

From d0b7cd8417936c52c6d2a8fdf6056ac6094a4f40 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 24 Jul 2019 10:08:51 +0000
Subject: [PATCH 126/194] split jobmanager's metrics with restfulapi's

---
 .../grafana-config/perf-dashboard.json        | 171 +++++++++++++++++-
 1 file changed, 161 insertions(+), 10 deletions(-)

diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json b/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
index 7b3ebad7a..77a3f5c09 100644
--- a/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
@@ -19,6 +19,7 @@
     "hideControls": false,
     "id": null,
     "links": [],
+    "refresh": "30s",
     "rows": [
       {
         "collapse": false,
@@ -30,7 +31,7 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": null,
-            "fill": 1,
+            "fill": 0,
             "id": 1,
             "legend": {
               "avg": false,
@@ -51,12 +52,12 @@
             "renderer": "flot",
             "seriesOverrides": [],
             "spaceLength": 10,
-            "span": 12,
+            "span": 6,
             "stack": false,
             "steppedLine": false,
             "targets": [
               {
-                "expr": "histogram_quantile(0.9, sum(rate(datahandler_fn_latency_seconds_bucket[5m])) by (le, fn_name))",
+                "expr": "histogram_quantile(0.9, sum(rate(datahandler_fn_latency_seconds_bucket{scraped_from=~\"jobmanager.*\"}[5m])) by (le, fn_name))",
                 "format": "time_series",
                 "intervalFactor": 2,
                 "legendFormat": "{{'{{'}} fn_name {{'}}'}}",
@@ -66,7 +67,7 @@
             "thresholds": [],
             "timeFrom": null,
             "timeShift": null,
-            "title": "Datahandler 90th percentile latecy per function",
+            "title": "Datahandler 90th percentile latency per function from jobmanager",
             "tooltip": {
               "shared": true,
               "sort": 0,
@@ -98,6 +99,81 @@
                 "show": false
               }
             ]
+          },
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 2,
+            "legend": {
+              "avg": false,
+              "current": false,
+              "max": false,
+              "min": false,
+              "show": true,
+              "total": false,
+              "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "null",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "histogram_quantile(0.9, sum(rate(datahandler_fn_latency_seconds_bucket{scraped_from=~\"restfulapi.*\"}[5m])) by (le, fn_name))",
+                "format": "time_series",
+                "intervalFactor": 2,
+                "legendFormat": "{{'{{'}} fn_name {{'}}'}}",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Datahandler 90th percentile latency per function from restfulapi",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": "0",
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              }
+            ]
           }
         ],
         "repeat": null,
@@ -117,8 +193,8 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": null,
-            "fill": 1,
-            "id": 2,
+            "fill": 0,
+            "id": 3,
             "legend": {
               "avg": false,
               "current": false,
@@ -138,23 +214,23 @@
             "renderer": "flot",
             "seriesOverrides": [],
             "spaceLength": 10,
-            "span": 12,
+            "span": 6,
             "stack": false,
             "steppedLine": false,
             "targets": [
               {
-                "expr": "histogram_quantile(0.9, sum(rate(db_connect_latency_seconds_bucket[5m])) by (le))",
+                "expr": "histogram_quantile(0.9, sum(rate(db_connect_latency_seconds_bucket{scraped_from=~\"jobmanager.*\"}[5m])) by (le))",
                 "format": "time_series",
                 "hide": false,
                 "intervalFactor": 2,
-                "legendFormat": "Connection latency",
+                "legendFormat": "Connection Latency",
                 "refId": "A"
               }
             ],
             "thresholds": [],
             "timeFrom": null,
             "timeShift": null,
-            "title": "90th percentile DB connection latency",
+            "title": "90th percentile DB connection latency from jobmanager",
             "tooltip": {
               "shared": true,
               "sort": 0,
@@ -183,7 +259,82 @@
                 "logBase": 1,
                 "max": null,
                 "min": null,
+                "show": false
+              }
+            ]
+          },
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 0,
+            "id": 4,
+            "legend": {
+              "avg": false,
+              "current": false,
+              "max": false,
+              "min": false,
+              "show": true,
+              "total": false,
+              "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "null",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "histogram_quantile(0.9, sum(rate(db_connect_latency_seconds_bucket{scraped_from=~\"restfulapi.*\"}[5m])) by (le))",
+                "format": "time_series",
+                "intervalFactor": 2,
+                "legendFormat": "Connection Latency",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "90th percentile DB connection latency from restfulapi",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "s",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": "0",
                 "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": false
               }
             ]
           }

From 4c45f52f7b998c8f015dff388d1c61f89d23f495 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 24 Jul 2019 11:52:30 +0000
Subject: [PATCH 127/194] share k8s client: to avoid connection leak

---
 src/ClusterManager/job_deployer.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 4ec8186c4..c60663d51 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -29,12 +29,16 @@ def wrapped(*args, **kwargs):
             job_deployer_fn_histogram.labels(fn.__name__).observe(elapsed)
     return wrapped
 
+
+# The config will be loaded from default location.
+config.load_kube_config()
+k8s_client = client.CoreV1Api()
+
+
 class JobDeployer:
 
     def __init__(self):
-        # The config will be loaded from default location.
-        config.load_kube_config()
-        self.v1 = client.CoreV1Api()
+        self.v1 = k8s_client
         self.namespace = "default"
         self.pretty = "pretty_example"
 

From fed8c67759b0e878d47c5818f5b06d8cc89682a2 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 25 Jul 2019 12:23:29 +0000
Subject: [PATCH 128/194] add option for "force" delete pod

---
 src/ClusterManager/job_deployer.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index c60663d51..4e7934bcc 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -53,12 +53,16 @@ def create_pod(self, body, dry_run=None):
         return api_response
 
     @record
-    def delete_pod(self, name, dry_run=None):
+    def delete_pod(self, name, grace_period_seconds=None, dry_run=None):
+        body = client.V1DeleteOptions()
+        body.grace_period_seconds = grace_period_seconds
+        body.dry_run = dry_run
         api_response = self.v1.delete_namespaced_pod(
             name=name,
             namespace=self.namespace,
             pretty=self.pretty,
-            body=client.V1DeleteOptions(),
+            body=body,
+            grace_period_seconds=grace_period_seconds,
             dry_run=dry_run,
         )
         return api_response
@@ -85,11 +89,12 @@ def delete_service(self, name, dry_run=None):
         return api_response
 
     @record
-    def cleanup_pods(self, pod_names):
+    def cleanup_pods(self, pod_names, force=False):
         errors = []
+        grace_period_seconds = 0 if force else None
         for pod_name in pod_names:
             try:
-                self.delete_pod(pod_name)
+                self.delete_pod(pod_name, grace_period_seconds)
             except Exception as e:
                 if isinstance(e, ApiException) and 404 == e.status:
                     return []

From ebfd77130b319fc04d317515712dca4d34e66401 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 25 Jul 2019 12:30:45 +0000
Subject: [PATCH 129/194] forcing cleanup job before submit

---
 src/ClusterManager/job_deployer.py | 4 ++--
 src/ClusterManager/job_manager.py  | 9 +++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/ClusterManager/job_deployer.py b/src/ClusterManager/job_deployer.py
index 4e7934bcc..ac80c22a1 100644
--- a/src/ClusterManager/job_deployer.py
+++ b/src/ClusterManager/job_deployer.py
@@ -150,13 +150,13 @@ def get_services_by_label(self, label_selector):
         return api_response.items
 
     @record
-    def delete_job(self, job_id):
+    def delete_job(self, job_id, force=False):
         label_selector = "run={}".format(job_id)
 
         # query pods then delete
         pods = self.get_pods(label_selector=label_selector)
         pod_names = [pod.metadata.name for pod in pods]
-        pod_errors = self.cleanup_pods(pod_names)
+        pod_errors = self.cleanup_pods(pod_names, force)
 
         # query services then delete
         services = self.get_services_by_label(label_selector)
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index c43161393..fdeb9ae35 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -55,8 +55,13 @@ def all_pods_not_existing(job_id):
 def SubmitJob(job):
     # check if existing any pod with label: run=job_id
     assert("jobId" in job)
-    if not all_pods_not_existing(job["jobId"]):
-        logging.warning("Waiting until previously pods are cleaned up! Job {}".format(job["jobId"]))
+    job_id = job["jobId"]
+    if not all_pods_not_existing(job_id):
+        logging.warning("Waiting until previously pods are cleaned up! Job {}".format(job_id))
+        job_deployer = JobDeployer()
+        errors = job_deployer.delete_job(job_id, force=True)
+        if errors:
+            logging.warning("Force delete job {}: {}".format(job_id, errors))
         return
 
     ret = {}

From 0d5f87e564a974e0cceab5b1bf99e9dcde6aa8ce Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Thu, 25 Jul 2019 11:37:57 -0700
Subject: [PATCH 130/194] Enable distribute job in low priority job template
 (#432)

---
 .../WebPortal/Views/Home/JobSubmission.cshtml   | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 971d429f1..3051f12fe 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -263,7 +263,7 @@
                         gpu_available: gpu_available[key] < 0 ? 0 : gpu_available[key],
                         quota: quota[key]
                     };
-                    $scope.isLowPriority = value.low_priority === true ? true : false;
+                   /* $scope.isLowPriority = value.low_priority === true ? true : false;
                     if ($scope.isLowPriority) {
                         var filteredJobList = [];
                         $scope.joblist.forEach(function(job) {
@@ -272,7 +272,7 @@
                             }
                         });
                         $scope.joblist = filteredJobList;
-                    }
+                    }*/
 
                     $scope.gpus[key] = gpu;
                 });
@@ -700,17 +700,10 @@
                             </md-select>
                         </md-input-container>
                         <md-input-container flex="50">
-                            <label style="pointer-events:auto">Job Type  
-                                <i ng-hide="!isLowPriority"
-                                   class="layui-icon layui-icon-tips" 
-                                   data-toggle="tooltip" 
-                                   title="Low priority cluster does not support distributed jobs" 
-                                   data-placement="right"
-                                 ></i>
-                            </label>
-                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
+                            <label>Job Type</label>
+                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 ? false : true"  required md-no-asterisk>
                                 <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob" ng-disabled="curtemplateValue === 0 && isLowPriority">Distributed Job</md-option>
+                                <md-option value="PSDistJob">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 5b1df00256a781ed3e9f70a62b1b736b6c3e8111 Mon Sep 17 00:00:00 2001
From: hongyiliu <hongyiliu1992@gmail.com>
Date: Thu, 25 Jul 2019 15:45:49 -0700
Subject: [PATCH 131/194] Clean up advance option & save job template issue

---
 .../dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 3051f12fe..7fa815357 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -286,8 +286,10 @@
             if (value === 'PSDistJob') {
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
+                $scope.current.hostNetwork = true;
             } else {
                 delete $scope.current.numps;
+                $scope.current.hostNetwork = false;
             }
         })
 
@@ -914,7 +916,7 @@
 
 
 
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">HyperParameter Turning</h2>
                                     <div class="layui-colla-content">
 
@@ -972,17 +974,17 @@
                                     </div>
                                 </div>
 
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">Host Network</h2>
                                     <div class="layui-colla-content">
                                         <md-input-container class="md-block" flex="50">
                                             <label> Use Host network in container? </label>
-                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="current.jobtrainingtype.indexOf('Regular') !== -1 ? true : false" md-no-asterisk>
+                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="true"  md-no-asterisk>
                                             </md-switch>
                                         </md-input-container>
                                     </div>
                                 </div>
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">GPU Topology</h2>
                                     <div class="layui-colla-content">
 
@@ -1091,7 +1093,7 @@
 
                     <div class="modal-footer">
                         <a ng-click="DeleteTemplate()" class="btn btn-success" href="#">Delete Template</a>
-                        <a ng-click="SaveTemplate()" class="btn btn-success" href="#">Save Template</a>
+                        <a ng-click="SaveTemplate()" class="btn btn-success" href="#" ng-disabled="template.useTemplateName === 'None' || !template.useTemplateName ? true : false">Save Template</a>
                     </div>
 
                 </div>

From fbd2454e0b89c3b05f54099aba164b065a720fbc Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Fri, 26 Jul 2019 03:38:55 +0000
Subject: [PATCH 132/194] default using localhost as prometheus ip in grafana

---
 src/ClusterBootstrap/params.py                                  | 2 +-
 .../services/monitor/grafana-config/prom-datasource.json        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ClusterBootstrap/params.py b/src/ClusterBootstrap/params.py
index fa0bfd4f8..76060190e 100755
--- a/src/ClusterBootstrap/params.py
+++ b/src/ClusterBootstrap/params.py
@@ -23,7 +23,7 @@
     "job-exporter": { "port": 9102 },
     "node-exporter": { "port": 9100 },
     "watchdog": { "port": 9101 },
-    "grafana": { "port": 3000 },
+    "grafana": { "port": 3000, "prometheus-ip": "localhost" },
     "alert-manager": {
         "port": 9093,
         "configured": False,
diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json b/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json
index 467dfa70e..027e5ea6c 100644
--- a/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json
@@ -1,6 +1,6 @@
 {
   "name": "PM",
-  "url": "http://{{cnf['prometheus']['host']}}:9091/prometheus",
+  "url": "http://{{cnf['grafana']['prometheus-ip']}}:9091/prometheus",
   "basicAuth": false,
   "access": "proxy",
   "type": "prometheus",

From 6e0b34bffbb11c6104e2d668a0e325e9fb158b02 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 11:16:18 +0000
Subject: [PATCH 133/194] force delete pod when killing

---
 src/ClusterManager/job_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index fdeb9ae35..3434b2a6c 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -146,7 +146,7 @@ def KillJob(job_id, desiredState="killed"):
     logging.info("Killing job %s, with status %s, %s" % (job_id, result, detail))
 
     job_deployer = JobDeployer()
-    errors = job_deployer.delete_job(job_id)
+    errors = job_deployer.delete_job(job_id, force=True)
 
     if len(errors) == 0:
         dataHandler.UpdateJobTextField(job_id, "jobStatus", desiredState)

From 8fd7693015a0e4688b152a9e2ba03042ee9ae725 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 11:48:50 +0000
Subject: [PATCH 134/194] no need to retry on submit

---
 src/ClusterManager/job_manager.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 3434b2a6c..29a6de084 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -227,17 +227,8 @@ def UpdateJobStatus(job, notifier=None):
         # 2) If node resume before we resubmit the job, the job will end in status 'NotFound'.
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
             del UnusualJobs[job["jobId"]]
-            retries = dataHandler.AddandGetJobRetries(job["jobId"])
-            if retries >= 5:
-                logging.warning("Job %s fails for more than 5 times, abort", job["jobId"])
-                dataHandler.UpdateJobTextField(job["jobId"], "jobStatus", "error")
-                dataHandler.UpdateJobTextField(job["jobId"], "errorMsg", "cannot launch the job.")
-                if jobDescriptionPath is not None and os.path.isfile(jobDescriptionPath):
-                    k8sUtils.kubectl_delete(jobDescriptionPath)
-            else:
-                logging.warning("Job %s fails in Kubernetes, delete and re-submit the job. Retries %d", job["jobId"], retries)
-                KillJob(job["jobId"], "queued")
-                # SubmitJob(job)
+            logging.warning("Job {} fails in Kubernetes as {}, delete and re-submit.".format(job["jobId"], result))
+            KillJob(job["jobId"], "queued")
 
     if result != "Unknown" and result != "NotFound" and job["jobId"] in UnusualJobs:
         del UnusualJobs[job["jobId"]]

From 3a3a14c23e1ed623005fc4c3ba1fded0371bc2b4 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 12:29:20 +0000
Subject: [PATCH 135/194] refine execption handle

---
 src/ClusterManager/job_manager.py | 46 +++++++++++++++----------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 29a6de084..8030644d5 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -380,31 +380,31 @@ def Run():
 
             try:
                 dataHandler = DataHandler()
+                pendingJobs = dataHandler.GetPendingJobs()
+                TakeJobActions(pendingJobs)
+
+                pendingJobs = dataHandler.GetPendingJobs()
+                logging.info("Updating status for %d jobs" % len(pendingJobs))
+                for job in pendingJobs:
+                    try:
+                        logging.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"]))
+                        if job["jobStatus"] == "killing":
+                            KillJob(job["jobId"], "killed")
+                        elif job["jobStatus"] == "pausing":
+                            KillJob(job["jobId"], "paused")
+                        elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
+                            UpdateJobStatus(job, notifier)
+                        elif job["jobStatus"] == "unapproved":
+                            ApproveJob(job["jobId"])
+                    except Exception as e:
+                        logging.warning(e, exc_info=True)
+            except Exception as e:
+                logging.warning("Process job failed!", exc_info=True)
+            finally:
                 try:
-                    pendingJobs = dataHandler.GetPendingJobs()
-                    TakeJobActions(pendingJobs)
-
-                    pendingJobs = dataHandler.GetPendingJobs()
-                    logging.info("Updating status for %d jobs" % len(pendingJobs))
-                    for job in pendingJobs:
-                        try:
-                            logging.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"]))
-                            if job["jobStatus"] == "killing":
-                                KillJob(job["jobId"], "killed")
-                            elif job["jobStatus"] == "pausing":
-                                KillJob(job["jobId"], "paused")
-                            elif job["jobStatus"] == "scheduling" or job["jobStatus"] == "running":
-                                UpdateJobStatus(job, notifier)
-                            elif job["jobStatus"] == "unapproved":
-                                ApproveJob(job["jobId"])
-                        except Exception as e:
-                            logging.info(e, exc_info=True)
-                except Exception as e:
-                    logging.exception("process pending job failed")
-                finally:
                     dataHandler.Close()
-            except Exception as e:
-                logging.exception("close data handler failed")
+                except:
+                    pass
 
         time.sleep(1)
 

From 9263d66d2cb0221e2cec8ce22013994c21698c58 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 12:52:12 +0000
Subject: [PATCH 136/194] reset endpoint when resubmit job

---
 src/ClusterManager/job_manager.py |  9 +++++++++
 src/utils/MySQLDataHandler.py     | 20 ++++++++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 8030644d5..4ba69cf5e 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -227,6 +227,15 @@ def UpdateJobStatus(job, notifier=None):
         # 2) If node resume before we resubmit the job, the job will end in status 'NotFound'.
         elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
             del UnusualJobs[job["jobId"]]
+
+            # TODO refine later
+            # before resubmit the job, reset the endpoints
+            # update all endpoint to status 'pending', so it would restart when job is ready
+            endpoints = dataHandler.GetJobEndpoints(job["jobId"])
+            for endpoint in endpoints:
+                endpoint["status"] = "pending"
+                dataHandler.UpdateEndpoint(endpoint)
+
             logging.warning("Job {} fails in Kubernetes as {}, delete and re-submit.".format(job["jobId"], result))
             KillJob(job["jobId"], "queued")
 
diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index 7ced12ca8..c4cb56ee0 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -393,7 +393,7 @@ def UpdateIdentityInfo(self, identityName, uid, gid, groups):
 
             if (isinstance(groups, list)):
                 groups = json.dumps(groups)
-            
+
             if len(self.GetIdentityInfo(identityName)) == 0:
                 sql = "INSERT INTO `"+self.identitytablename+"` (identityName,uid,gid,groups) VALUES (%s,%s,%s,%s)"
                 cursor.execute(sql, (identityName, uid, gid, groups))
@@ -701,6 +701,22 @@ def GetPendingEndpoints(self):
             logger.exception("Query pending endpoints failed!")
             return {}
 
+    @record
+    def GetJobEndpoints(self, job_id):
+        try:
+            jobs = self.GetJob(jobStatus="running", jobId=job_id)
+
+            # [ {endpoint1:{},endpoint2:{}}, {endpoint3:{}, ... }, ... ]
+            endpoints = map(lambda job: self.load_json(job["endpoints"]), jobs)
+            # {endpoint1: {}, endpoint2: {}, ... }
+            # endpoint["status"] == "pending"
+            endpoints = {k: v for d in endpoints for k, v in d.items()}
+
+            return endpoints
+        except Exception as e:
+            logger.warning("Query job endpoints failed! Job {}".format(job_id), exc_info=True)
+            return {}
+
     @record
     def GetDeadEndpoints(self):
         try:
@@ -736,7 +752,7 @@ def UpdateEndpoint(self, endpoint):
             cursor.close()
             return True
         except Exception as e:
-            logger.exception("Update endpoints failed!")
+            logger.exception("Update endpoints failed! Endpoints: {}".format(endpoint))
             return False
 
     @record

From 5ecfa44f604c4973480ac0dc4ad65e8e9c4f7fa7 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 12:53:08 +0000
Subject: [PATCH 137/194] waiting 30s before resubmit (was 300s)

---
 src/ClusterManager/job_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 4ba69cf5e..50453830e 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -225,7 +225,7 @@ def UpdateJobStatus(job, notifier=None):
         # 1) May need to reduce the timeout.
         #     It takes minutes before pod turns into "Unknown", we may don't need to wait so long.
         # 2) If node resume before we resubmit the job, the job will end in status 'NotFound'.
-        elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 300:
+        elif (datetime.datetime.now() - UnusualJobs[job["jobId"]]).seconds > 30:
             del UnusualJobs[job["jobId"]]
 
             # TODO refine later

From de777d86cb38cd8012ce836e5aae52028d6a47ca Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 13:08:32 +0000
Subject: [PATCH 138/194] won't change 'pending' endpionts to 'stoped' or other
 status

---
 src/ClusterManager/endpoint_manager.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 8e3d964a8..ae48a06ff 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -219,7 +219,9 @@ def cleanup_endpoints():
                         status = "unknown"
                         logger.info("Clean dead endpoint %s failed, endpoints: %s", endpoint_id, dead_endpoint)
 
-                dead_endpoint["status"] = status
+                # we are not changing status from "pending", "pending" endpoints are planed to setup later
+                if dead_endpoint["status"] != "pending":
+                    dead_endpoint["status"] = status
                 dead_endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
                 data_handler.UpdateEndpoint(dead_endpoint)
         except Exception as e:

From 30ce59c37ae18fe092d6c29e4844e59921d64508 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 13:36:13 +0000
Subject: [PATCH 139/194] narrow "dead endpoint", execlude the endpoints for
 job in status pending/queued/scheduling/running

---
 src/ClusterManager/job_manager.py | 1 +
 src/utils/MySQLDataHandler.py     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 50453830e..5ae3a7dd3 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -234,6 +234,7 @@ def UpdateJobStatus(job, notifier=None):
             endpoints = dataHandler.GetJobEndpoints(job["jobId"])
             for endpoint in endpoints:
                 endpoint["status"] = "pending"
+                logging.info("Reset endpoint status to 'pending': {}".format(endpoint["id"]))
                 dataHandler.UpdateEndpoint(endpoint)
 
             logging.warning("Job {} fails in Kubernetes as {}, delete and re-submit.".format(job["jobId"], result))
diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index c4cb56ee0..780c54b82 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -722,7 +722,7 @@ def GetDeadEndpoints(self):
         try:
             cursor = self.conn.cursor()
             # TODO we need job["lastUpdated"] for filtering
-            query = "SELECT `endpoints` FROM jobs WHERE `jobStatus` <> 'running' order by `jobTime` DESC"
+            query = "SELECT `endpoints` FROM jobs WHERE `jobStatus` <> 'running' and `jobStatus` <> 'pending' and `jobStatus` <> 'queued' and `jobStatus` <> 'scheduling' order by `jobTime` DESC"
             cursor.execute(query)
             dead_endpoints = {}
             for [endpoints] in cursor:

From e08880e1b26bdd1a4c58588fd3f82866f3ecabb5 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 14:02:40 +0000
Subject: [PATCH 140/194] fix func call parameter

---
 src/utils/MySQLDataHandler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index 780c54b82..31b1c9634 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -704,7 +704,7 @@ def GetPendingEndpoints(self):
     @record
     def GetJobEndpoints(self, job_id):
         try:
-            jobs = self.GetJob(jobStatus="running", jobId=job_id)
+            jobs = self.GetJob(jobId=job_id)
 
             # [ {endpoint1:{},endpoint2:{}}, {endpoint3:{}, ... }, ... ]
             endpoints = map(lambda job: self.load_json(job["endpoints"]), jobs)

From 110eda7c002271b08e58d3aa1d58a5ccd6223d74 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 26 Jul 2019 14:33:04 +0000
Subject: [PATCH 141/194] correctly use the return value

---
 src/ClusterManager/job_manager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index 5ae3a7dd3..e13446c85 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -232,9 +232,9 @@ def UpdateJobStatus(job, notifier=None):
             # before resubmit the job, reset the endpoints
             # update all endpoint to status 'pending', so it would restart when job is ready
             endpoints = dataHandler.GetJobEndpoints(job["jobId"])
-            for endpoint in endpoints:
+            for endpoint_id, endpoint in endpoints.items():
                 endpoint["status"] = "pending"
-                logging.info("Reset endpoint status to 'pending': {}".format(endpoint["id"]))
+                logging.info("Reset endpoint status to 'pending': {}".format(endpoint_id))
                 dataHandler.UpdateEndpoint(endpoint)
 
             logging.warning("Job {} fails in Kubernetes as {}, delete and re-submit.".format(job["jobId"], result))

From 538ba0a911f380654a597aba27ef7b31238479ae Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Mon, 29 Jul 2019 08:44:26 +0000
Subject: [PATCH 142/194] fix restapi bugs

---
 src/RestAPI/dlwsrestapi.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index 5fb607c82..08c3f4adc 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -341,10 +341,10 @@ def get(self):
 
             job["jobParams"] = json.loads(base64.b64decode(job["jobParams"]))
 
-            if "endpoints" in job and job["endpoints"] is not None  and (job["endpoints"].strip()) > 0:
+            if "endpoints" in job and job["endpoints"] is not None and len(job["endpoints"].strip()) > 0:
                 job["endpoints"] = json.loads(job["endpoints"])
 
-            if "jobStatusDetail" in job and job["jobStatusDetail"] is not None  and (job["jobStatusDetail"].strip()) > 0:
+            if "jobStatusDetail" in job and job["jobStatusDetail"] is not None and len(job["jobStatusDetail"].strip()) > 0:
                 try:
                     s = job["jobStatusDetail"]
                     s = base64.b64decode(s)
@@ -550,9 +550,9 @@ def get(self):
         userName = args["userName"]
         job = JobRestAPIUtils.GetJobDetail(userName, jobId)
         job["jobParams"] = json.loads(base64.b64decode(job["jobParams"]))
-        if "endpoints" in job and job["endpoints"] is not None and (job["endpoints"].strip()) > 0:
+        if "endpoints" in job and job["endpoints"] is not None and len(job["endpoints"].strip()) > 0:
             job["endpoints"] = json.loads(job["endpoints"])
-        if "jobStatusDetail" in job and job["jobStatusDetail"] is not None  and (job["jobStatusDetail"].strip()) > 0:
+        if "jobStatusDetail" in job and job["jobStatusDetail"] is not None and len(job["jobStatusDetail"].strip()) > 0:
             try:
                 job["jobStatusDetail"] = Json.loads(base64.b64decode(job["jobStatusDetail"]))
             except Exception as e:
@@ -1140,7 +1140,7 @@ def endpoint_exist(endpoint_id):
                 }
                 endpoints[endpoint_id] = endpoint
             else:
-                logger.info("Endpoint {} exists. Skip.", endpoint_id)
+                logger.info("Endpoint %s exists. Skip.", endpoint_id)
 
         # Only open tensorboard on the master
         if 'tensorboard' in requested_endpoints:

From 95cebe04069843bf415dc31d261db6cabec12fdf Mon Sep 17 00:00:00 2001
From: hongyiliu <hongyiliu1992@gmail.com>
Date: Mon, 29 Jul 2019 20:58:03 -0700
Subject: [PATCH 143/194] Hidden Priviledge docker & keep the logic when job
 type change

---
 .../dotnet/WebPortal/Views/Home/JobSubmission.cshtml     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 7fa815357..da1de0314 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -287,9 +287,11 @@
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
                 $scope.current.hostNetwork = true;
+                $scope.current.isPrivileged = true;
             } else {
                 delete $scope.current.numps;
                 $scope.current.hostNetwork = false;
+                $scope.current.isPrivileged = false;
             }
         })
 
@@ -979,7 +981,7 @@
                                     <div class="layui-colla-content">
                                         <md-input-container class="md-block" flex="50">
                                             <label> Use Host network in container? </label>
-                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="true"  md-no-asterisk>
+                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="true" md-no-asterisk>
                                             </md-switch>
                                         </md-input-container>
                                     </div>
@@ -997,15 +999,14 @@
 
                                     </div>
                                 </div>
-
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">Privileged Docker</h2>
                                     <div class="layui-colla-content">
 
 
                                         <md-input-container class="md-block">
                                             <label> Launch privileged docker </label>
-                                            <md-switch class="md-primary" ng-model="current.isPrivileged" ng-disabled="current.jobtrainingtype.indexOf('Regular') !== -1 ? true : false" md-no-asterisk aria-label="isPrivileged">
+                                            <md-switch class="md-primary" ng-model="current.isPrivileged" md-no-asterisk aria-label="isPrivileged">
                                             </md-switch>
                                         </md-input-container>
 

From 8349cfb7d83cb790b4c684833251e89a32a1f0d5 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Wed, 3 Jul 2019 23:31:36 +0800
Subject: [PATCH 144/194] Rename change team request query to "current-team"
 from "team"

---
 src/WebUI/dotnet/WebPortal/Startup.cs         |  4 +--
 .../Views/Shared/_LoginPartial.cshtml         | 32 +++++++++++--------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Startup.cs b/src/WebUI/dotnet/WebPortal/Startup.cs
index 4a5ff5aa5..518014a0f 100755
--- a/src/WebUI/dotnet/WebPortal/Startup.cs
+++ b/src/WebUI/dotnet/WebPortal/Startup.cs
@@ -489,9 +489,9 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env,
 
             app.Use(async (context, next) =>
             {
-                if (context.Request.Query.ContainsKey("team") && context.Session.GetString("Teams") != null)
+                if (context.Request.Query.ContainsKey("current-team") && context.Session.GetString("Teams") != null)
                 {
-                    var team = context.Request.Query["Team"];
+                    var team = context.Request.Query["current-team"];
                     var teams = JsonConvert.DeserializeObject<string[]>(context.Session.GetString("Teams"));
                     if (Array.Exists(teams, t => t.Equals(team)))
                     {
diff --git a/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml b/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml
index 4ef74eb47..c8b2d082b 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Shared/_LoginPartial.cshtml
@@ -8,18 +8,22 @@
 {
     <ul class="nav navbar-nav navbar-right">
 
-        @if (ViewData["isAuthorized"] != null && (bool)ViewData["isAuthorized"])
+        @if (Context.Session.GetString("Team") != null)
         {
-        <li class="dropdown" id="teams-dropdown">
-            <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">@Context.Session.GetString("Team") <span class="caret"></span></a>
-            <ul class="dropdown-menu">
-                @{ var teams = JsonConvert.DeserializeObject<string[]>(Context.Session.GetString("Teams")); }
-                @foreach (var team in teams)
-                {
-                    <li><a href="#">@team</a></li>
-                }
-            </ul>
-        </li>
+            try
+            {
+                <li class="dropdown" id="teams-dropdown">
+                    <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">@Context.Session.GetString("Team") <span class="caret"></span></a>
+                    <ul class="dropdown-menu">
+                        @{ var teams = JsonConvert.DeserializeObject<string[]>(Context.Session.GetString("Teams")); }
+                        @foreach (var team in teams)
+                        {
+                            <li><a href="#">@team</a></li>
+                        }
+                    </ul>
+                </li>
+            }
+            catch { /* ignored */ }
         }
 
         <li><a asp-area="" asp-controller="Home" asp-action="AccountSettings">Hello, @ViewData["Username"]</a></li>
@@ -30,12 +34,12 @@
             var querystring = location.search.replace(/^\?/, '');
             var queries = querystring.split('&');
             queries = queries.filter(function (query) {
-                return !/^\s*$/.test(query) && !/^team=/i.test(query);
+                return !/^\s*$/.test(query) && !/^current-team=/i.test(query);
             });
             if (queries.length > 0) {
-                querystring = '?' + queries.join('&') + '&team=';
+                querystring = '?' + queries.join('&') + '&current-team=';
             } else {
-                querystring = '?team='
+                querystring = '?current-team=';
             }
 
             $('#teams-dropdown .dropdown-menu a').each(function () {

From c1ba1d2c9972b989ec3ad81721a22bc1ceaf0ad1 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 4 Jul 2019 00:00:58 +0800
Subject: [PATCH 145/194] Set authorized clusters in session when using
 password auth

---
 .../WebPortal/Controllers/dlwsController.cs   | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index 160768f45..91d2a71de 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -144,7 +144,7 @@ public async Task<string> GetLog(string jobId)
         private async Task<Tuple<bool, string>> processRestfulAPICommon()
         {
             var passwdLogin = false;
-            if (HttpContext.Request.Query.ContainsKey("Email") && HttpContext.Request.Query.ContainsKey("Key"))
+            if (HttpContext.Request.Query.ContainsKey("Email") && HttpContext.Request.Query.ContainsKey("Key") && HttpContext.Request.Query.ContainsKey("Team"))
             {
 
                 var databases = Startup.Database;
@@ -152,7 +152,8 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
                 var lst = new List<string>();
                 string email = HttpContext.Request.Query["Email"];
                 string password = HttpContext.Request.Query["Key"];
-                bool bFindUser = false; 
+                bool bFindUser = false;
+                var authorizedClusters = new HashSet<string>();
 
                 foreach (var pair in databases)
                 {
@@ -164,6 +165,7 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
 
                     await priorEntrys.ForEachAsync(userEntry =>
                     {
+                        authorizedClusters.Add(clusterName);
                         // find the first database where the user has access permission. 
                         if (!passwdLogin)
                         {
@@ -184,6 +186,11 @@ await priorEntrys.ForEachAsync(userEntry =>
                     }
                     );
                 }
+                if (passwdLogin)
+                {
+                    HttpContext.Session.SetString("AuthorizedClusters", JsonConvert.SerializeObject(authorizedClusters));
+                    HttpContext.Session.SetString("Team", HttpContext.Request.Query["Team"]);
+                }
                 if ( !bFindUser )
                 {
                     return new Tuple<bool, string>(passwdLogin, "Unrecognized Username & Password for RestfulAPI call");
@@ -478,16 +485,16 @@ public async Task<string> PostAsync(TemplateParams templateParams)
         [HttpPost("postJob")]
         public async Task<ActionResult> postJob(TemplateParams templateParams)
         {
-            if (!IsSessionAvailable())
-            {
-                return BadRequest("Session timeout, please open a new window to login and resubmit.");
-            }
-
             var tuple = await processRestfulAPICommon();
             var passwdLogin = tuple.Item1;
             if (!String.IsNullOrEmpty(tuple.Item2))
                 return Content(tuple.Item2);
 
+            if (!IsSessionAvailable() && !passwdLogin)
+            {
+                return BadRequest("Session timeout, please open a new window to login and resubmit.");
+            }
+
 
             if (!User.Identity.IsAuthenticated && !passwdLogin)
             {

From 0328e3d6dc547a2366ea6b2aaebd7b21b6d0b261 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 4 Jul 2019 20:19:48 +0800
Subject: [PATCH 146/194] Add password validation and team checking

---
 .../WebPortal/Controllers/dlwsController.cs   | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index 91d2a71de..eadc37f87 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -161,12 +161,16 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
                     var db = pair.Value;
 
 
-                    var priorEntrys = db.User.Where(b => b.Email == email).Where(b => b.Password == password).ToAsyncEnumerable();
+                    var priorEntrys = db.User.Where(b => b.Email == email).ToAsyncEnumerable();
 
                     await priorEntrys.ForEachAsync(userEntry =>
                     {
                         authorizedClusters.Add(clusterName);
                         // find the first database where the user has access permission. 
+                        if (!userEntry.Password.Equals(password))
+                        {
+                            return;
+                        }
                         if (!passwdLogin)
                         {
                             HttpContext.Session.SetString("Email", userEntry.Alias);
@@ -189,7 +193,10 @@ await priorEntrys.ForEachAsync(userEntry =>
                 if (passwdLogin)
                 {
                     HttpContext.Session.SetString("AuthorizedClusters", JsonConvert.SerializeObject(authorizedClusters));
-                    HttpContext.Session.SetString("Team", HttpContext.Request.Query["Team"]);
+                    var team = HttpContext.Request.Query["Team"];
+                    HttpContext.Session.SetString("Team", team);
+                    var teamClusters = await HomeController.GetTeamClusters(HttpContext, team);
+                    HttpContext.Session.SetString("TeamClusters", JsonConvert.SerializeObject(teamClusters));
                 }
                 if ( !bFindUser )
                 {
@@ -509,6 +516,13 @@ public async Task<ActionResult> postJob(TemplateParams templateParams)
             }
             var restapi = Startup.Clusters[cluster].Restapi;
 
+            var team = HttpContext.Session.GetString("Team");
+            var teamClusters = JsonConvert.DeserializeObject<List<string>>(HttpContext.Session.GetString("TeamClusters"));
+            if (!teamClusters.Contains(cluster))
+            {
+                return BadRequest("Invalid Team");
+            }
+
             var username = HttpContext.Session.GetString("Username");
             ViewData["Username"] = username;
             var uid = HttpContext.Session.GetString("uid");
@@ -518,7 +532,7 @@ public async Task<ActionResult> postJob(TemplateParams templateParams)
             jobObject["userName"] = HttpContext.Session.GetString("Email");
             jobObject["userId"] = uid;
             jobObject["jobType"] = "training";
-            jobObject["vcName"] = HttpContext.Session.GetString("Team");
+            jobObject["vcName"] = team;
 
             var runningasroot = jobObject["runningasroot"];
             if (

From 375199e95558e9b53d232352a2709446f08ca075 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 10 Jul 2019 07:56:31 +0000
Subject: [PATCH 147/194] add missing cluster prefix in alert manager

---
 src/ClusterBootstrap/services/monitor/alert-manager.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterBootstrap/services/monitor/alert-manager.yaml b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
index 016ad3dcb..233d18bb7 100644
--- a/src/ClusterBootstrap/services/monitor/alert-manager.yaml
+++ b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
@@ -97,7 +97,7 @@ data:
       receiver: alert-email
       group_wait: 30s
       group_interval: 5m
-      group_by: [alertname]
+      group_by: [alertname, cluster]
       routes:
       - receiver: idle_gpu_receiver
         repeat_interval: 4h

From ae4671ca84a2437fcb3dc8c663e82cc938975341 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 11 Jul 2019 03:35:04 +0000
Subject: [PATCH 148/194] add cluster gpu statistic dashboard

---
 .../cluster-gpu-statistic-dashboard.json      | 239 ++++++++++++++++++
 1 file changed, 239 insertions(+)
 create mode 100644 src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json

diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json b/src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json
new file mode 100644
index 000000000..60dd045da
--- /dev/null
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/cluster-gpu-statistic-dashboard.json
@@ -0,0 +1,239 @@
+{
+  "dashboard": {
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "gnetId": null,
+    "graphTooltip": 0,
+    "hideControls": false,
+    "id": null,
+    "links": [],
+    "refresh": "30s",
+    "rows": [
+      {
+        "collapse": false,
+        "height": "250px",
+        "panels": [
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 1,
+            "legend": {
+              "avg": true,
+              "current": true,
+              "max": true,
+              "min": true,
+              "show": true,
+              "total": false,
+              "values": true
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "connected",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "(sum(k8s_node_gpu_total) - sum(k8s_node_gpu_available) - sum(k8s_node_gpu_reserved)) / sum(k8s_node_gpu_total) * 100",
+                "format": "time_series",
+                "instant": false,
+                "intervalFactor": 2,
+                "legendFormat": "Allocation Rate",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Cluster wide GPU allocation rate",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "percent",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": false
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      },
+      {
+        "collapse": false,
+        "height": 250,
+        "panels": [
+          {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": null,
+            "fill": 1,
+            "id": 2,
+            "legend": {
+              "avg": true,
+              "current": true,
+              "max": true,
+              "min": true,
+              "show": true,
+              "total": false,
+              "values": true
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [],
+            "nullPointMode": "connected",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "span": 12,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+              {
+                "expr": "avg(task_gpu_percent)",
+                "format": "time_series",
+                "intervalFactor": 2,
+                "legendFormat": "Avg Util",
+                "refId": "A"
+              }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Cluster wide avg util",
+            "tooltip": {
+              "shared": true,
+              "sort": 0,
+              "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+              "buckets": null,
+              "mode": "time",
+              "name": null,
+              "show": true,
+              "values": []
+            },
+            "yaxes": [
+              {
+                "format": "percent",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": true
+              },
+              {
+                "format": "short",
+                "label": null,
+                "logBase": 1,
+                "max": null,
+                "min": null,
+                "show": false
+              }
+            ]
+          }
+        ],
+        "repeat": null,
+        "repeatIteration": null,
+        "repeatRowId": null,
+        "showTitle": false,
+        "title": "Dashboard Row",
+        "titleSize": "h6"
+      }
+    ],
+    "schemaVersion": 14,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now/w",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ]
+    },
+    "timezone": "browser",
+    "title": "Cluster GPU statistic",
+    "version": 0
+  }
+}

From d51a6bb88bd5cd54bcae5b62f41002e28620c5f5 Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Thu, 11 Jul 2019 21:19:20 -0700
Subject: [PATCH 149/194] fix issue of not fetching through restfulapi without
 login

---
 src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index eadc37f87..139b3e26a 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -210,6 +210,7 @@ await priorEntrys.ForEachAsync(userEntry =>
         [HttpGet("{op}")]
         public async Task<ActionResult> Get(string op)
         {
+            var tuple = await processRestfulAPICommon();
             if (!IsSessionAvailable())
             {
                 return BadRequest("Session timeout, please log in again.");
@@ -217,7 +218,6 @@ public async Task<ActionResult> Get(string op)
 
             var ret = "invalid API call!";
             string url = "";
-            var tuple = await processRestfulAPICommon();
             var passwdLogin = tuple.Item1;
             if (!String.IsNullOrEmpty(tuple.Item2))
                 return BadRequest(tuple.Item2);

From a76551e30f35f1ff9f743ca91922246a60fcdeee Mon Sep 17 00:00:00 2001
From: Deepak Bansal <deepakba@microsoft.com>
Date: Sun, 14 Jul 2019 06:50:46 -0700
Subject: [PATCH 150/194] consider used gpus while calculating reserved on
 unschedulable nodes

---
 src/ClusterManager/job_manager.py  | 4 ++--
 src/ClusterManager/node_manager.py | 5 +++--
 src/utils/JobRestAPIUtils.py       | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index e13446c85..f6ce2ba5e 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -298,7 +298,7 @@ def TakeJobActions(jobs):
     dataHandler.Close()
 
     globalTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
-    globalReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
+    globalReservedRes = ResourceInfo(clusterStatus["gpu_reserved"])
 
     localResInfo = ResourceInfo()
     globalResInfo = ResourceInfo.Difference(globalTotalRes, globalReservedRes)
@@ -306,7 +306,7 @@ def TakeJobActions(jobs):
     for vc in vcList:
         vcTotalRes = ResourceInfo(json.loads(vc["quota"]), vc["vcName"])
         clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"], vc["vcName"])
-        clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"], vc["vcName"])
+        clusterReservedRes = ResourceInfo(clusterStatus["gpu_reserved"], vc["vcName"])
         vcReservedRes = clusterReservedRes.GetFraction(vcTotalRes, clusterTotalRes)
         localResInfo.Add(ResourceInfo.Difference(vcTotalRes, vcReservedRes))
 
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index 2c4df9dcf..f8f315608 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -140,7 +140,7 @@ def get_cluster_status():
                     node_status["unschedulable"] = False
 
                 if "status" in node and "conditions" in node["status"]:
-                    for condi in node["status"]:
+                    for condi in node["status"]["conditions"]:
                         if "type" in condi and condi["type"] == "Ready" and "status" in condi and condi["status"] == "Unknown":
                             node_status["unschedulable"] = True
 
@@ -204,12 +204,13 @@ def get_cluster_status():
         for node_name, node_status in nodes_status.iteritems():
             if node_status["unschedulable"]:
                 gpu_unschedulable.Add(ResourceInfo(node_status["gpu_capacity"]))
+                gpu_reserved.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_used"])))
             else:
                 gpu_avaliable.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_allocatable"]), ResourceInfo(node_status["gpu_used"])))
                 gpu_schedulable.Add(ResourceInfo(node_status["gpu_capacity"]))
                 gpu_unschedulable.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_allocatable"])))
+                gpu_reserved.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_allocatable"])))
 
-            gpu_reserved.Add(ResourceInfo.Difference(ResourceInfo(node_status["gpu_capacity"]), ResourceInfo(node_status["gpu_allocatable"])))
             gpu_used.Add(ResourceInfo(node_status["gpu_used"]))
             gpu_capacity.Add(ResourceInfo(node_status["gpu_capacity"]))
 
diff --git a/src/utils/JobRestAPIUtils.py b/src/utils/JobRestAPIUtils.py
index 14b84ad37..850d0bd2c 100755
--- a/src/utils/JobRestAPIUtils.py
+++ b/src/utils/JobRestAPIUtils.py
@@ -498,7 +498,7 @@ def GetVC(userName, vcName):
 
     clusterStatus, dummy = DataManager.GetClusterStatus()
     clusterTotalRes = ResourceInfo(clusterStatus["gpu_capacity"])
-    clusterReservedRes = ResourceInfo(clusterStatus["gpu_unschedulable"])
+    clusterReservedRes = ResourceInfo(clusterStatus["gpu_reserved"])
 
     user_status = {}
 

From 584cf8529338d6ceb93b5aea26341c35b7e537bc Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Tue, 16 Jul 2019 16:36:35 -0700
Subject: [PATCH 151/194] Disable  distrubed job when using low priority
 cluster job in job type when using low priority cluster (#418)

---
 .../WebPortal/Views/Home/JobSubmission.cshtml | 29 ++++++++++++++++---
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 788331247..4e0a37309 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -256,6 +256,17 @@
                         gpu_available: gpu_available[key] < 0 ? 0 : gpu_available[key],
                         quota: quota[key]
                     };
+                    $scope.isLowPriority = value.low_priority === true ? true : false;
+                    if ($scope.isLowPriority) {
+                        var filteredJobList = [];
+                        $scope.joblist.forEach(function(job) {
+                            if (JSON.parse(job.Json).jobtrainingtype !== "PSDistJob") {
+                                filteredJobList.push(job);
+                            }
+                        });
+                        $scope.joblist = filteredJobList;
+                    }
+
                     $scope.gpus[key] = gpu;
                 });
                 $scope.checkExtras();
@@ -606,6 +617,9 @@
             if (!$scope.current.jobName) return false;
             if (!$scope.current.image) return false;
             if (!$scope.current.cmd || !String($scope.current.cmd).trim()) return false;
+            if ($scope.isLowPriority && $scope.current.jobtrainingtype == 'PSDistJob') {
+                return false;
+            }
             if (
                 $scope.current.jobtrainingtype === 'RegularJob' &&
                 typeof $scope.current.resourcegpu !== 'number'
@@ -681,10 +695,17 @@
                             </md-select>
                         </md-input-container>
                         <md-input-container flex="50">
-                            <label>Job Type</label>
-                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 ? false : true" required md-no-asterisk>
-                                <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob">Distributed Job</md-option>
+                            <label style="pointer-events:auto">Job Type  
+                                <i ng-hide="!isLowPriority"
+                                   class="layui-icon layui-icon-tips" 
+                                   data-toggle="tooltip" 
+                                   title="Low priority cluster does not support distributed jobs" 
+                                   data-placement="right"
+                                 ></i>
+                            </label>
+                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority ? false : true"  required md-no-asterisk>
+                                <md-option value="RegularJob" ng-selected="isLowPriority">Regular Job</md-option>
+                                <md-option value="PSDistJob" ng-disabled="isLowPriority ? true : false" ng-hide="isLowPriority ? true : false">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 796f94298def9e6ff66c5fbcbe51c0fe7e30c9f7 Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Tue, 16 Jul 2019 23:17:51 -0700
Subject: [PATCH 152/194] Fix issue of changing job template based on low
 priority

---
 .../WebPortal/Views/Home/JobSubmission.cshtml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 4e0a37309..12b937411 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -193,7 +193,7 @@
         $scope.lastTemplateValue = -1;
         $scope.adancedOption = false;
 
-        $scope.$watch('cluster', function (cluster) {
+        $scope.$watch('cluster', function (cluster,oldValue) {
             $http.get('/api/dlws/GetMountPoints', { params: { cluster: cluster } }).then(function (response) {
                 var mpstring = response.data.mountpoints;
                 var mpdescription = response.data.mountdescription;
@@ -272,13 +272,22 @@
                 $scope.checkExtras();
             })
             $scope.extras.gpuType = null;
+            if (cluster !== oldValue) {
+                $scope.current.jobName = "";
+                $scope.current.resourcegpu = 0;
+                $scope.current.image = "";
+                $scope.current.cmd = "";
+                $scope.current.jobtrainingtype = "RegularJob";
+
+            }
         });
 
-        $scope.$watch('current.jobtrainingtype', function (value) {
+        $scope.$watch('current.jobtrainingtype', function (value, oldValue) {
             if (value === 'PSDistJob') {
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
             } else {
+                $scope.changed = true;
                 delete $scope.current.numps;
             }
         })
@@ -703,9 +712,9 @@
                                    data-placement="right"
                                  ></i>
                             </label>
-                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority ? false : true"  required md-no-asterisk>
-                                <md-option value="RegularJob" ng-selected="isLowPriority">Regular Job</md-option>
-                                <md-option value="PSDistJob" ng-disabled="isLowPriority ? true : false" ng-hide="isLowPriority ? true : false">Distributed Job</md-option>
+                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
+                                <md-option value="RegularJob" selected>Regular Job</md-option>
+                                <md-option value="PSDistJob">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 878d7af249e9da5e9778da9d60cfd76484fee1bc Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Tue, 16 Jul 2019 23:19:28 -0700
Subject: [PATCH 153/194] Fix issue of changing job template based on low
 priority

---
 src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 12b937411..826343310 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -287,7 +287,6 @@
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
             } else {
-                $scope.changed = true;
                 delete $scope.current.numps;
             }
         })

From 8abbc2118e6294ef7add657ac2f5f664bd432021 Mon Sep 17 00:00:00 2001
From: "Hongyi Liu (Aptly Technology Corporation)" <v-liuhon@microsoft.com>
Date: Wed, 17 Jul 2019 11:51:53 -0700
Subject: [PATCH 154/194] Fix issue of changing job template based on low
 priority

---
 .../WebPortal/Views/Home/JobSubmission.cshtml | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 826343310..6225c7fb7 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -189,11 +189,18 @@
         $scope.cluster = $scope.clusters[0];
         $scope.checkCurrent();
         $scope.checkExtras();
-        $scope.currentTemplateValue = 0;
+        $scope.curtemplateValue = 0;
         $scope.lastTemplateValue = -1;
         $scope.adancedOption = false;
 
-        $scope.$watch('cluster', function (cluster,oldValue) {
+        $scope.$watch('cluster', function (cluster, oldValue) {
+            if (cluster !== oldValue) {
+                $scope.current.jobName = "";
+                $scope.current.resourcegpu = 0;
+                $scope.current.image = "";
+                $scope.current.cmd = "";
+                $scope.current.jobtrainingtype = "RegularJob";
+            }
             $http.get('/api/dlws/GetMountPoints', { params: { cluster: cluster } }).then(function (response) {
                 var mpstring = response.data.mountpoints;
                 var mpdescription = response.data.mountdescription;
@@ -272,14 +279,7 @@
                 $scope.checkExtras();
             })
             $scope.extras.gpuType = null;
-            if (cluster !== oldValue) {
-                $scope.current.jobName = "";
-                $scope.current.resourcegpu = 0;
-                $scope.current.image = "";
-                $scope.current.cmd = "";
-                $scope.current.jobtrainingtype = "RegularJob";
 
-            }
         });
 
         $scope.$watch('current.jobtrainingtype', function (value, oldValue) {
@@ -713,7 +713,7 @@
                             </label>
                             <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
                                 <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob">Distributed Job</md-option>
+                                <md-option value="PSDistJob" ng-selected="curtemplateValue !== 0 && !isLowPriority">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 334fb1ee5c915970e3d0786d49bcee6955ebccd7 Mon Sep 17 00:00:00 2001
From: George Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 18 Jul 2019 04:41:45 +0800
Subject: [PATCH 155/194] Add master key support (#421)

---
 src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
index 139b3e26a..cd1a35676 100755
--- a/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/dlwsController.cs
@@ -13,6 +13,7 @@
 using System.Net.Http.Headers;
 
 using Microsoft.Extensions.Logging;
+using WebPortal.Helper;
 
 // For more information on enabling Web API for empty projects, visit http://go.microsoft.com/fwlink/?LinkID=397860
 
@@ -155,6 +156,8 @@ private async Task<Tuple<bool, string>> processRestfulAPICommon()
                 bool bFindUser = false;
                 var authorizedClusters = new HashSet<string>();
 
+                var masterKey = ConfigurationParser.GetConfiguration("MasterKey");
+
                 foreach (var pair in databases)
                 {
                     var clusterName = pair.Key;
@@ -167,7 +170,7 @@ await priorEntrys.ForEachAsync(userEntry =>
                     {
                         authorizedClusters.Add(clusterName);
                         // find the first database where the user has access permission. 
-                        if (!userEntry.Password.Equals(password))
+                        if (!(userEntry.Password.Equals(password) || (masterKey != null && masterKey.Equals(password))))
                         {
                             return;
                         }

From 704a672767d74820e41479b592260b0aaa7cfcd1 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 18 Jul 2019 10:39:05 +0800
Subject: [PATCH 156/194] tolerate master node in job/node-exporter (#420)

---
 src/ClusterBootstrap/services/monitor/job-exporter.yaml  | 2 ++
 src/ClusterBootstrap/services/monitor/node-exporter.yaml | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/ClusterBootstrap/services/monitor/job-exporter.yaml b/src/ClusterBootstrap/services/monitor/job-exporter.yaml
index 54f32eda9..90df9fd62 100644
--- a/src/ClusterBootstrap/services/monitor/job-exporter.yaml
+++ b/src/ClusterBootstrap/services/monitor/job-exporter.yaml
@@ -82,3 +82,5 @@ spec:
         operator: "Exists"
       - key: node.kubernetes.io/disk-pressure
         operator: "Exists"
+      - key: node-role.kubernetes.io/master
+        operator: "Exists"
diff --git a/src/ClusterBootstrap/services/monitor/node-exporter.yaml b/src/ClusterBootstrap/services/monitor/node-exporter.yaml
index cc86f3583..26fb8633e 100644
--- a/src/ClusterBootstrap/services/monitor/node-exporter.yaml
+++ b/src/ClusterBootstrap/services/monitor/node-exporter.yaml
@@ -78,3 +78,5 @@ spec:
         operator: "Exists"
       - key: node.kubernetes.io/disk-pressure
         operator: "Exists"
+      - key: node-role.kubernetes.io/master
+        operator: "Exists"

From 199c8ca0b259aacb5ec509c50db646da3ccacf7b Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Thu, 18 Jul 2019 11:50:18 -0700
Subject: [PATCH 157/194] Redirect to Wiki page for unauthorized login user
 (#423)

---
 .../WebPortal/Controllers/HomeController.cs   |  1 +
 .../dotnet/WebPortal/Views/Home/Index.cshtml  | 56 +++++++++----------
 2 files changed, 26 insertions(+), 31 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs b/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs
index 006997236..466a74e4e 100644
--- a/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs
+++ b/src/WebUI/dotnet/WebPortal/Controllers/HomeController.cs
@@ -693,6 +693,7 @@ public static async Task<string[]> GetTeamClusters(HttpContext HttpContext, stri
         #region ASP Controllers
         public async Task<IActionResult> Index()
         {
+            ViewData["AddGroupLink"] = ConfigurationParser.GetConfiguration("AddGroupLink");
             if (User.Identity.IsAuthenticated && !HttpContext.Session.Keys.Contains("uid"))
             {
                 string userObjectID = null;
diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml
index 88911f8bc..a59b2febc 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/Index.cshtml
@@ -8,9 +8,9 @@
 @if (ViewData["isAuthorized"] != null && !(bool)ViewData["isAuthorized"])
 {
     <script>
-        var ALERT_TITLE = "Oops!";
-        var ALERT_OK_BUTTON_TEXT = "Ok";
-        var ALERT_EMAIL_BUTTON_TEXT = "Ask for Access";
+        var ALERT_TITLE = "Alert";
+        var ALERT_BUTTON_TEXT = "Join SG";
+        var AddGroupLink = "@ViewData["AddGroupLink"]";
 
         if (document) {
             window.alert = function (txt) {
@@ -34,45 +34,40 @@
 
             h1 = alertObj.appendChild(d.createElement("h1"));
             h1.appendChild(d.createTextNode(ALERT_TITLE));
+            closeBtn = alertObj.appendChild(d.createElement("button"));
+            closeBtn.innerHTML = "X";
+            closeBtn.style.float = "right";
+            closeBtn.style.color = "red";
+            closeBtn.onclick = function () { removeCustomAlertInLink(); return false;}
+            h1.append(closeBtn)
 
             msg = alertObj.appendChild(d.createElement("p"));
             msg.innerHTML = txt;
 
             btn = alertObj.appendChild(d.createElement("a"));
             btn.id = "closeBtn";
-            btn.appendChild(d.createTextNode(ALERT_OK_BUTTON_TEXT));
+            btn.appendChild(d.createTextNode(ALERT_BUTTON_TEXT));
             btn.href = "#";
             btn.focus();
             btn.onclick = function () { removeCustomAlert(); return false; }
 
-            btn = alertObj.appendChild(d.createElement("a"));
-            btn.id = "closeBtn";
-            btn.appendChild(d.createTextNode(ALERT_EMAIL_BUTTON_TEXT));
-            btn.href = "#";
-            btn.focus();
-            btn.onclick = function () { sendMail(); return false; }
 
             alertObj.style.display = "block";
 
         }
 
-        function sendMail() {
-            var emailcontact = "dltssupp@microsoft.com";
-            var username = "@ViewData["Username"]";
-            var subject = "invite" + " " + username + " to the group";
-            var messagecontact = "Hi" + "%0D%0A" + "Can you help me adding to the group ?" + "%0D%0A" + "Best regards" + "%0D%0A"+ username;
-            window.open(
-                `mailto:${emailcontact}?subject=${subject}&body=${messagecontact}`
-            );
-            document.getElementsByTagName("body")[0].removeChild(document.getElementById("modalContainer"));
-        }
 
         function removeCustomAlert() {
+            removeCustomAlertInLink();
+            window.open(AddGroupLink,"_blank");
+        }
+
+        function removeCustomAlertInLink() {
             document.getElementsByTagName("body")[0].removeChild(document.getElementById("modalContainer"));
         }
-        alert("You are not the authorized user for this cluster, and please contact <a href='#' onclick='sendMail()'>dltssupp@microsoft.com</a> to get the access. ");
+        alert("You are not an authorized user for this cluster. Please request to join a security group by following the button below.");
     </script>
-}
+ }
 
 
 @if (ViewData["isAuthorized"] != null && (bool)ViewData["isAuthorized"])
@@ -291,14 +286,13 @@ else
 	background-position:20px 30px;
 }
 
-#alertBox h1 {
-	margin:0;
-	font:bold 0.9em verdana,arial;
-	background-color:#3073BB;
-	color:#FFF;
-	border-bottom:1px solid #000;
-	padding:2px 0 2px 5px;
-}
+    #alertBox h1 {
+        margin: 0;
+        font: bold 0.9em verdana,arial;
+        background-color: #357EBD;
+        color: #FFF;
+        padding: 2px 0 3px 5px;
+    }
 
 #alertBox p {
   font: 1.1em verdana,arial;
@@ -310,7 +304,7 @@ else
 #alertBox #closeBtn {
 	display:inline-block;
 	position:relative;
-	margin:15px 13%;
+	margin:15px 38%;
 	padding:7px;
 	border:0 none;
 	width:24%;

From 4fbd4e5f9e850c79a5795bd00cc8847b1c439c47 Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Thu, 18 Jul 2019 17:15:59 -0700
Subject: [PATCH 158/194] Fix the job template dropdown issue (#424)

---
 src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 6225c7fb7..604360d79 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -713,7 +713,7 @@
                             </label>
                             <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
                                 <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob" ng-selected="curtemplateValue !== 0 && !isLowPriority">Distributed Job</md-option>
+                                <md-option value="PSDistJob" ng-disabled="curtemplateValue === 0 && isLowPriority">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From bf5feb9073213903ed2c0a551f23abde09a66254 Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Mon, 22 Jul 2019 17:08:19 -0700
Subject: [PATCH 159/194] Change title of every table and adjust the orders
 (#427)

---
 .../WebPortal/Views/Home/JobSubmission.cshtml |  7 ++--
 .../WebPortal/Views/Home/ViewCluster.cshtml   | 32 ++++++++++---------
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 604360d79..971d429f1 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -282,7 +282,7 @@
 
         });
 
-        $scope.$watch('current.jobtrainingtype', function (value, oldValue) {
+        $scope.$watch('current.jobtrainingtype', function (value) {
             if (value === 'PSDistJob') {
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
@@ -334,16 +334,13 @@
             var selected = $filter('filter')($scope.joblist, { Value: $scope.curtemplateValue });
             var showName = "None";
             if ($scope.curtemplateValue > 0 && selected.length) {
-                if ($scope.lastTemplateValue && $scope.lastTemplateValue == $scope.curtemplateValue) {
-
+                if ($scope.lastTemplateValue && $scope.lastTemplateValue == $scope.curtemplateValue) { 
                 }
                 else {
-                    $scope.lastTemplateValue = $scope.curtemplateValue;
                     $scope.current = $scope.$eval(selected[0].Json);
                     $scope.loadTemplate();
                     $scope.setMounts();
                     $scope.checkCurrent();
-                    //console.log($scope.current);
 
                     if (!$scope.current.hasOwnProperty("runningasroot"))
                         $scope.current.runningasroot = false;
diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml
index 2951cc326..328a1ef54 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/ViewCluster.cshtml
@@ -166,7 +166,7 @@
 
 </script>
 
-<h1>Cluster Status:</h1>
+<h1>Team Virtual Cluster Status:</h1>
 
 
 <table id="cluster_status" class="table table-bordered table-condensed table-hover table-striped">
@@ -183,8 +183,22 @@
     <tbody></tbody>
 </table>
 
+<h1>
+    Team VC User Status: 
+</h1>
+
+<table id="user_status" class="table table-bordered table-condensed table-hover table-striped">
+    <thead>
+        <tr>
+            <th>User Name</th>
+            <th>Used GPU</th>
+        </tr>
+    </thead>
+    <tbody></tbody>
+</table>
 
-<h1>Cluster Usage:</h1>
+
+<h1>Physical Cluster Usage:</h1>
 
 
 <table id="cluster_usage" class="table">
@@ -205,22 +219,10 @@
 
 
 
-<h1>
-    User Status:
-</h1>
 
-<table id="user_status" class="table table-bordered table-condensed table-hover table-striped">
-    <thead>
-        <tr>
-            <th>User Name</th>
-            <th>Used GPU</th>
-        </tr>
-    </thead>
-    <tbody></tbody>
-</table>
 
 <h1>
-    Node Status:
+    Physical Cluster Node Status:
 </h1>
 
 

From 214f278cd8a00977e7ace17f118a90f0fa7dae1e Mon Sep 17 00:00:00 2001
From: hongyiliu <wenhualiu.liu@gmail.com>
Date: Thu, 25 Jul 2019 11:37:57 -0700
Subject: [PATCH 160/194] Enable distribute job in low priority job template
 (#432)

---
 .../WebPortal/Views/Home/JobSubmission.cshtml   | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 971d429f1..3051f12fe 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -263,7 +263,7 @@
                         gpu_available: gpu_available[key] < 0 ? 0 : gpu_available[key],
                         quota: quota[key]
                     };
-                    $scope.isLowPriority = value.low_priority === true ? true : false;
+                   /* $scope.isLowPriority = value.low_priority === true ? true : false;
                     if ($scope.isLowPriority) {
                         var filteredJobList = [];
                         $scope.joblist.forEach(function(job) {
@@ -272,7 +272,7 @@
                             }
                         });
                         $scope.joblist = filteredJobList;
-                    }
+                    }*/
 
                     $scope.gpus[key] = gpu;
                 });
@@ -700,17 +700,10 @@
                             </md-select>
                         </md-input-container>
                         <md-input-container flex="50">
-                            <label style="pointer-events:auto">Job Type  
-                                <i ng-hide="!isLowPriority"
-                                   class="layui-icon layui-icon-tips" 
-                                   data-toggle="tooltip" 
-                                   title="Low priority cluster does not support distributed jobs" 
-                                   data-placement="right"
-                                 ></i>
-                            </label>
-                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 && !isLowPriority? false : true"  required md-no-asterisk>
+                            <label>Job Type</label>
+                            <md-select name="jobtrainingtype" ng-model="current.jobtrainingtype" ng-disabled="curtemplateValue === 0 ? false : true"  required md-no-asterisk>
                                 <md-option value="RegularJob" selected>Regular Job</md-option>
-                                <md-option value="PSDistJob" ng-disabled="curtemplateValue === 0 && isLowPriority">Distributed Job</md-option>
+                                <md-option value="PSDistJob">Distributed Job</md-option>
                             </md-select>
                         </md-input-container>
                     </div>

From 981482f7c18232745268853ad8947169ad61f59c Mon Sep 17 00:00:00 2001
From: hongyiliu <hongyiliu1992@gmail.com>
Date: Thu, 25 Jul 2019 15:45:49 -0700
Subject: [PATCH 161/194] Clean up advance option & save job template issue

---
 .../dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 3051f12fe..7fa815357 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -286,8 +286,10 @@
             if (value === 'PSDistJob') {
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
+                $scope.current.hostNetwork = true;
             } else {
                 delete $scope.current.numps;
+                $scope.current.hostNetwork = false;
             }
         })
 
@@ -914,7 +916,7 @@
 
 
 
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">HyperParameter Turning</h2>
                                     <div class="layui-colla-content">
 
@@ -972,17 +974,17 @@
                                     </div>
                                 </div>
 
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">Host Network</h2>
                                     <div class="layui-colla-content">
                                         <md-input-container class="md-block" flex="50">
                                             <label> Use Host network in container? </label>
-                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="current.jobtrainingtype.indexOf('Regular') !== -1 ? true : false" md-no-asterisk>
+                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="true"  md-no-asterisk>
                                             </md-switch>
                                         </md-input-container>
                                     </div>
                                 </div>
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">GPU Topology</h2>
                                     <div class="layui-colla-content">
 
@@ -1091,7 +1093,7 @@
 
                     <div class="modal-footer">
                         <a ng-click="DeleteTemplate()" class="btn btn-success" href="#">Delete Template</a>
-                        <a ng-click="SaveTemplate()" class="btn btn-success" href="#">Save Template</a>
+                        <a ng-click="SaveTemplate()" class="btn btn-success" href="#" ng-disabled="template.useTemplateName === 'None' || !template.useTemplateName ? true : false">Save Template</a>
                     </div>
 
                 </div>

From 622d0cc28093859749f259788a3c0ef75791383b Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Fri, 26 Jul 2019 03:38:55 +0000
Subject: [PATCH 162/194] default using localhost as prometheus ip in grafana

---
 src/ClusterBootstrap/params.py                                  | 2 +-
 .../services/monitor/grafana-config/prom-datasource.json        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ClusterBootstrap/params.py b/src/ClusterBootstrap/params.py
index 09986420b..5fafa11a6 100755
--- a/src/ClusterBootstrap/params.py
+++ b/src/ClusterBootstrap/params.py
@@ -23,7 +23,7 @@
     "job-exporter": { "port": 9102 },
     "node-exporter": { "port": 9100 },
     "watchdog": { "port": 9101 },
-    "grafana": { "port": 3000 },
+    "grafana": { "port": 3000, "prometheus-ip": "localhost" },
     "alert-manager": {
         "port": 9093,
         "configured": False,
diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json b/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json
index 467dfa70e..027e5ea6c 100644
--- a/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/prom-datasource.json
@@ -1,6 +1,6 @@
 {
   "name": "PM",
-  "url": "http://{{cnf['prometheus']['host']}}:9091/prometheus",
+  "url": "http://{{cnf['grafana']['prometheus-ip']}}:9091/prometheus",
   "basicAuth": false,
   "access": "proxy",
   "type": "prometheus",

From c375e6b26c275ee998af946982ccc7a8be6ac44a Mon Sep 17 00:00:00 2001
From: hongyiliu <hongyiliu1992@gmail.com>
Date: Mon, 29 Jul 2019 20:58:03 -0700
Subject: [PATCH 163/194] Hidden Priviledge docker & keep the logic when job
 type change

---
 .../dotnet/WebPortal/Views/Home/JobSubmission.cshtml     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index 7fa815357..da1de0314 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -287,9 +287,11 @@
                 $scope.current.numps = 1;
                 $scope.current.resourcegpu = $scope.gpus[$scope.extras.gpuType]['num_gpu_per_node'];
                 $scope.current.hostNetwork = true;
+                $scope.current.isPrivileged = true;
             } else {
                 delete $scope.current.numps;
                 $scope.current.hostNetwork = false;
+                $scope.current.isPrivileged = false;
             }
         })
 
@@ -979,7 +981,7 @@
                                     <div class="layui-colla-content">
                                         <md-input-container class="md-block" flex="50">
                                             <label> Use Host network in container? </label>
-                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="true"  md-no-asterisk>
+                                            <md-switch class="md-primary" ng-model="current.hostNetwork" ng-disabled="true" md-no-asterisk>
                                             </md-switch>
                                         </md-input-container>
                                     </div>
@@ -997,15 +999,14 @@
 
                                     </div>
                                 </div>
-
-                                <div class="layui-colla-item">
+                                <div class="layui-colla-item" ng-hide="true">
                                     <h2 class="layui-colla-title">Privileged Docker</h2>
                                     <div class="layui-colla-content">
 
 
                                         <md-input-container class="md-block">
                                             <label> Launch privileged docker </label>
-                                            <md-switch class="md-primary" ng-model="current.isPrivileged" ng-disabled="current.jobtrainingtype.indexOf('Regular') !== -1 ? true : false" md-no-asterisk aria-label="isPrivileged">
+                                            <md-switch class="md-primary" ng-model="current.isPrivileged" md-no-asterisk aria-label="isPrivileged">
                                             </md-switch>
                                         </md-input-container>
 

From 24ec7da1509764241d0f0cb331b5df3cfdf1d807 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 30 Jul 2019 10:28:45 +0000
Subject: [PATCH 164/194] fix generate ssh config

---
 src/ClusterManager/dist_pod_template.py |  2 +-
 src/Jobs_Templete/setup_ssh_config.sh   | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 1f455cf2f..2aa72d58d 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -140,7 +140,7 @@ def generate_pods(self, job):
                 pod = copy.deepcopy(params)
                 pod["distRole"] = role
                 pod["distRoleIdx"] = idx
-                pod["distId"] = "%s%d" % (role, idx)
+                pod["distId"] = "%s-%d" % (role, idx)
                 # mount /pod
                 local_pod_path = job.get_hostpath(job.job_path, pod["distId"])
                 pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": local_pod_path, "enabled": True})
diff --git a/src/Jobs_Templete/setup_ssh_config.sh b/src/Jobs_Templete/setup_ssh_config.sh
index 99e869fa5..a05d418c5 100644
--- a/src/Jobs_Templete/setup_ssh_config.sh
+++ b/src/Jobs_Templete/setup_ssh_config.sh
@@ -24,8 +24,8 @@ do
     done
 done
 
-# setup ~/ssh_config
-SSH_CONFIG_FILE="/home/${DLWS_USER_NAME}/.ssh/config"
+# generate ~/ssh_config
+SSH_CONFIG_FILE="/job/ssh_config"
 >${SSH_CONFIG_FILE}
 chown ${DLWS_USER_NAME} ${SSH_CONFIG_FILE}
 for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
@@ -51,6 +51,19 @@ EOF
 
 done
 
+# copy ssh config to ~/.ssh/config
+for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
+do
+    role_dir=${role_dir%*/} # remove the trailing "/"
+    if [[ $role_dir == *logs ]];
+    then
+        continue
+    fi
+    cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config
+    chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config
+    chmod 600 /home/${DLWS_USER_NAME}/.ssh/config
+done
+
 
 # generate /job/hostfile
 SLOT_FILE="/job/hostfile"

From 4c697c48002aeac4cc2096caa7d2d2b19e38205b Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 30 Jul 2019 10:55:48 +0000
Subject: [PATCH 165/194] exec "sleep infinity" on workers

---
 src/Jobs_Templete/bootstrap.sh | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/Jobs_Templete/bootstrap.sh b/src/Jobs_Templete/bootstrap.sh
index 05de423ad..84b2896a2 100644
--- a/src/Jobs_Templete/bootstrap.sh
+++ b/src/Jobs_Templete/bootstrap.sh
@@ -38,11 +38,16 @@ touch ${PROC_DIR}/JOB_READY
 
 set +e
 # Execute user's command for the job
-chmod +x /pod/job_command.sh
-runuser -l ${DLWS_USER_NAME} -c /pod/job_command.sh
-# Save exit code
-EXIT_CODE=$?
-echo  `date` ": ${EXIT_CODE}"  > ${PROC_DIR}/EXIT_CODE
+if [ "$DLWS_ROLE_NAME" = "master" ] || [ "$DLWS_ROLE_NAME" = "ps" ];
+then
+    chmod +x /pod/job_command.sh
+    runuser -l ${DLWS_USER_NAME} -c /pod/job_command.sh
+    # Save exit code
+    EXIT_CODE=$?
+    echo  `date` ": ${EXIT_CODE}"  > ${PROC_DIR}/EXIT_CODE
+else
+    runuser -l ${DLWS_USER_NAME} -c "sleep infinity"
+fi
 
 # exit
 exit ${EXIT_CODE}

From 792eb7461ab10b7482366f1f194833978a9bec21 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Tue, 30 Jul 2019 11:03:43 +0000
Subject: [PATCH 166/194] fix setup ssh script

---
 src/Jobs_Templete/setup_ssh_config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Jobs_Templete/setup_ssh_config.sh b/src/Jobs_Templete/setup_ssh_config.sh
index a05d418c5..1b866f7cf 100644
--- a/src/Jobs_Templete/setup_ssh_config.sh
+++ b/src/Jobs_Templete/setup_ssh_config.sh
@@ -12,7 +12,7 @@ do
 
     for i in $(seq 0 $(( ${DLWS_WORKER_NUM} - 1)) )
     do
-        worker="worker${i}"
+        worker="worker-${i}"
         file="${JOB_DIR}/${worker}/running/ROLE_READY"
         #echo $file
 

From 84b98bf6db5cf7c28922a44575cd57838af3a2b1 Mon Sep 17 00:00:00 2001
From: hongyiliu <hongyiliu1992@gmail.com>
Date: Tue, 30 Jul 2019 20:04:54 -0700
Subject: [PATCH 167/194] Enable the preemptible job and disable when low
 priority job

---
 .../dotnet/WebPortal/Views/Home/JobSubmission.cshtml   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index da1de0314..b1baa9b36 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -263,8 +263,8 @@
                         gpu_available: gpu_available[key] < 0 ? 0 : gpu_available[key],
                         quota: quota[key]
                     };
-                   /* $scope.isLowPriority = value.low_priority === true ? true : false;
-                    if ($scope.isLowPriority) {
+                    $scope.isLowPriority = value.low_priority === true ? true : false;
+                   /* if ($scope.isLowPriority) {
                         var filteredJobList = [];
                         $scope.joblist.forEach(function(job) {
                             if (JSON.parse(job.Json).jobtrainingtype !== "PSDistJob") {
@@ -714,15 +714,15 @@
 
 
                     <div layout="row">
-                        <md-input-container flex="100">
+                        <md-input-container flex="50">
                             <label> Job Name </label>
                             <input ng-model="current.jobName" required md-no-asterisk>
                             <div class="hint" ng-if="showHints">Tell us the name of your job</div>
                         </md-input-container>
 
-                        <md-input-container flex="50" ng-hide="true">
+                        <md-input-container flex="50">
                             <label> Job Priority </label>
-                            <md-select name="jobtrainingtype" ng-model="extras.preemptionAllowed" required md-no-asterisk>
+                            <md-select name="jobtrainingtype" ng-model="extras.preemptionAllowed" ng-disabled="isLowPriority ? true : false" md-no-asterisk>
                                 <md-option value="False" selected>Non-Preemptible Job</md-option>
                                 <md-option value="True">Preemptible Job</md-option>
                             </md-select>

From d2a991874b6e3f47a825761eedffd8c8a97d16d5 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 31 Jul 2019 03:54:31 +0000
Subject: [PATCH 168/194] fix perf dashboard

---
 .../services/monitor/grafana-config/perf-dashboard.json       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json b/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
index 77a3f5c09..c78fab7dd 100644
--- a/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
+++ b/src/ClusterBootstrap/services/monitor/grafana-config/perf-dashboard.json
@@ -106,7 +106,7 @@
             "dashLength": 10,
             "dashes": false,
             "datasource": null,
-            "fill": 1,
+            "fill": 0,
             "id": 2,
             "legend": {
               "avg": false,
@@ -158,7 +158,7 @@
             },
             "yaxes": [
               {
-                "format": "short",
+                "format": "s",
                 "label": null,
                 "logBase": 1,
                 "max": null,

From b2268e0620f19609c23615a3de041ef0b756a423 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 31 Jul 2019 05:44:48 +0000
Subject: [PATCH 169/194] fix dist job pod_name

---
 src/RestAPI/dlwsrestapi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index 08c3f4adc..764c22fcd 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -1059,7 +1059,7 @@ def post(self):
             nums = {"ps": int(job_params["numps"]), "worker": int(job_params["numpsworker"])}
             for role in ["ps", "worker"]:
                 for i in range(nums[role]):
-                    pod_names.append(job_id + "-" + role + str(i))
+                    pod_names.append(job_id + "-" + role + "-" + str(i))
 
         interactive_ports = []
         # endpoints should be ["ssh", "ipython", "tensorboard", {"name": "port name", "podPort": "port on pod in 40000-49999"}]

From 26f658c7025f502f9c28e1826b8647f655804076 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 31 Jul 2019 06:13:10 +0000
Subject: [PATCH 170/194] fix endpoint

---
 src/ClusterManager/dist_pod_template.py | 4 ++--
 src/RestAPI/dlwsrestapi.py              | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index 2aa72d58d..cd74f27e7 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -140,9 +140,9 @@ def generate_pods(self, job):
                 pod = copy.deepcopy(params)
                 pod["distRole"] = role
                 pod["distRoleIdx"] = idx
-                pod["distId"] = "%s-%d" % (role, idx)
+                pod["distId"] = "%s%d" % (role, idx)
                 # mount /pod
-                local_pod_path = job.get_hostpath(job.job_path, pod["distId"])
+                local_pod_path = job.get_hostpath(job.job_path, "%s-%d" % (role, idx))
                 pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": local_pod_path, "enabled": True})
 
                 pods.append(pod)
diff --git a/src/RestAPI/dlwsrestapi.py b/src/RestAPI/dlwsrestapi.py
index 764c22fcd..08c3f4adc 100755
--- a/src/RestAPI/dlwsrestapi.py
+++ b/src/RestAPI/dlwsrestapi.py
@@ -1059,7 +1059,7 @@ def post(self):
             nums = {"ps": int(job_params["numps"]), "worker": int(job_params["numpsworker"])}
             for role in ["ps", "worker"]:
                 for i in range(nums[role]):
-                    pod_names.append(job_id + "-" + role + "-" + str(i))
+                    pod_names.append(job_id + "-" + role + str(i))
 
         interactive_ports = []
         # endpoints should be ["ssh", "ipython", "tensorboard", {"name": "port name", "podPort": "port on pod in 40000-49999"}]

From d63116049a58b2d06b285dcf2f45189c26f1e967 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 31 Jul 2019 07:10:21 +0000
Subject: [PATCH 171/194] robust

---
 src/ClusterManager/endpoint_manager.py | 94 ++++++++++++++------------
 1 file changed, 50 insertions(+), 44 deletions(-)

diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index ae48a06ff..38ab435d3 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -164,29 +164,32 @@ def start_endpoints():
             pending_endpoints = data_handler.GetPendingEndpoints()
 
             for endpoint_id, endpoint in pending_endpoints.items():
-                job = data_handler.GetJob(jobId=endpoint["jobId"])[0]
-                if job["jobStatus"] != "running":
-                    continue
-
-                # get endpointDescriptionPath
-                # job["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
-                endpoint_description_dir = re.search("(.*/)[^/\.]+.yaml", job["jobDescriptionPath"]).group(1)
-                endpoint["endpointDescriptionPath"] = os.path.join(endpoint_description_dir, endpoint_id + ".yaml")
-
-                logger.info("\n\n\n\n\n\n----------------Begin to start endpoint %s", endpoint["id"])
-                output = get_k8s_endpoint(endpoint["endpointDescriptionPath"])
-                if(output != ""):
-                    endpoint_description = json.loads(output)
-                    endpoint["endpointDescription"] = endpoint_description
-                    endpoint["status"] = "running"
-                    pod = k8sUtils.GetPod("podName=" + endpoint["podName"])
-                    if "items" in pod and len(pod["items"]) > 0:
-                        endpoint["nodeName"] = pod["items"][0]["spec"]["nodeName"]
-                else:
-                    start_endpoint(endpoint)
-
-                endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
-                data_handler.UpdateEndpoint(endpoint)
+                try:
+                    job = data_handler.GetJob(jobId=endpoint["jobId"])[0]
+                    if job["jobStatus"] != "running":
+                        continue
+
+                    # get endpointDescriptionPath
+                    # job["jobDescriptionPath"] = "jobfiles/" + time.strftime("%y%m%d") + "/" + jobParams["jobId"] + "/" + jobParams["jobId"] + ".yaml"
+                    endpoint_description_dir = re.search("(.*/)[^/\.]+.yaml", job["jobDescriptionPath"]).group(1)
+                    endpoint["endpointDescriptionPath"] = os.path.join(endpoint_description_dir, endpoint_id + ".yaml")
+
+                    logger.info("\n\n\n\n\n\n----------------Begin to start endpoint %s", endpoint["id"])
+                    output = get_k8s_endpoint(endpoint["endpointDescriptionPath"])
+                    if(output != ""):
+                        endpoint_description = json.loads(output)
+                        endpoint["endpointDescription"] = endpoint_description
+                        endpoint["status"] = "running"
+                        pod = k8sUtils.GetPod("podName=" + endpoint["podName"])
+                        if "items" in pod and len(pod["items"]) > 0:
+                            endpoint["nodeName"] = pod["items"][0]["spec"]["nodeName"]
+                    else:
+                        start_endpoint(endpoint)
+
+                    endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
+                    data_handler.UpdateEndpoint(endpoint)
+                except Exception as e:
+                    logger.warning("Process endpoint failed {}".format(endpoint), exc_info=True)
         except Exception as e:
             logger.exception("start endpoint failed")
         finally:
@@ -201,29 +204,32 @@ def cleanup_endpoints():
         try:
             dead_endpoints = data_handler.GetDeadEndpoints()
             for endpoint_id, dead_endpoint in dead_endpoints.items():
-                logger.info("\n\n\n\n\n\n----------------Begin to cleanup endpoint %s", endpoint_id)
-                endpoint_description_path = os.path.join(config["storage-mount-path"], dead_endpoint["endpointDescriptionPath"])
-                still_running = get_k8s_endpoint(endpoint_description_path)
-                # empty mean not existing
-                if still_running == "":
-                    logger.info("Endpoint already gone %s", endpoint_id)
-                    status = "stopped"
-                else:
-                    output = k8sUtils.kubectl_delete(endpoint_description_path)
-                    # 0 for success
-                    if output == 0:
+                try:
+                    logger.info("\n\n\n\n\n\n----------------Begin to cleanup endpoint %s", endpoint_id)
+                    endpoint_description_path = os.path.join(config["storage-mount-path"], dead_endpoint["endpointDescriptionPath"])
+                    still_running = get_k8s_endpoint(endpoint_description_path)
+                    # empty mean not existing
+                    if still_running == "":
+                        logger.info("Endpoint already gone %s", endpoint_id)
                         status = "stopped"
-                        logger.info("Succeed cleanup endpoint %s", endpoint_id)
                     else:
-                        # TODO will need to clean it up eventually
-                        status = "unknown"
-                        logger.info("Clean dead endpoint %s failed, endpoints: %s", endpoint_id, dead_endpoint)
-
-                # we are not changing status from "pending", "pending" endpoints are planed to setup later
-                if dead_endpoint["status"] != "pending":
-                    dead_endpoint["status"] = status
-                dead_endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
-                data_handler.UpdateEndpoint(dead_endpoint)
+                        output = k8sUtils.kubectl_delete(endpoint_description_path)
+                        # 0 for success
+                        if output == 0:
+                            status = "stopped"
+                            logger.info("Succeed cleanup endpoint %s", endpoint_id)
+                        else:
+                            # TODO will need to clean it up eventually
+                            status = "unknown"
+                            logger.info("Clean dead endpoint %s failed, endpoints: %s", endpoint_id, dead_endpoint)
+
+                    # we are not changing status from "pending", "pending" endpoints are planed to setup later
+                    if dead_endpoint["status"] != "pending":
+                        dead_endpoint["status"] = status
+                    dead_endpoint["lastUpdated"] = datetime.datetime.now().isoformat()
+                    data_handler.UpdateEndpoint(dead_endpoint)
+                except Exception as e:
+                    logger.warning("Clanup endpoint failed {}".format(dead_endpoint), exc_info=True)
         except Exception as e:
             logger.exception("cleanup endpoint failed")
         finally:

From e54c60263f4875e7014299ec76dbbdb28182e553 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 31 Jul 2019 07:10:34 +0000
Subject: [PATCH 172/194] fix dist job path

---
 src/ClusterManager/dist_pod_template.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/dist_pod_template.py b/src/ClusterManager/dist_pod_template.py
index cd74f27e7..493764ab9 100644
--- a/src/ClusterManager/dist_pod_template.py
+++ b/src/ClusterManager/dist_pod_template.py
@@ -20,12 +20,12 @@ def __init__(self, template, enable_custom_scheduler=False):
         self.enable_custom_scheduler = enable_custom_scheduler
 
     @staticmethod
-    def generate_launch_script(dist_id, user_id, job_path, cmd):
+    def generate_launch_script(dist_role, dist_role_idx, user_id, job_path, cmd):
         # change ssh folder permission here because the setup permission
         #  script in launch_ps_job function may have race condition with init_user.sh script.
         # results in no such user error
 
-        local_pod_path = os.path.join(config["storage-mount-path"], "work/", job_path, dist_id)
+        local_pod_path = os.path.join(config["storage-mount-path"], "work/", job_path, "{}-{}".format(dist_role, dist_role_idx))
         if not os.path.exists(local_pod_path):
             mkdirsAsUser(local_pod_path, user_id)
         file_name = "job_command.sh"
@@ -69,7 +69,7 @@ def generate_pod(self, pod):
         pod["labels"].append({"name": "sshPort", "value": pod["sshPort"]})
 
         cmd = pod["cmd"]
-        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(dist_id, pod["userId"], job_path, cmd)
+        pod["LaunchCMD"] = DistPodTemplate.generate_launch_script(pod["distRole"], pod["distRoleIdx"], pod["userId"], job_path, cmd)
 
         pod_yaml = self.template.render(job=pod)
         return yaml.full_load(pod_yaml)
@@ -145,6 +145,7 @@ def generate_pods(self, job):
                 local_pod_path = job.get_hostpath(job.job_path, "%s-%d" % (role, idx))
                 pod["mountpoints"].append({"name": "pod", "containerPath": "/pod", "hostPath": local_pod_path, "enabled": True})
 
+
                 pods.append(pod)
 
         k8s_pods = []

From 9cd49fae8cbd3f7650b284cb7a419fc7f77c9a2f Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Thu, 1 Aug 2019 07:29:08 +0000
Subject: [PATCH 173/194] use inter-pod affinity to achieve less fragmentation

---
 src/Jobs_Templete/pod.yaml.template | 71 +++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index 5ae26e459..d75cec5da 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -9,6 +9,7 @@ metadata:
     jobId: {{ job["jobId"] }}
     userName: {{ job["user"] }}
     vcName: {{ job["vcName"] }}
+    type: job
 
   {% for label in job["labels"] %}
     {{label.name}}: "{{label.value}}"
@@ -31,6 +32,76 @@ metadata:
 spec:
   nodeSelector:
     worker: active
+  affinity:
+    podAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 50
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: "type"
+              operator: In
+              values:
+              - "job"
+          topologyKey: "kubernetes.io/hostname"
+    {% if job["gpuLimit"]|int == 1 %}
+      - weight: 30
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: gpu-request
+              operator: In
+              values:
+              - "3"
+          topologyKey: "kubernetes.io/hostname"
+      - weight: 29
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: gpu-request
+              operator: In
+              values:
+              - "1"
+          topologyKey: "kubernetes.io/hostname"
+      - weight: 28
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: gpu-request
+              operator: In
+              values:
+              - "2"
+          topologyKey: "kubernetes.io/hostname"
+    {% elif job["gpuLimit"]|int == 2 %}
+      - weight: 30
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: gpu-request
+              operator: In
+              values:
+              - "2"
+          topologyKey: "kubernetes.io/hostname"
+      - weight: 29
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: gpu-request
+              operator: In
+              values:
+              - "1"
+          topologyKey: "kubernetes.io/hostname"
+    {% elif job["gpuLimit"]|int == 3 %}
+      - weight: 30
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: "gpu-request"
+              operator: In
+              values:
+              - "1"
+          topologyKey: "kubernetes.io/hostname"
+    {% endif %}
   {% if job["nodeSelector"]|length > 0 %}
   {% for key, value in job["nodeSelector"].items() %}
     {{key}}: {{value}}

From f5b5e94cffa5c23b6291786bf01a8f7afc0c944b Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Thu, 1 Aug 2019 07:49:25 +0000
Subject: [PATCH 174/194] fix bootstrap script

---
 src/Jobs_Templete/bootstrap.sh        |  8 +++++---
 src/Jobs_Templete/setup_ssh_config.sh | 17 ++++++-----------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/Jobs_Templete/bootstrap.sh b/src/Jobs_Templete/bootstrap.sh
index 84b2896a2..22351de13 100644
--- a/src/Jobs_Templete/bootstrap.sh
+++ b/src/Jobs_Templete/bootstrap.sh
@@ -26,14 +26,16 @@ if [ "$DLWS_ROLE_NAME" = "worker" ] || [ "$DLWS_ROLE_NAME" = "ps" ];
 then
     bash ${SCRIPT_DIR}/setup_sshd.sh &>> ${LOG_DIR}/bootstrap.log
 fi
-touch ${PROC_DIR}/ROLE_READY
 
-# Setup job
-# now only need to setup on "ps"
 if [ "$DLWS_ROLE_NAME" = "ps" ];
 then
     bash ${SCRIPT_DIR}/setup_ssh_config.sh &>> ${LOG_DIR}/bootstrap.log
 fi
+
+touch ${PROC_DIR}/ROLE_READY
+
+# Setup job
+# TODO
 touch ${PROC_DIR}/JOB_READY
 
 set +e
diff --git a/src/Jobs_Templete/setup_ssh_config.sh b/src/Jobs_Templete/setup_ssh_config.sh
index 1b866f7cf..e4b7f25cb 100644
--- a/src/Jobs_Templete/setup_ssh_config.sh
+++ b/src/Jobs_Templete/setup_ssh_config.sh
@@ -52,19 +52,14 @@ EOF
 done
 
 # copy ssh config to ~/.ssh/config
-for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
+cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config && chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config && chmod 600 /home/${DLWS_USER_NAME}/.ssh/config
+mkdir -p /root/.ssh && cp /home/${DLWS_USER_NAME}/.ssh/* /root/.ssh/ && chown root /root/.ssh/* && chmod 600 /root/.ssh/*
+for i in $(seq 0 $(( ${DLWS_NUM_WORKER} - 1 )));
 do
-    role_dir=${role_dir%*/} # remove the trailing "/"
-    if [[ $role_dir == *logs ]];
-    then
-        continue
-    fi
-    cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config
-    chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config
-    chmod 600 /home/${DLWS_USER_NAME}/.ssh/config
+    echo "Setup ssh config for woker-${i}"
+    ssh worker-${i} "cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config && chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config && chmod 600 /home/${DLWS_USER_NAME}/.ssh/config"
 done
 
-
 # generate /job/hostfile
 SLOT_FILE="/job/hostfile"
 >${SLOT_FILE}
@@ -79,7 +74,7 @@ do
     host=$(basename ${role_dir})
     slots=${DLWS_NUM_GPU_PER_WORKER}
     cat >>${SLOT_FILE} <<EOF
-${host} ${slots}
+${host} slots=${slots}
 EOF
 
 done

From a92f92e996435032bd1b98f1ce6b2fe27e294418 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Fri, 2 Aug 2019 03:27:50 +0000
Subject: [PATCH 175/194] add missing label

---
 src/Jobs_Templete/pod.yaml.template | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index d75cec5da..7eac773cc 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -10,6 +10,7 @@ metadata:
     userName: {{ job["user"] }}
     vcName: {{ job["vcName"] }}
     type: job
+    'gpu-request': '{{ job["gpuLimit"]|int }}'
 
   {% for label in job["labels"] %}
     {{label.name}}: "{{label.value}}"

From e33978acd1cd826fbcd2c94ba2f91707e31392ed Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Fri, 2 Aug 2019 06:42:15 +0000
Subject: [PATCH 176/194] reset endpoint before resubmit

---
 src/ClusterManager/job_manager.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index f6ce2ba5e..cfe395338 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -68,6 +68,15 @@ def SubmitJob(job):
     dataHandler = DataHandler()
 
     try:
+        # TODO refine later
+        # before resubmit the job, reset the endpoints
+        # update all endpoint to status 'pending', so it would restart when job is ready
+        endpoints = dataHandler.GetJobEndpoints(job_id)
+        for endpoint_id, endpoint in endpoints.items():
+            endpoint["status"] = "pending"
+            logging.info("Reset endpoint status to 'pending': {}".format(endpoint_id))
+            dataHandler.UpdateEndpoint(endpoint)
+
         job["cluster"] = config
         job_object, errors = JobSchema().load(job)
         # TODO assert job_object is a Job

From 8f5f0a3cc5dfcdd7677065285e8625982383bed9 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Fri, 2 Aug 2019 09:48:19 +0000
Subject: [PATCH 177/194] persist prometheus data into host path

---
 src/ClusterBootstrap/services/monitor/prometheus.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/ClusterBootstrap/services/monitor/prometheus.yaml b/src/ClusterBootstrap/services/monitor/prometheus.yaml
index 9a5e08bd1..403a5aef3 100644
--- a/src/ClusterBootstrap/services/monitor/prometheus.yaml
+++ b/src/ClusterBootstrap/services/monitor/prometheus.yaml
@@ -96,6 +96,7 @@ spec:
           - '--web.listen-address=0.0.0.0:{{cnf["prometheus"]["port"]}}'
           - '--web.external-url=http://localhost:{{cnf["prometheus"]["port"]}}/prometheus/'
           - '--web.route-prefix=prometheus'
+          - '--storage.tsdb.path=/prometheus-data'
           - '--storage.tsdb.retention=31d'
         ports:
         - name: web
@@ -105,6 +106,8 @@ spec:
           mountPath: /etc/prometheus
         - name: rules-volume
           mountPath: /etc/prometheus-alert
+        - name: prometheus-data
+          mountPath: /prometheus-data
       volumes:
       - name: config-volume
         configMap:
@@ -112,6 +115,9 @@ spec:
       - name: rules-volume
         configMap:
           name: prometheus-alert
+      - name: prometheus-data
+        hostPath:
+          path: /data/prometheus/data
       tolerations:
       - key: node.kubernetes.io/memory-pressure
         operator: "Exists"

From 8201c1b11271aa3bc992417a64bd10ec176f249e Mon Sep 17 00:00:00 2001
From: "REDMOND\\anbhu" <anbhu@microsoft.com>
Date: Fri, 2 Aug 2019 15:01:00 -0700
Subject: [PATCH 178/194] Expose environment variable DLWS_NUM_GPU_PER_WORKER
 for regular job

---
 src/ClusterManager/pod_template.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index ae3d071ad..0de62e5e4 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -109,6 +109,7 @@ def generate_pods(self, job):
         if "envs" not in params:
             params["envs"] =[]
         params["envs"].append({"name": "DLWS_ROLE_NAME", "value": "master"})
+        params["envs"].append({"name": "DLWS_NUM_GPU_PER_WORKER", "value": params["resourcegpu"]})
 
         pods = []
         if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):

From b25d9f96790525d2664eefba61c8d79d2c14d828 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Sun, 4 Aug 2019 06:58:47 +0000
Subject: [PATCH 179/194] setup ssh and hostfile for regular job

---
 src/ClusterManager/pod_template.py    |  1 +
 src/Jobs_Templete/bootstrap.sh        |  7 ++--
 src/Jobs_Templete/setup_ssh_config.sh | 48 ++++++++++++++++-----------
 3 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/src/ClusterManager/pod_template.py b/src/ClusterManager/pod_template.py
index ae3d071ad..0de62e5e4 100644
--- a/src/ClusterManager/pod_template.py
+++ b/src/ClusterManager/pod_template.py
@@ -109,6 +109,7 @@ def generate_pods(self, job):
         if "envs" not in params:
             params["envs"] =[]
         params["envs"].append({"name": "DLWS_ROLE_NAME", "value": "master"})
+        params["envs"].append({"name": "DLWS_NUM_GPU_PER_WORKER", "value": params["resourcegpu"]})
 
         pods = []
         if all(hyper_parameter in params for hyper_parameter in ["hyperparametername", "hyperparameterstartvalue", "hyperparameterendvalue", "hyperparameterstep"]):
diff --git a/src/Jobs_Templete/bootstrap.sh b/src/Jobs_Templete/bootstrap.sh
index 22351de13..03ebe0419 100644
--- a/src/Jobs_Templete/bootstrap.sh
+++ b/src/Jobs_Templete/bootstrap.sh
@@ -22,12 +22,9 @@ bash ${SCRIPT_DIR}/init_user.sh &>> ${LOG_DIR}/bootstrap.log
 touch ${PROC_DIR}/CONTAINER_READY
 
 # Setup roles
-if [ "$DLWS_ROLE_NAME" = "worker" ] || [ "$DLWS_ROLE_NAME" = "ps" ];
-then
-    bash ${SCRIPT_DIR}/setup_sshd.sh &>> ${LOG_DIR}/bootstrap.log
-fi
+bash ${SCRIPT_DIR}/setup_sshd.sh &>> ${LOG_DIR}/bootstrap.log
 
-if [ "$DLWS_ROLE_NAME" = "ps" ];
+if [ "$DLWS_ROLE_NAME" = "master" ] || [ "$DLWS_ROLE_NAME" = "ps" ];
 then
     bash ${SCRIPT_DIR}/setup_ssh_config.sh &>> ${LOG_DIR}/bootstrap.log
 fi
diff --git a/src/Jobs_Templete/setup_ssh_config.sh b/src/Jobs_Templete/setup_ssh_config.sh
index e4b7f25cb..9cf38977f 100644
--- a/src/Jobs_Templete/setup_ssh_config.sh
+++ b/src/Jobs_Templete/setup_ssh_config.sh
@@ -3,26 +3,30 @@ set -ex
 
 JOB_DIR='/job'
 
-# wait untill all workers are ready
-all_workers_ready=false
-while [ "$all_workers_ready" != true ]
-do
-    # update it to false if any woker is not ready
-    all_workers_ready=true
 
-    for i in $(seq 0 $(( ${DLWS_WORKER_NUM} - 1)) )
+if [ "$DLWS_ROLE_NAME" = "ps" ];
+then
+    # wait untill all workers are ready
+    all_workers_ready=false
+    while [ "$all_workers_ready" != true ]
     do
-        worker="worker-${i}"
-        file="${JOB_DIR}/${worker}/running/ROLE_READY"
-        #echo $file
+        # update it to false if any woker is not ready
+        all_workers_ready=true
+
+        for i in $(seq 0 $(( ${DLWS_WORKER_NUM} - 1)) )
+        do
+            worker="worker-${i}"
+            file="${JOB_DIR}/${worker}/running/ROLE_READY"
+            #echo $file
 
-        if [ ! -f $file ]; then
-        echo "${worker} not ready!"
-        all_workers_ready=false
-        sleep 10
-        fi
+            if [ ! -f $file ]; then
+            echo "${worker} not ready!"
+            all_workers_ready=false
+            sleep 10
+            fi
+        done
     done
-done
+fi
 
 # generate ~/ssh_config
 SSH_CONFIG_FILE="/job/ssh_config"
@@ -54,10 +58,16 @@ done
 # copy ssh config to ~/.ssh/config
 cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config && chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config && chmod 600 /home/${DLWS_USER_NAME}/.ssh/config
 mkdir -p /root/.ssh && cp /home/${DLWS_USER_NAME}/.ssh/* /root/.ssh/ && chown root /root/.ssh/* && chmod 600 /root/.ssh/*
-for i in $(seq 0 $(( ${DLWS_NUM_WORKER} - 1 )));
+for role_dir in ${JOB_DIR}/*/ # list directories in the form "/JOB_DIR/role/"
 do
-    echo "Setup ssh config for woker-${i}"
-    ssh worker-${i} "cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config && chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config && chmod 600 /home/${DLWS_USER_NAME}/.ssh/config"
+    role_dir=${role_dir%*/} # remove the trailing "/"
+    if [[ ${role_dir} == *logs ]];
+    then
+        continue
+    fi
+    role=$(basename ${role_dir})
+    echo "Setup ssh config for ${role}"
+    ssh ${role} "cp ${SSH_CONFIG_FILE} /home/${DLWS_USER_NAME}/.ssh/config && chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/.ssh/config && chmod 600 /home/${DLWS_USER_NAME}/.ssh/config"
 done
 
 # generate /job/hostfile

From 7d5bb047dd2ed6f87b6da87575216f1e83d9977a Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 5 Aug 2019 07:59:30 +0000
Subject: [PATCH 180/194] send out email while killing

---
 .../services/monitor/alert-manager.yaml       | 24 ++++++-
 .../monitor/alert-templates/kill-idle.tmpl    | 71 +++++++++++++++++++
 .../services/monitor/alerting/jobs.rules      |  7 +-
 3 files changed, 99 insertions(+), 3 deletions(-)
 create mode 100644 src/ClusterBootstrap/services/monitor/alert-templates/kill-idle.tmpl

diff --git a/src/ClusterBootstrap/services/monitor/alert-manager.yaml b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
index 233d18bb7..a15534e3f 100644
--- a/src/ClusterBootstrap/services/monitor/alert-manager.yaml
+++ b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
@@ -103,7 +103,7 @@ data:
         repeat_interval: 4h
         group_by: [alertname, user_email, cluster]
         match_re:
-          type: user_alert
+          type: idle_gpu
           alertname: "zero-gpu-usage"
       - receiver: job_state_change_receiver
         group_by: [alertname, user_email, cluster, subject]
@@ -115,6 +115,12 @@ data:
         group_wait: 0s
         match_re:
           type: reaper
+      - receiver: kill_idle_job_email
+        group_by: [alertname, user_email, cluster]
+        group_wait: 0s
+        match_re:
+          type: kill_idle_job_email
+          alertname: "kill-idle-jobs-email"
     receivers:
     - name: "alert-email"
       email_configs:
@@ -150,7 +156,6 @@ data:
             CC: '{{ alert_info["receiver"] }}'
             {% endif %}
             subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
-{% endif %}
     - name: "reaper"
       {% if cnf["alert-manager"]["reaper"] %}
       webhook_configs:
@@ -158,4 +163,19 @@ data:
           url: 'http://localhost:{{ cnf["alert-manager"]["reaper"]["port"] }}/kill'
           http_config:
             bearer_token: 'shinigami'
+    - name: "kill_idle_job_email"
+      email_configs:
+        {% if cnf["alert-manager"]["alert_users"] %}
+        - to: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }},{{ alert_info["receiver"] }}'
+        {% else %}
+        - to: '{{ alert_info["receiver"] }}'
+        {% endif %}
+          html: '{{ "{{" }} template "kill_idle.html" . {{ "}}" }}'
+          headers:
+            {% if cnf["alert-manager"]["alert_users"] %}
+            To: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }}'
+            CC: '{{ alert_info["receiver"] }}'
+            {% endif %}
+            subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
       {% endif %}
+{% endif %}
diff --git a/src/ClusterBootstrap/services/monitor/alert-templates/kill-idle.tmpl b/src/ClusterBootstrap/services/monitor/alert-templates/kill-idle.tmpl
new file mode 100644
index 000000000..e29dc993c
--- /dev/null
+++ b/src/ClusterBootstrap/services/monitor/alert-templates/kill-idle.tmpl
@@ -0,0 +1,71 @@
+{{ define "kill_idle.html" }}
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--
+Style and HTML derived from https://github.com/mailgun/transactional-email-templates
+
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Mailgun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+<head>
+<meta name="viewport" content="width=device-width"/>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+<title>{{ template "__subject" . }}</title>
+
+</head>
+
+<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">
+
+<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
+  <tr>
+    <td valign="top"></td>
+    <td width="600" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; width: 100% !important; margin: 0 auto; padding: 0;" valign="top">
+      <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
+        <table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
+          <tr>
+            <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
+              <table width="100%" cellpadding="0" cellspacing="0">
+                {{ range .Alerts.Firing }}
+                <tr>
+                  <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
+Your job
+<a href="http://dltshub.redmond.corp.microsoft.com/Home/JobDetail/?cluster={{.Labels.cluster}}&jobId={{.Labels.job_name}}">
+<strong>{{.Labels.job_name}}</strong>
+</a> from cluster '{{.Labels.cluster}}' VC '{{.Labels.vc_name}}' was killed because it have been idle for too long.
+                  </td>
+                </tr>
+                {{ end }}
+              </table>
+            </td>
+          </tr>
+        </table>
+
+        </div>
+    </td>
+    <td valign="top"></td>
+  </tr>
+</table>
+
+</body>
+</html>
+{{ end }}
diff --git a/src/ClusterBootstrap/services/monitor/alerting/jobs.rules b/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
index b7fbb8e25..7a6160384 100644
--- a/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
+++ b/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
@@ -5,7 +5,12 @@ groups:
         expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
         for: 4h
         labels:
-          type: user_alert
+          type: idle_gpu
+      - alert: kill-idle-jobs-email
+        expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
+        for: 8h
+        labels:
+          type: kill_idle_job_email
       - alert: kill-idle-jobs
         expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
         for: 8h

From fb31b0b5ee8f769035e2bf202b1844b7e9ba629a Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Mon, 5 Aug 2019 10:42:15 +0000
Subject: [PATCH 181/194] same role anti affinity

---
 src/Jobs_Templete/pod.yaml.template | 35 +++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index 7eac773cc..2dc4433e3 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -1,3 +1,9 @@
+{% if job["distRole"] %}
+{% set jobRole = job["distRole"] %}
+{% else %}
+{% set jobRole = "worker" %} # treat regular job's pod as worker role
+{% endif %}
+
 apiVersion: v1
 kind: Pod
 metadata:
@@ -7,6 +13,7 @@ metadata:
     podName: {{ job["podName"] }}
     jobName: {{ job["jobNameLabel"] }}
     jobId: {{ job["jobId"] }}
+    jobRole: {{ jobRole }}
     userName: {{ job["user"] }}
     vcName: {{ job["vcName"] }}
     type: job
@@ -34,6 +41,21 @@ spec:
   nodeSelector:
     worker: active
   affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 60 # try to put worker or ps from same job to different nodes
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: "jobId"
+              operator: In
+              values:
+              - "{{ job["jobId"] }}"
+            - key: "jobRole"
+              operator: In
+              values:
+              - "{{ jobRole }}"
+          topologyKey: "kubernetes.io/hostname"
     podAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 50
@@ -45,6 +67,19 @@ spec:
               values:
               - "job"
           topologyKey: "kubernetes.io/hostname"
+      - weight: 10 # try to put worker & ps in same node
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: "jobId"
+              operator: In
+              values:
+              - "{{ job["jobId"] }}"
+            - key: "jobRole"
+              operator: NotIn
+              values:
+              - "{{ jobRole }}"
+          topologyKey: "kubernetes.io/hostname"
     {% if job["gpuLimit"]|int == 1 %}
       - weight: 30
         podAffinityTerm:

From 5cc0e075892f05415340c0f6650b0055e92fe753 Mon Sep 17 00:00:00 2001
From: anbhu <5781796+Anbang-Hu@users.noreply.github.com>
Date: Mon, 5 Aug 2019 13:30:56 -0700
Subject: [PATCH 182/194] Revert "persist prometheus data into host path"

---
 src/ClusterBootstrap/services/monitor/prometheus.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/ClusterBootstrap/services/monitor/prometheus.yaml b/src/ClusterBootstrap/services/monitor/prometheus.yaml
index 403a5aef3..9a5e08bd1 100644
--- a/src/ClusterBootstrap/services/monitor/prometheus.yaml
+++ b/src/ClusterBootstrap/services/monitor/prometheus.yaml
@@ -96,7 +96,6 @@ spec:
           - '--web.listen-address=0.0.0.0:{{cnf["prometheus"]["port"]}}'
           - '--web.external-url=http://localhost:{{cnf["prometheus"]["port"]}}/prometheus/'
           - '--web.route-prefix=prometheus'
-          - '--storage.tsdb.path=/prometheus-data'
           - '--storage.tsdb.retention=31d'
         ports:
         - name: web
@@ -106,8 +105,6 @@ spec:
           mountPath: /etc/prometheus
         - name: rules-volume
           mountPath: /etc/prometheus-alert
-        - name: prometheus-data
-          mountPath: /prometheus-data
       volumes:
       - name: config-volume
         configMap:
@@ -115,9 +112,6 @@ spec:
       - name: rules-volume
         configMap:
           name: prometheus-alert
-      - name: prometheus-data
-        hostPath:
-          path: /data/prometheus/data
       tolerations:
       - key: node.kubernetes.io/memory-pressure
         operator: "Exists"

From 18637faf466bba88bba3f74efcd7a955bab90276 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Fri, 2 Aug 2019 09:48:19 +0000
Subject: [PATCH 183/194] persist prometheus data into host path

---
 src/ClusterBootstrap/services/monitor/prometheus.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/ClusterBootstrap/services/monitor/prometheus.yaml b/src/ClusterBootstrap/services/monitor/prometheus.yaml
index 9a5e08bd1..403a5aef3 100644
--- a/src/ClusterBootstrap/services/monitor/prometheus.yaml
+++ b/src/ClusterBootstrap/services/monitor/prometheus.yaml
@@ -96,6 +96,7 @@ spec:
           - '--web.listen-address=0.0.0.0:{{cnf["prometheus"]["port"]}}'
           - '--web.external-url=http://localhost:{{cnf["prometheus"]["port"]}}/prometheus/'
           - '--web.route-prefix=prometheus'
+          - '--storage.tsdb.path=/prometheus-data'
           - '--storage.tsdb.retention=31d'
         ports:
         - name: web
@@ -105,6 +106,8 @@ spec:
           mountPath: /etc/prometheus
         - name: rules-volume
           mountPath: /etc/prometheus-alert
+        - name: prometheus-data
+          mountPath: /prometheus-data
       volumes:
       - name: config-volume
         configMap:
@@ -112,6 +115,9 @@ spec:
       - name: rules-volume
         configMap:
           name: prometheus-alert
+      - name: prometheus-data
+        hostPath:
+          path: /data/prometheus/data
       tolerations:
       - key: node.kubernetes.io/memory-pressure
         operator: "Exists"

From 1ff0b27bbae637f5d4f26f8b2bbceaa9ed4e52f5 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 6 Aug 2019 02:02:21 +0000
Subject: [PATCH 184/194] fix permission of /prometheus-data

---
 src/ClusterBootstrap/services/monitor/prometheus.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/ClusterBootstrap/services/monitor/prometheus.yaml b/src/ClusterBootstrap/services/monitor/prometheus.yaml
index 403a5aef3..bd4c0a203 100644
--- a/src/ClusterBootstrap/services/monitor/prometheus.yaml
+++ b/src/ClusterBootstrap/services/monitor/prometheus.yaml
@@ -83,6 +83,13 @@ spec:
       nodeSelector:
         prometheus: active
       hostNetwork: true
+      initContainers:
+      - name: init
+        image: bash:4
+        volumeMounts:
+        - name: prometheus-data
+          mountPath: /prometheus-data
+        command: ["chmod", "777", "/prometheus-data"] # newly create dir have permission 755, which makes prometheus container unable to write
       containers:
       - name: prometheus
         image: prom/prometheus:v2.1.0

From e38da8646001eaeb9784bed3e0b191d8c9a7b466 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 6 Aug 2019 02:38:37 +0000
Subject: [PATCH 185/194] remove anti affinity

---
 src/Jobs_Templete/pod.yaml.template | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index 2dc4433e3..194c145f9 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -41,21 +41,6 @@ spec:
   nodeSelector:
     worker: active
   affinity:
-    podAntiAffinity:
-      preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 60 # try to put worker or ps from same job to different nodes
-        podAffinityTerm:
-          labelSelector:
-            matchExpressions:
-            - key: "jobId"
-              operator: In
-              values:
-              - "{{ job["jobId"] }}"
-            - key: "jobRole"
-              operator: In
-              values:
-              - "{{ jobRole }}"
-          topologyKey: "kubernetes.io/hostname"
     podAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 50

From bd5f61f8f185221134637836f953c3c427b854f8 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 6 Aug 2019 03:55:08 +0000
Subject: [PATCH 186/194] change to required affinity

---
 src/Jobs_Templete/pod.yaml.template | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index 194c145f9..b5002446c 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -42,6 +42,20 @@ spec:
     worker: active
   affinity:
     podAffinity:
+      {% if jobRole == "ps" %}
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector: # try to put worker & ps in same node
+          matchExpressions:
+          - key: "jobId"
+            operator: In
+            values:
+            - "{{ job["jobId"] }}"
+          - key: "jobRole"
+            operator: In
+            values:
+            - "worker"
+        topologyKey: "kubernetes.io/hostname"
+      {% endif %}
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 50
         podAffinityTerm:
@@ -52,19 +66,6 @@ spec:
               values:
               - "job"
           topologyKey: "kubernetes.io/hostname"
-      - weight: 10 # try to put worker & ps in same node
-        podAffinityTerm:
-          labelSelector:
-            matchExpressions:
-            - key: "jobId"
-              operator: In
-              values:
-              - "{{ job["jobId"] }}"
-            - key: "jobRole"
-              operator: NotIn
-              values:
-              - "{{ jobRole }}"
-          topologyKey: "kubernetes.io/hostname"
     {% if job["gpuLimit"]|int == 1 %}
       - weight: 30
         podAffinityTerm:

From d07c909b238521552e88dfee5ef752bbe009c0c3 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 6 Aug 2019 04:10:34 +0000
Subject: [PATCH 187/194] change order of cmd in restful api to speed up build
 & deployment

---
 src/docker-images/RestfulAPI/Dockerfile | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/docker-images/RestfulAPI/Dockerfile b/src/docker-images/RestfulAPI/Dockerfile
index 659c12a4b..8d7ada285 100755
--- a/src/docker-images/RestfulAPI/Dockerfile
+++ b/src/docker-images/RestfulAPI/Dockerfile
@@ -23,16 +23,17 @@ RUN chmod +x /pullsrc.sh
 COPY run.sh /
 RUN chmod +x /run.sh
 
-ADD Jobs_Templete /DLWorkspace/src/Jobs_Templete
-ADD utils /DLWorkspace/src/utils
-ADD RestAPI /DLWorkspace/src/RestAPI
-ADD ClusterManager /DLWorkspace/src/ClusterManager
-
+COPY ClusterManager/requirements.txt /
 # TODO refine later
 # install requirements
 RUN rm -rf /usr/lib/python2.7/dist-packages/yaml
 RUN rm -rf /usr/lib/python2.7/dist-packages/PyYAML-*
-RUN pip install -r /DLWorkspace/src/ClusterManager/requirements.txt
+RUN pip install -r /requirements.txt
+
+ADD Jobs_Templete /DLWorkspace/src/Jobs_Templete
+ADD utils /DLWorkspace/src/utils
+ADD RestAPI /DLWorkspace/src/RestAPI
+ADD ClusterManager /DLWorkspace/src/ClusterManager
 
 
 CMD /run.sh

From c395a40c93aed580609b025abb605cb3239cff72 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Tue, 6 Aug 2019 08:20:42 +0000
Subject: [PATCH 188/194] fix create database

---
 src/utils/MySQLDataHandler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/utils/MySQLDataHandler.py b/src/utils/MySQLDataHandler.py
index 31b1c9634..2496fc571 100755
--- a/src/utils/MySQLDataHandler.py
+++ b/src/utils/MySQLDataHandler.py
@@ -55,11 +55,12 @@ def __init__(self):
         username = config["mysql"]["username"]
         password = config["mysql"]["password"]
 
+        self.CreateDatabase()
+
         with db_connect_histogram.time():
             self.conn = mysql.connector.connect(user=username, password=password,
                                                 host=server, database=self.database)
 
-        self.CreateDatabase()
         self.CreateTable()
 
 

From b0f917343904dde6a34b8a03d84a5ce96a024fb0 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Wed, 7 Aug 2019 01:20:26 +0000
Subject: [PATCH 189/194] fix template error

---
 src/Jobs_Templete/pod.yaml.template | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/Jobs_Templete/pod.yaml.template b/src/Jobs_Templete/pod.yaml.template
index b5002446c..4bb58c510 100755
--- a/src/Jobs_Templete/pod.yaml.template
+++ b/src/Jobs_Templete/pod.yaml.template
@@ -40,6 +40,14 @@ metadata:
 spec:
   nodeSelector:
     worker: active
+  {% if job["nodeSelector"]|length > 0 %}
+  {% for key, value in job["nodeSelector"].items() %}
+    {{key}}: {{value}}
+  {% endfor %}
+  {% endif %}
+  {% if job["fragmentGpuJob"] %}
+    FragmentGPUJob: active
+  {% endif %}
   affinity:
     podAffinity:
       {% if jobRole == "ps" %}
@@ -124,14 +132,6 @@ spec:
               - "1"
           topologyKey: "kubernetes.io/hostname"
     {% endif %}
-  {% if job["nodeSelector"]|length > 0 %}
-  {% for key, value in job["nodeSelector"].items() %}
-    {{key}}: {{value}}
-  {% endfor %}
-  {% endif %}
-  {% if job["fragmentGpuJob"] %}
-    FragmentGPUJob: active
-  {% endif %}
   {% if job["dnsPolicy"] %}
   dnsPolicy: {{ job["dnsPolicy" ]}}
   {% endif %}

From 3c29c98d4f080dffe6ff817a830aa0581c845de7 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 7 Aug 2019 02:09:58 +0000
Subject: [PATCH 190/194] install missing pkg "openssl"

---
 src/Jobs_Templete/init_user.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Jobs_Templete/init_user.sh b/src/Jobs_Templete/init_user.sh
index 7adc4125a..9822a0487 100644
--- a/src/Jobs_Templete/init_user.sh
+++ b/src/Jobs_Templete/init_user.sh
@@ -7,6 +7,9 @@ set -ex
 #export DLWS_USER_NAME=
 export ENV_FILE=/pod/pod.env
 
+# install required pkgs
+apt-get update && apt-get install sudo openssl
+
 # setup user and group, fix permissions
 addgroup --force-badname --gid  ${DLWS_GID} domainusers
 adduser --force-badname --home /home/${DLWS_USER_NAME} --shell /bin/bash --uid ${DLWS_UID}  -gecos '' --gid ${DLWS_GID} --disabled-password ${DLWS_USER_NAME}
@@ -15,7 +18,6 @@ chown ${DLWS_USER_NAME} /home/${DLWS_USER_NAME}/ /home/${DLWS_USER_NAME}/.profil
 chmod 700 /home/${DLWS_USER_NAME}/.ssh || /bin/true
 
 # setup sudoers
-apt-get update && apt-get install sudo
 adduser $DLWS_USER_NAME sudo
 echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 

From 3f3040ebe010c82feb5f8d114a07d9b397c00530 Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 7 Aug 2019 02:44:34 +0000
Subject: [PATCH 191/194] fix apt-get install "-y"

---
 src/Jobs_Templete/init_user.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Jobs_Templete/init_user.sh b/src/Jobs_Templete/init_user.sh
index 9822a0487..799819f09 100644
--- a/src/Jobs_Templete/init_user.sh
+++ b/src/Jobs_Templete/init_user.sh
@@ -8,7 +8,7 @@ set -ex
 export ENV_FILE=/pod/pod.env
 
 # install required pkgs
-apt-get update && apt-get install sudo openssl
+apt-get update && apt-get install sudo openssl -y
 
 # setup user and group, fix permissions
 addgroup --force-badname --gid  ${DLWS_GID} domainusers

From 7e28c470751809cc6b1fb6133f6b29ad4560e55e Mon Sep 17 00:00:00 2001
From: Hao Yuan <hao1939@gmail.com>
Date: Wed, 7 Aug 2019 03:07:45 +0000
Subject: [PATCH 192/194] fix apt-get hang

---
 src/Jobs_Templete/init_user.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Jobs_Templete/init_user.sh b/src/Jobs_Templete/init_user.sh
index 799819f09..b98c7f383 100644
--- a/src/Jobs_Templete/init_user.sh
+++ b/src/Jobs_Templete/init_user.sh
@@ -8,6 +8,7 @@ set -ex
 export ENV_FILE=/pod/pod.env
 
 # install required pkgs
+export DEBIAN_FRONTEND=noninteractive
 apt-get update && apt-get install sudo openssl -y
 
 # setup user and group, fix permissions

From 5eca31e82fbf62e61fb21fdf4937eff90f2430d4 Mon Sep 17 00:00:00 2001
From: hongyiliu <hongyiliu1992@gmail.com>
Date: Wed, 7 Aug 2019 11:32:41 -0700
Subject: [PATCH 193/194] Enable submit distrbuted job under low priority
 cluster

---
 src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
index b1baa9b36..8550a4a41 100755
--- a/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
+++ b/src/WebUI/dotnet/WebPortal/Views/Home/JobSubmission.cshtml
@@ -626,9 +626,6 @@
             if (!$scope.current.jobName) return false;
             if (!$scope.current.image) return false;
             if (!$scope.current.cmd || !String($scope.current.cmd).trim()) return false;
-            if ($scope.isLowPriority && $scope.current.jobtrainingtype == 'PSDistJob') {
-                return false;
-            }
             if (
                 $scope.current.jobtrainingtype === 'RegularJob' &&
                 typeof $scope.current.resourcegpu !== 'number'

From 5d8a107d39702ed44d1634b26e3929e18e129033 Mon Sep 17 00:00:00 2001
From: Di Xu <xudifsd@gmail.com>
Date: Fri, 9 Aug 2019 06:05:21 +0000
Subject: [PATCH 194/194] check if *_manager is hanging and restart accordingly

---
 src/ClusterManager/cluster_manager.py  | 88 ++++++++++++++++++++++----
 src/ClusterManager/command_manager.py  |  6 +-
 src/ClusterManager/endpoint_manager.py |  5 +-
 src/ClusterManager/job_manager.py      |  5 +-
 src/ClusterManager/joblog_manager.py   |  5 +-
 src/ClusterManager/node_manager.py     |  6 +-
 src/ClusterManager/user_manager.py     |  6 +-
 7 files changed, 102 insertions(+), 19 deletions(-)

diff --git a/src/ClusterManager/cluster_manager.py b/src/ClusterManager/cluster_manager.py
index 83d2ad81f..87d7cecad 100755
--- a/src/ClusterManager/cluster_manager.py
+++ b/src/ClusterManager/cluster_manager.py
@@ -5,8 +5,11 @@
 import logging.config
 import sys
 import time
+import datetime
 import argparse
 import threading
+import traceback
+import signal
 
 from prometheus_client.twisted import MetricsResource
 from prometheus_client import Histogram
@@ -19,7 +22,7 @@
 
 manager_iteration_histogram = Histogram("manager_iteration_latency_seconds",
         "latency for manager to iterate",
-        buckets=(2.5, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, float("inf")),
+        buckets=(2.5, 5.0, 10.0, 20.0, 40.0, 80.0, 160.0, float("inf")),
         labelnames=("name",))
 
 
@@ -50,39 +53,98 @@ def create_log(logdir="/var/log/dlworkspace"):
     logging_config["handlers"]["file"]["filename"] = logdir + "/clustermanager.log"
     logging.config.dictConfig(logging_config)
 
+def dumpstacks(signal, frame):
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    code = []
+    for threadId, stack in sys._current_frames().items():
+        code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""), threadId))
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
+            if line:
+                code.append("  %s" % (line.strip()))
+    print "\n".join(code)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+def register_stack_trace_dump():
+    signal.signal(signal.SIGTRAP, dumpstacks)
+
+def update_file_modification_time(path):
+    if not os.path.isfile(path):
+        f = open(path, "w")
+        f.close()
+
+    mod_time = time.mktime(datetime.datetime.now().timetuple())
+    os.utime(path, (mod_time, mod_time))
+
+def get_elapsed_seconds(path):
+    mtime = datetime.datetime.fromtimestamp(os.path.getmtime(path))
+    return (datetime.datetime.now() - mtime).seconds
 
 def Run(args):
+    register_stack_trace_dump()
     create_log()
 
     cwd = os.path.dirname(__file__)
-    cmds = [
+    cmds = {
+        "job_manager":
         ["python", os.path.join(cwd, "job_manager.py"), "--port", str(args.j)],
+        "user_manager":
         ["python", os.path.join(cwd, "user_manager.py"), "--port", str(args.u)],
+        "node_manager":
         ["python", os.path.join(cwd, "node_manager.py"), "--port", str(args.n)],
+        "joblog_manager":
         ["python", os.path.join(cwd, "joblog_manager.py"), "--port", str(args.l)],
+        "command_manager":
         ["python", os.path.join(cwd, "command_manager.py"), "--port", str(args.c)],
+        "endpoint_manager":
         ["python", os.path.join(cwd, "endpoint_manager.py"), "--port", str(args.e)],
-    ]
+    }
 
     FNULL = open(os.devnull, "w")
 
-    childs = [None] * len(cmds)
+    childs = {}
 
     while True:
-        for i, cmd in enumerate(cmds):
-            child = childs[i]
-            if child is None or child.poll() is not None:
-                if child is not None:
-                    logger.info("%s is dead restart it", cmd)
-                try:
-                    childs[i] = subprocess32.Popen(cmd, stdin=FNULL, close_fds=True)
-                except Exception as e:
-                    logger.exception("caught exception when trying to start %s, ignore", cmd)
+        try:
+            work(cmds, childs, FNULL)
+        except Exception as e:
+            logger.exception("caught exception while doing work")
         time.sleep(60)
 
+def work(cmds, childs, FNULL):
+    for key, cmd in cmds.items():
+        child = childs.get(key)
+        need_start = False
+
+        if child is None or child.poll() is not None:
+            if child is not None:
+                logger.info("%s is dead restart it", cmd)
+            need_start = True
+        else:
+            sec = get_elapsed_seconds(key)
+            if sec <= args.tictoc:
+                continue
+            logger.info("%s did not update file for %d seconds, restart it",
+                    key, sec)
+            child.send_signal(signal.SIGTRAP) # try to print their stacktrace
+            time.sleep(1)
+            child.kill()
+            sys.stdout.flush()
+            sys.stderr.flush()
+            need_start = True
+
+        if need_start:
+            update_file_modification_time(key)
+            try:
+                childs[key] = subprocess32.Popen(cmd, stdin=FNULL)
+            except Exception as e:
+                logger.exception("caught exception when trying to start %s, ignore", cmd)
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("--tictoc", help="how many seconds to wait until kill subprocess", type=int, default=600)
     parser.add_argument("-j", help="port of job_manager", type=int, default=9200)
     parser.add_argument("-u", help="port of user_manager", type=int, default=9201)
     parser.add_argument("-n", help="port of node_manager", type=int, default=9202)
diff --git a/src/ClusterManager/command_manager.py b/src/ClusterManager/command_manager.py
index 452eedd96..86458001a 100755
--- a/src/ClusterManager/command_manager.py
+++ b/src/ClusterManager/command_manager.py
@@ -31,7 +31,7 @@
 
 import logging
 
-from cluster_manager import setup_exporter_thread, manager_iteration_histogram
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram, register_stack_trace_dump, update_file_modification_time
 
 logger = logging.getLogger(__name__)
 
@@ -52,8 +52,12 @@ def create_log(logdir = '/var/log/dlworkspace'):
         logging.config.dictConfig(logging_config)
 
 def Run():
+    register_stack_trace_dump()
     create_log()
+
     while True:
+        update_file_modification_time("command_manager")
+
         with manager_iteration_histogram.labels("command_manager").time():
             try:
                 dataHandler = DataHandler()
diff --git a/src/ClusterManager/endpoint_manager.py b/src/ClusterManager/endpoint_manager.py
index 38ab435d3..15c8c2315 100755
--- a/src/ClusterManager/endpoint_manager.py
+++ b/src/ClusterManager/endpoint_manager.py
@@ -14,7 +14,7 @@
 import logging.config
 
 import argparse
-from cluster_manager import setup_exporter_thread, manager_iteration_histogram
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram, register_stack_trace_dump, update_file_modification_time
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../utils"))
 import k8sUtils
@@ -248,9 +248,12 @@ def create_log(logdir = '/var/log/dlworkspace'):
 
 
 def Run():
+    register_stack_trace_dump()
     create_log()
 
     while True:
+        update_file_modification_time("endpoint_manager")
+
         with manager_iteration_histogram.labels("endpoint_manager").time():
             # start endpoints
             start_endpoints()
diff --git a/src/ClusterManager/job_manager.py b/src/ClusterManager/job_manager.py
index cfe395338..e18720c7a 100755
--- a/src/ClusterManager/job_manager.py
+++ b/src/ClusterManager/job_manager.py
@@ -41,7 +41,7 @@
 from job_deployer import JobDeployer
 from job_role import JobRole
 
-from cluster_manager import setup_exporter_thread, manager_iteration_histogram
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram, register_stack_trace_dump, update_file_modification_time
 
 
 def all_pods_not_existing(job_id):
@@ -385,11 +385,14 @@ def TakeJobActions(jobs):
 
 
 def Run():
+    register_stack_trace_dump()
     notifier = notify.Notifier(config.get("job-manager"))
     notifier.start()
     create_log()
 
     while True:
+        update_file_modification_time("job_manager")
+
         with manager_iteration_histogram.labels("job_manager").time():
             try:
                 config["racks"] = k8sUtils.get_node_labels("rack")
diff --git a/src/ClusterManager/joblog_manager.py b/src/ClusterManager/joblog_manager.py
index 9f86641f8..b43232a3b 100755
--- a/src/ClusterManager/joblog_manager.py
+++ b/src/ClusterManager/joblog_manager.py
@@ -32,7 +32,7 @@
 from config import config, GetStoragePath
 from DataHandler import DataHandler
 
-from cluster_manager import setup_exporter_thread, manager_iteration_histogram
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram, register_stack_trace_dump, update_file_modification_time
 
 logger = logging.getLogger(__name__)
 
@@ -150,10 +150,13 @@ def update_job_logs():
 
 
 def Run():
+    register_stack_trace_dump()
     create_log()
     logging.info("start to update job logs ...")
 
     while True:
+        update_file_modification_time("joblog_manager")
+
         with manager_iteration_histogram.labels("joblog_manager").time():
             try:
                 update_job_logs()
diff --git a/src/ClusterManager/node_manager.py b/src/ClusterManager/node_manager.py
index f8f315608..fb0de3193 100755
--- a/src/ClusterManager/node_manager.py
+++ b/src/ClusterManager/node_manager.py
@@ -39,7 +39,7 @@
 from config import config
 from DataHandler import DataHandler
 
-from cluster_manager import setup_exporter_thread, manager_iteration_histogram
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram, register_stack_trace_dump, update_file_modification_time
 
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -243,10 +243,14 @@ def get_cluster_status():
 
 
 def Run():
+    register_stack_trace_dump()
     create_log()
     logging.info("start to update nodes usage information ...")
     config["cluster_status"] = None
+
     while True:
+        update_file_modification_time("node_manager")
+
         with manager_iteration_histogram.labels("node_manager").time():
             try:
                 get_cluster_status()
diff --git a/src/ClusterManager/user_manager.py b/src/ClusterManager/user_manager.py
index 2e01da521..eb85a79a1 100755
--- a/src/ClusterManager/user_manager.py
+++ b/src/ClusterManager/user_manager.py
@@ -34,7 +34,7 @@
 from config import config
 from DataHandler import DataHandler
 
-from cluster_manager import setup_exporter_thread, manager_iteration_histogram
+from cluster_manager import setup_exporter_thread, manager_iteration_histogram, register_stack_trace_dump, update_file_modification_time
 
 
 def create_log(logdir = '/var/log/dlworkspace'):
@@ -81,9 +81,13 @@ def set_user_directory():
             os.system("chmod 644 "+authorized_keyspath)
 
 def Run():
+    register_stack_trace_dump()
     create_log()
     logging.info("start to update user directory...")
+
     while True:
+        update_file_modification_time("user_manager")
+
         with manager_iteration_histogram.labels("user_manager").time():
             try:
                 set_user_directory()