microsoft · hongzhili · Sep 25, 2019 · Jul 3, 2019 · Jul 3, 2019 · Jun 26, 2019
diff --git a/src/ClusterBootstrap/deploy.py b/src/ClusterBootstrap/deploy.py
@@ -2831,8 +2831,8 @@ def start_one_kube_service(fname):
             pass
 
     if fname == "./deploy/services/jobmanager/jobmanager.yaml":
-        # recreate the configmap init-user-script
-        run_kubectl( ["create configmap init-user-script --from-file=../Jobs_Templete/init_user.sh -o yaml --dry-run | ./deploy/bin/kubectl apply -f -"] )
+        # recreate the configmap dlws-scripts
+        run_kubectl( ["create configmap dlws-scripts --from-file=../Jobs_Templete/ -o yaml --dry-run | ./deploy/bin/kubectl apply -f -"] )
 
     run_kubectl( ["create", "-f", fname ] )
 

diff --git a/src/ClusterBootstrap/params.py b/src/ClusterBootstrap/params.py
@@ -23,14 +23,19 @@
     "job-exporter": { "port": 9102 },
     "node-exporter": { "port": 9100 },
     "watchdog": { "port": 9101 },
-    "grafana": { "port": 3000 },
+    "grafana": { "port": 3000, "prometheus-ip": "localhost" },
     "alert-manager": {
         "port": 9093,
         "configured": False,
         "alert_users": False,
         # If want to deploy with alert-manager, should config
         # configured with True, and fill appropriate value to:
         # smtp_url, smtp_from, smtp_auth_username, smtp_auth_password and receiver
+        "reaper": {
+            "dry-run": True,
+            "port": "9500",
+            "restful-url": "http://localhost:5000",
+        }
     },
 
     "mysql_port": "3306",

diff --git a/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml b/src/ClusterBootstrap/services/jobmanager/jobmanager.yaml
@@ -13,6 +13,9 @@ spec:
       labels:
         jobmanager-node: pod
         app: jobmanager
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
     spec:
       {% if cnf["dnsPolicy"] %}    
       dnsPolicy: {{cnf["dnsPolicy"]}}
@@ -39,7 +42,40 @@ spec:
         - mountPath: {{cnf["storage-mount-path"]}}/jobfiles
           name: dlwsdatajobfiles
         - mountPath: /var/log/dlworkspace
-          name: log                                                    
+          name: log
+        ports:
+        - containerPort: 9200
+          hostPort: 9200
+          name: job-mgr
+          protocol: TCP
+        - containerPort: 9201
+          hostPort: 9201
+          name: user-mgr
+          protocol: TCP
+        - containerPort: 9202
+          hostPort: 9202
+          name: node-mgr
+          protocol: TCP
+        - containerPort: 9203
+          hostPort: 9203
+          name: joblog-mgr
+          protocol: TCP
+        - containerPort: 9204
+          hostPort: 9204
+          name: cmd-mgr
+          protocol: TCP
+        - containerPort: 9205
+          hostPort: 9205
+          name: endpoint-mgr
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          initialDelaySeconds: 3
+          periodSeconds: 30
+          successThreshold: 1
+          tcpSocket:
+            port: 9200
+          timeoutSeconds: 10
       volumes:
       - name: certs
         hostPath:

diff --git a/src/ClusterBootstrap/services/monitor/alert-manager.yaml b/src/ClusterBootstrap/services/monitor/alert-manager.yaml
@@ -24,7 +24,7 @@ spec:
       hostNetwork: true
       containers:
       - name: alert-manager
-        image: prom/alertmanager:v0.15.1
+        image: prom/alertmanager:v0.18.0
         args:
           - '--config.file=/etc/alertmanager/config.yml'
           - '--storage.path=/alertmanager'
@@ -40,6 +40,23 @@ spec:
           mountPath: /alertmanager
         - name: templates-volume
           mountPath: /etc/alertmanager/template
+      {% if cnf["alert-manager"]["reaper"] %}
+      - name: reaper
+        image: {{cnf["worker-dockerregistry"]}}{{cnf["dockerprefix"]}}reaper:{{cnf["dockertag"]}}
+        command:
+          - 'python'
+          - '/reaper/main.py'
+          - '--port'
+          - '{{ cnf["alert-manager"]["reaper"]["port"] }}'
+          - '--restful_url'
+          - '{{ cnf["alert-manager"]["reaper"]["restful-url"] }}'
+          {% if cnf["alert-manager"]["reaper"]["dry-run"] %}
+          - '--dry_run'
+          {% endif %}
+        ports:
+        - name: alert-manager
+          containerPort: {{ cnf["alert-manager"]["reaper"]["port"] }}
+      {% endif %}
       volumes:
       - name: config-volume
         configMap:
@@ -80,22 +97,38 @@ data:
       receiver: alert-email
       group_wait: 30s
       group_interval: 5m
-      group_by: [alertname]
+      group_by: [alertname, cluster]
       routes:
-      - receiver: task_user
+      - receiver: idle_gpu_receiver
         repeat_interval: 4h
         group_by: [alertname, user_email, cluster]
         match_re:
-          type: user_alert
+          type: idle_gpu
           alertname: "zero-gpu-usage"
+      - receiver: job_state_change_receiver
+        group_by: [alertname, user_email, cluster, subject]
+        match_re:
+          type: user_alert
+          alertname: "job-state-changed"
+      - receiver: reaper
+        group_by: [alertname, user_email, job_name]
+        group_wait: 0s
+        match_re:
+          type: reaper
+      - receiver: kill_idle_job_email
+        group_by: [alertname, user_email, cluster]
+        group_wait: 0s
+        match_re:
+          type: kill_idle_job_email
+          alertname: "kill-idle-jobs-email"
     receivers:
     - name: "alert-email"
       email_configs:
         - to: {{ alert_info["receiver"] }}
           html: '{{ "{{" }} template "email.html" . {{ "}}" }}'
           headers:
             subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
-    - name: "task_user"
+    - name: "idle_gpu_receiver"
       email_configs:
         {% if cnf["alert-manager"]["alert_users"] %}
         - to: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }},{{ alert_info["receiver"] }}'
@@ -109,4 +142,40 @@ data:
             CC: '{{ alert_info["receiver"] }}'
             {% endif %}
             subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
+    - name: "job_state_change_receiver"
+      email_configs:
+        {% if cnf["alert-manager"]["alert_users"] %}
+        - to: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }},{{ alert_info["receiver"] }}'
+        {% else %}
+        - to: '{{ alert_info["receiver"] }}'
+        {% endif %}
+          html: '{{ "{{" }} template "job_state.html" . {{ "}}" }}'
+          headers:
+            {% if cnf["alert-manager"]["alert_users"] %}
+            To: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }}'
+            CC: '{{ alert_info["receiver"] }}'
+            {% endif %}
+            subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
+    - name: "reaper"
+      {% if cnf["alert-manager"]["reaper"] %}
+      webhook_configs:
+        - send_resolved: False
+          url: 'http://localhost:{{ cnf["alert-manager"]["reaper"]["port"] }}/kill'
+          http_config:
+            bearer_token: 'shinigami'
+    - name: "kill_idle_job_email"
+      email_configs:
+        {% if cnf["alert-manager"]["alert_users"] %}
+        - to: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }},{{ alert_info["receiver"] }}'
+        {% else %}
+        - to: '{{ alert_info["receiver"] }}'
+        {% endif %}
+          html: '{{ "{{" }} template "kill_idle.html" . {{ "}}" }}'
+          headers:
+            {% if cnf["alert-manager"]["alert_users"] %}
+            To: '{{ "{{" }} .GroupLabels.user_email {{ "}}" }}'
+            CC: '{{ alert_info["receiver"] }}'
+            {% endif %}
+            subject: '{{ "{{" }} .GroupLabels.cluster {{ "}}" }}: {{ "{{" }} template "__subject" . {{ "}}" }}'
+      {% endif %}
 {% endif %}
diff --git a/src/ClusterBootstrap/services/monitor/alert-templates/job_state.tmpl b/src/ClusterBootstrap/services/monitor/alert-templates/job_state.tmpl
@@ -0,0 +1,71 @@
+{{ define "job_state.html" }}
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--
+Style and HTML derived from https://github.com/mailgun/transactional-email-templates
+
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Mailgun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+<head>
+<meta name="viewport" content="width=device-width"/>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+<title>{{ template "__subject" . }}</title>
+
+</head>
+
+<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">
+
+<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
+  <tr>
+    <td valign="top"></td>
+    <td width="600" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; width: 100% !important; margin: 0 auto; padding: 0;" valign="top">
+      <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
+        <table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
+          <tr>
+            <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
+              <table width="100%" cellpadding="0" cellspacing="0">
+                {{ range .Alerts.Firing }}
+                <tr>
+                  <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
+Your job
+<a href="http://dltshub.redmond.corp.microsoft.com/Home/JobDetail/?cluster={{.Labels.cluster}}&jobId={{.Labels.job_name}}">
+<strong>{{.Labels.job_name}}</strong>
+</a> from cluster '{{.Labels.cluster}}' has changed to state of {{.Labels.job_state}}.
+                  </td>
+                </tr>
+                {{ end }}
+              </table>
+            </td>
+          </tr>
+        </table>
+
+        </div>
+    </td>
+    <td valign="top"></td>
+  </tr>
+</table>
+
+</body>
+</html>
+{{ end }}
diff --git a/src/ClusterBootstrap/services/monitor/alert-templates/kill-idle.tmpl b/src/ClusterBootstrap/services/monitor/alert-templates/kill-idle.tmpl
@@ -0,0 +1,71 @@
+{{ define "kill_idle.html" }}
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--
+Style and HTML derived from https://github.com/mailgun/transactional-email-templates
+
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Mailgun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+<head>
+<meta name="viewport" content="width=device-width"/>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+<title>{{ template "__subject" . }}</title>
+
+</head>
+
+<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">
+
+<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
+  <tr>
+    <td valign="top"></td>
+    <td width="600" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; width: 100% !important; margin: 0 auto; padding: 0;" valign="top">
+      <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
+        <table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
+          <tr>
+            <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
+              <table width="100%" cellpadding="0" cellspacing="0">
+                {{ range .Alerts.Firing }}
+                <tr>
+                  <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
+Your job
+<a href="http://dltshub.redmond.corp.microsoft.com/Home/JobDetail/?cluster={{.Labels.cluster}}&jobId={{.Labels.job_name}}">
+<strong>{{.Labels.job_name}}</strong>
+</a> from cluster '{{.Labels.cluster}}' VC '{{.Labels.vc_name}}' was killed because it have been idle for too long.
+                  </td>
+                </tr>
+                {{ end }}
+              </table>
+            </td>
+          </tr>
+        </table>
+
+        </div>
+    </td>
+    <td valign="top"></td>
+  </tr>
+</table>
+
+</body>
+</html>
+{{ end }}
diff --git a/src/ClusterBootstrap/services/monitor/alerting/jobs.rules b/src/ClusterBootstrap/services/monitor/alerting/jobs.rules
@@ -5,4 +5,14 @@ groups:
         expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
         for: 4h
         labels:
-          type: user_alert
+          type: idle_gpu
+      - alert: kill-idle-jobs-email
+        expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
+        for: 8h
+        labels:
+          type: kill_idle_job_email
+      - alert: kill-idle-jobs
+        expr: avg(task_gpu_percent) by (user_email, job_name, vc_name) == 0
+        for: 8h
+        labels:
+          type: reaper