Initial prototype for slurm stats

.gitignore

@@ -4,4 +4,4 @@ terraform.tfstate*
 inventory
 config-drive.iso
 venv
-collections
+collections

README.md

@@ -20,6 +20,34 @@ NB: Working DNS is a requirement.
     yum install terraform
     terraform init
 
+# Passwords
+
+Prior to running any other playbooks, you need to define a set of passwords. You can
+use the `generate-passwords.yml` playbook to automate this process:
+
+```
+ansible-playbook generate-passwords.yml
+```
+
+This will output a set of passwords <`repository root>/inventory/group_vars/all/passwords.yml`.
+Placing them in the inventory means that they will be defined for all playbooks.
+
+It is recommended to encrypt the contents of this file prior to commiting to git:
+
+```
+ansible-vault encrypt inventory/group_vars/all/passwords.yml
+```
+
+You will then need to provide a password when running the playbooks e.g:
+
+```
+ansible-playbook monitoring-db.yml --tags grafana --ask-vault-password
+```
+
+See the [Ansible vault documentation](https://docs.ansible.com/ansible/latest/user_guide/vault.html) for more details.
+
+# Usage
+
 NB: For development of roles/collections you may want to use this alternative to `ansible-galaxy ...`:
 
     mkdir roles

ansible.cfg

@@ -5,7 +5,8 @@ stderr_callback = debug
 gathering = smart
 forks = 30
 host_key_checking = False
+inventory = inventory
 
 [ssh_connection]
 ssh_args = -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
-pipelining = True
+pipelining = True

config/elastic/internal_users.yml

@@ -0,0 +1,23 @@
+---
+# See: https://aws.amazon.com/blogs/opensource/change-passwords-open-distro-for-elasticsearch/
+
+# This is the internal user database
+# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+
+_meta:
+  type: "internalusers"
+  config_version: 2
+
+# Define your internal users here
+
+admin:
+  hash: "{{ secrets_openhpc_elasticsearch_admin_password | password_hash('bcrypt') }}"
+  reserved: true
+  backend_roles:
+  - "admin"
+  description: "Admin user"
+
+kibanaserver:
+  hash: "{{ secrets_openhpc_elasticsearch_kibana_password | password_hash('bcrypt') }}"
+  reserved: true
+  description: "Used by kibana to connect to elastic"

config/filebeat/filebeat.yml

@@ -0,0 +1,56 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+setup.ilm:
+  # Failed to connect to backoff(elasticsearch(https://localhost:9200)): Connection marked as failed because
+  # the onConnect callback failed: request checking for ILM availability failed: 500 Internal Server Error:
+  #  {"error":{"root_cause":[{"type":"security_exception","reason":"Unexpected exception indices:admin/get"}],
+  # "type":"security_exception","reason":"Unexpected exception indices:admin/get"},"status":500}
+  enabled: false
+  rollover_alias: "filebeat"
+  pattern: "{now/d}-000001"
+
+filebeat.inputs:
+  - type: log
+    json.add_error_key: true
+    paths:
+      - '/logs/slurm-stats/*.json'
+    fields:
+      event.kind: event
+    fields_under_root: true
+
+processors:
+  - timestamp:
+      field: json.End
+      layouts:
+        - '2006-01-02T15:04:05'
+      test:
+        - '2020-06-17T10:17:48'
+  - timestamp:
+      target_field: 'event.end'
+      field: json.End
+      layouts:
+        - '2006-01-02T15:04:05'
+      test:
+        - '2020-06-17T10:17:48'
+  - timestamp:
+      target_field: 'event.start'
+      field: json.Start
+      layouts:
+        - '2006-01-02T15:04:05'
+      test:
+        - '2020-06-17T10:17:48'
+  - convert:
+      fields:
+        - {from: "json.NNodes", type: "integer"}
+        - {from: "json.NCPUS", type: "integer"}
+        - {from: "json.ElapsedRaw", type: "integer"}
+
+output.elasticsearch:
+  hosts: ["{{ groups.elastic.0 }}:9200"]
+  protocol: "https"
+  ssl.verification_mode: none
+  username: "admin"
+  password: "{{ secrets_openhpc_elasticsearch_admin_password }}"

config/prometheus/rules/precompute.rules

@@ -0,0 +1,20 @@
+# Required for openhpc dashboard
+
+groups:
+- name: opehnpc
+  interval: 60s
+  rules:
+  - record: node_cpu_system_seconds:record
+    expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode="system",job="node"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node"}[60s])))
+  - record: node_cpu_user_seconds:record
+    expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode="user",job="node"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node"}[60s])))
+  - record: node_cpu_iowait_seconds:record
+    expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode="iowait",job="node"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node"}[60s])))
+  - record: node_cpu_other_seconds:record
+    expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode!="idle",mode!="user",mode!="system",mode!="iowait",job="node"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node"}[60s])))
+  - record: node_cpu_scaling_frequency_hertz_avg:record
+    expr: avg by (instance) (node_cpu_scaling_frequency_hertz)
+  - record: node_cpu_scaling_frequency_hertz_min:record
+    expr: min by (instance) (node_cpu_scaling_frequency_hertz)
+  - record: node_cpu_scaling_frequency_hertz_max:record
+    expr: max by (instance) (node_cpu_scaling_frequency_hertz)

filter_plugins/utils.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2020, StackHPC
+# Apache 2 License
+
+from ansible.errors import AnsibleError, AnsibleFilterError
+from collections import defaultdict
+import jinja2
+from ansible.module_utils.six import string_types
+import os.path
+
+def readfile(fpath):
+    if not os.path.isfile(fpath):
+        return ""
+    with open(fpath) as f:
+        return f.read()
+
+class FilterModule(object):
+    ''' Ansible core jinja2 filters '''
+
+    def filters(self):
+        return {
+            # jinja2 overrides
+            'readfile': readfile
+        }

generate-passwords.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Generate passwords.yml
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Include password generation role
+      include_role:
+        name: passwords

inventory.tpl

@@ -22,3 +22,18 @@ ${cluster_name}_login
 
 [cluster_compute:children]
 ${cluster_name}_compute
+
+[elastic:children]
+${cluster_name}_login
+
+[kibana:children]
+${cluster_name}_login
+
+[slurm_stats:children]
+${cluster_name}_login
+
+[podman:children]
+elastic
+kibana
+slurm_stats
+

main.tf

@@ -30,6 +30,53 @@ variable "compute_image" {
   default = "CentOS-8-GenericCloud-8.2.2004-20200611.2.x86_64"
 }
 
+resource "openstack_networking_secgroup_v2" "secgroup_slurm_login" {
+  name        = "secgroup_slurm_login"
+  description = "Rules for the slurm login node"
+  # Fully manage with terraform
+  delete_default_rules = true
+}
+
+resource "openstack_networking_secgroup_v2" "secgroup_slurm_compute" {
+  name        = "secgroup_slurm_compute"
+  description = "Rules for the slurm compute node"
+  # Fully manage with terraform
+  delete_default_rules = true
+}
+
+resource "openstack_networking_secgroup_rule_v2" "secgroup_slurm_login_rule_egress_v4" {
+  direction         = "egress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.secgroup_slurm_login.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "secgroup_slurm_login_rule_ingress_tcp_v4" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  # NOTE: You will want to lock down the ports in a production environment. This will require
+  # setting of static ports for the NFS server see:
+  # https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/storage_administration_guide/s2-nfs-nfs-firewall-config
+  port_range_min    = 1
+  protocol          = "tcp"
+  port_range_max    = 65535
+  security_group_id = openstack_networking_secgroup_v2.secgroup_slurm_login.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "secgroup_slurm_compute_rule_egress_v4" {
+  direction         = "egress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.secgroup_slurm_compute.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "secgroup_slurm_compute_rule_ingress_tcp_v4" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  port_range_min    = 1
+  protocol          = "tcp"
+  port_range_max    = 65535
+  security_group_id = openstack_networking_secgroup_v2.secgroup_slurm_compute.id
+}
+
 resource "openstack_compute_instance_v2" "login" {
 
   name = "${var.cluster_name}-login-0"
@@ -39,6 +86,7 @@ resource "openstack_compute_instance_v2" "login" {
   network {
     name = "ilab"
   }
+  security_groups = [openstack_networking_secgroup_v2.secgroup_slurm_login.id]
 }
 
 
@@ -54,6 +102,7 @@ resource "openstack_compute_instance_v2" "compute" {
   network {
     name = "ilab"
   }
+  security_groups = [openstack_networking_secgroup_v2.secgroup_slurm_compute.id]
 }
 
 # TODO: needs fixing for case where creation partially fails resulting in "compute.network is empty list of object"
@@ -65,5 +114,5 @@ resource "local_file" "hosts" {
                             "computes": openstack_compute_instance_v2.compute,
                           },
                           )
-  filename = "${path.module}/inventory"
+  filename = "${path.module}/inventory/hosts"
 }