salt: use roles retrieved from Node objects

This allows to set `node-role.kubernetes.io/$ROLE` labels on a Node object, where `$ROLE` can be `bootstrap`, `etcd`, `master` and `node`, which will then determine the `highstate` of the node. Furthermore, it requires a `metalk8s.scality.com/version` label to select a MetalK8s version (actually, `saltenv`) to be used. It also removes the `node-role.kubernetes.io/master` Taint from the bootstrap Node, and replaces it with a `node-role.kubernetes.io/bootstrap` Taint, which is tolerated by CoreDNS and the UI. This is a special-case of the `node-role.kubernetes.io/master` Taint which would not permit such services to run on the node. See: #925 See: #925 See: #742 See: #742 See: #903 See: #903
scality · Apr 10, 2019 · 7ca2336 · 7ca2336
1 parent 074180b
commit 7ca2336
Show file tree

Hide file tree

Showing 19 changed files with 189 additions and 9 deletions.
diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py
@@ -73,6 +73,24 @@ def task__deploy_salt_tree() -> Iterator[types.TaskDict]:
 
 # List of salt files to install.
 SALT_FILES : Tuple[Union[Path, targets.FileTarget], ...] = (
+    targets.TemplateFile(
+        task_name='top.sls',
+        source=constants.ROOT/'salt'/'top.sls.in',
+        destination=constants.ISO_ROOT/'salt'/'top.sls',
+        context={'VERSION': constants.SHORT_VERSION},
+        file_dep=[constants.VERSION_FILE],
+    ),
+
+    Path('salt/metalk8s/roles/minion.sls'),
+    Path('salt/metalk8s/roles/bootstrap.sls'),
+    Path('salt/metalk8s/roles/salt-master.sls'),
+    Path('salt/metalk8s/roles/registry.sls'),
+    Path('salt/metalk8s/roles/repository.sls'),
+    Path('salt/metalk8s/roles/ca.sls'),
+    Path('salt/metalk8s/roles/etcd.sls'),
+    Path('salt/metalk8s/roles/master.sls'),
+    Path('salt/metalk8s/roles/node.sls'),
+
     Path('salt/metalk8s/orchestrate/bootstrap_without_master.sls'),
     Path('salt/metalk8s/orchestrate/deploy_etcd_on_new_node.sls'),
     Path('salt/metalk8s/orchestrate/bootstrap_with_master.sls'),
@@ -156,7 +174,13 @@ def task__deploy_salt_tree() -> Iterator[types.TaskDict]:
     Path('salt/metalk8s/kubeadm/init/kubeconfig/scheduler.sls'),
 
     Path('salt/metalk8s/kubeadm/init/mark-control-plane/init.sls'),
-    Path('salt/metalk8s/kubeadm/init/mark-control-plane/configured.sls'),
+    targets.TemplateFile(
+        task_name='configured.sls',
+        source=constants.ROOT/'salt'/'metalk8s'/'kubeadm'/'init'/'mark-control-plane'/'configured.sls.in',
+        destination=constants.ISO_ROOT/'salt'/'metalk8s'/'kubeadm'/'init'/'mark-control-plane'/'configured.sls',
+        context={'VERSION': constants.SHORT_VERSION},
+        file_dep=[constants.VERSION_FILE],
+    ),
 
     Path('salt/metalk8s/kubeadm/init/addons/init.sls'),
     Path('salt/metalk8s/kubeadm/init/addons/kube-proxy.sls'),
@@ -213,6 +237,7 @@ def task__deploy_salt_tree() -> Iterator[types.TaskDict]:
     Path('salt/_modules/metalk8s.py'),
 
     Path('salt/_pillar/metalk8s.py'),
+    Path('salt/_pillar/metalk8s_nodes.py'),
 
     Path('salt/_roster/kubernetes_nodes.py'),
 

diff --git a/examples/new-node.yaml b/examples/new-node.yaml
@@ -12,8 +12,12 @@ metadata:
     metalk8s.scality.com/ssh-key-path: <PATH/TO/SSH/KEY>
     metalk8s.scality.com/ssh-sudo: <true if the user is not root>
   labels:
-    beta.kubernetes.io/os: linux
+    metalk8s.scality.com/version: '2.0'
+    node-role.kubernetes.io/master: ''
+    node-role.kubernetes.io/etcd: ''
 spec:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/etcd
diff --git a/examples/new-node_vagrant.yaml b/examples/new-node_vagrant.yaml
@@ -12,8 +12,12 @@ metadata:
     metalk8s.scality.com/ssh-key-path: /etc/metalk8s/pki/preshared_key_for_k8s_nodes
     metalk8s.scality.com/ssh-sudo: "true" 
   labels:
-    beta.kubernetes.io/os: linux
+    metalk8s.scality.com/version: '2.0'
+    node-role.kubernetes.io/master: ''
+    node-role.kubernetes.io/etcd: ''
 spec:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/etcd
diff --git a/salt/_pillar/metalk8s_nodes.py b/salt/_pillar/metalk8s_nodes.py
@@ -0,0 +1,80 @@
+import os.path
+import logging
+
+try:
+    import kubernetes.client
+    import kubernetes.config
+    HAS_DEPS = True
+except ImportError:
+    HAS_DEPS = False
+
+
+VERSION_LABEL = 'metalk8s.scality.com/version'
+ROLE_LABEL_PREFIX = 'node-role.kubernetes.io/'
+
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'metalk8s_nodes'
+
+def __virtual__():
+    if HAS_DEPS:
+        return __virtualname__
+    else:
+        return False, 'Missing Kubernetes client library dependency'
+
+
+def node_info(node, ca_minion):
+    result = {
+        'roles': [],
+        'version': None,
+    }
+
+    roles = set()
+
+    if VERSION_LABEL in node.metadata.labels:
+        result['version'] = node.metadata.labels[VERSION_LABEL]
+
+    for (label, value) in node.metadata.labels.items():
+        if label.startswith(ROLE_LABEL_PREFIX):
+            role = label[len(ROLE_LABEL_PREFIX):]
+            if role:
+                roles.add(role)
+
+    if node.metadata.name == ca_minion:
+        roles.add('ca')
+
+    result['roles'] = list(roles)
+
+    return result
+
+
+def ext_pillar(minion_id, pillar, kubeconfig):
+    if not os.path.isfile(kubeconfig):
+        log.warning(
+            '%s: kubeconfig not found at %s', __virtualname__, kubeconfig)
+        return {}
+
+    ca_minion = None
+    if 'metalk8s' in pillar:
+        if 'ca' in pillar['metalk8s']:
+            ca_minion = pillar['metalk8s']['ca'].get('minion', None)
+
+    client = kubernetes.config.new_client_from_config(
+        config_file=kubeconfig,
+    )
+
+    coreV1 = kubernetes.client.CoreV1Api(api_client=client)
+
+    node_list = coreV1.list_node()
+
+    pillar_nodes = dict(
+        (node.metadata.name, node_info(node, ca_minion))
+        for node in node_list.items)
+
+    return {
+        'metalk8s': {
+            'nodes': pillar_nodes,
+            'node': pillar_nodes.get(minion_id, None),
+        },
+    }
diff --git a/salt/metalk8s/kubeadm/init/addons/files/coredns_deployment.yaml.j2 b/salt/metalk8s/kubeadm/init/addons/files/coredns_deployment.yaml.j2
@@ -23,7 +23,7 @@ spec:
       tolerations:
         - key: "CriticalAddonsOnly"
           operator: "Exists"
-        - key: "node-role.kubernetes.io/master"
+        - key: "node-role.kubernetes.io/bootstrap"
           operator: "Exists"
           effect: "NoSchedule"
       nodeSelector:

diff --git a/...dm/init/mark-control-plane/configured.sls → ...init/mark-control-plane/configured.sls.in b/...dm/init/mark-control-plane/configured.sls → ...init/mark-control-plane/configured.sls.in
@@ -3,23 +3,47 @@
 {% set kubeconfig = "/etc/kubernetes/admin.conf" %}
 {% set context = "kubernetes-admin@kubernetes" %}
 
-Apply control-plane role label:
+Apply control-plane master role label:
   kubernetes.node_label_present:
     - name: "node-role.kubernetes.io/master"
     - kubeconfig: {{ kubeconfig }}
     - context: {{ context }}
     - node: {{ hostname }}
     - value: ""
 
-Apply control-plane taints:
+Apply control-plane etcd role label:
+  kubernetes.node_label_present:
+    - name: "node-role.kubernetes.io/etcd"
+    - kubeconfig: {{ kubeconfig }}
+    - context: {{ context }}
+    - node: {{ hostname }}
+    - value: ""
+
+Apply control-plane bootstrap role label:
+  kubernetes.node_label_present:
+    - name: "node-role.kubernetes.io/bootstrap"
+    - kubeconfig: {{ kubeconfig }}
+    - context: {{ context }}
+    - node: {{ hostname }}
+    - value: ""
+
+Apply control-plane bootstrap taint:
   kubernetes.node_taints_present:
     - name: {{ hostname }}
     - kubeconfig: {{ kubeconfig }}
     - context: {{ context }}
     - taints:
-        - key: "node-role.kubernetes.io/master"
+        - key: "node-role.kubernetes.io/bootstrap"
           effect: "NoSchedule"
 
+Apply node version label:
+  kubernetes.node_label_present:
+    - kubeconfig: {{ kubeconfig }}
+    - context: {{ context }}
+    - name: "metalk8s.scality.com/version"
+    - node: {{ hostname }}
+    - value: "@@VERSION"
+
 Annotate with CRI socket information:
   kubernetes.node_annotation_present:
     - name: "kubeadm.alpha.kubernetes.io/cri-socket"

diff --git a/salt/metalk8s/roles/bootstrap.sls b/salt/metalk8s/roles/bootstrap.sls
@@ -0,0 +1,2 @@
+Deploy bootstrap:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/ca.sls b/salt/metalk8s/roles/ca.sls
@@ -0,0 +1,2 @@
+Deploy CA:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/etcd.sls b/salt/metalk8s/roles/etcd.sls
@@ -0,0 +1,2 @@
+Deploy etcd member:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/master.sls b/salt/metalk8s/roles/master.sls
@@ -0,0 +1,2 @@
+Deploy control plane node:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/minion.sls b/salt/metalk8s/roles/minion.sls
@@ -0,0 +1,2 @@
+Deploy Salt minion:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/node.sls b/salt/metalk8s/roles/node.sls
@@ -0,0 +1,2 @@
+Deploy workload plane node:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/registry.sls b/salt/metalk8s/roles/registry.sls
@@ -0,0 +1,2 @@
+Deploy container image registry:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/repository.sls b/salt/metalk8s/roles/repository.sls
@@ -0,0 +1,2 @@
+Deploy package repository:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/roles/salt-master.sls b/salt/metalk8s/roles/salt-master.sls
@@ -0,0 +1,2 @@
+Deploy Salt master:
+  test.succeed_without_changes
diff --git a/salt/metalk8s/salt/master/files/master_99-metalk8s.conf b/salt/metalk8s/salt/master/files/master_99-metalk8s.conf
@@ -1,8 +1,11 @@
 peer:
   .*:
     - x509.sign_remote_certificate
+
 ext_pillar:
   - metalk8s: /etc/metalk8s/bootstrap.yaml
+  - metalk8s_nodes: /etc/kubernetes/admin.conf
+
 roster_defaults:
   minion_opts:
     use_superseded:

diff --git a/salt/metalk8s/ui/files/metalk8s-ui_deployment.yaml b/salt/metalk8s/ui/files/metalk8s-ui_deployment.yaml
@@ -15,7 +15,7 @@ spec:
         k8s-app: ui
     spec:
       tolerations:
-      - key: "node-role.kubernetes.io/master"
+      - key: "node-role.kubernetes.io/bootstrap"
         operator: "Exists"
         effect: "NoSchedule"
       nodeSelector:

diff --git a/salt/top.sls.in b/salt/top.sls.in
@@ -0,0 +1,22 @@
+metalk8s-@@VERSION:
+  'metalk8s:node:version:@@VERSION':
+    - match: pillar
+    - metalk8s.roles.minion
+  'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:bootstrap':
+    - match: compound
+    - metalk8s.roles.bootstrap
+    - metalk8s.roles.salt-master
+    - metalk8s.roles.registry
+    - metalk8s.roles.repository
+  'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:ca':
+    - match: compound
+    - metalk8s.roles.ca
+  'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:etcd':
+    - match: compound
+    - metalk8s.roles.etcd
+  'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:master':
+    - match: compound
+    - metalk8s.roles.master
+  'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:node':
+    - match: compound
+    - metalk8s.roles.node
diff --git a/tests/post/steps/files/busybox.yaml b/tests/post/steps/files/busybox.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
 spec:
   tolerations:
-  - key: "node-role.kubernetes.io/master"
+  - key: "node-role.kubernetes.io/bootstrap"
     operator: "Equal"
     effect: "NoSchedule"
   containers: