From 86c73e32f8f2c8b2b651bd71a2855ff116c27c83 Mon Sep 17 00:00:00 2001
From: Zhiyuan He <362583303@qq.com>
Date: Thu, 17 Dec 2020 17:08:51 +0800
Subject: [PATCH] [Multiple Hardwares] [Installation] Check if resource matches
 layout.yaml (#5181)

* zhiyuhe/1216

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

fix

* fix

* fix
---
 contrib/kubespray/config/config.yaml          |   6 -
 contrib/kubespray/config/layout.yaml          |   7 +-
 contrib/kubespray/environment-check.yml       |  26 ++
 contrib/kubespray/example/config.yml          |  84 ----
 contrib/kubespray/example/openpai.yml         | 362 ------------------
 .../quick-start/pre-check.yml.template        |  21 +
 .../basic-resource/defaults/main.yml          |   0
 .../requirement/basic-resource/tasks/main.yml |  12 +
 .../amd.com_gpu/defaults/main.yml             |   0
 .../amd.com_gpu/tasks/main.yml                |   0
 .../nvidia.com_gpu/defaults/main.yml          |   0
 .../nvidia.com_gpu/tasks/main.yml             | 106 +++++
 .../requirement/worker/defaults/main.yml      |   3 -
 .../requirement/worker/tasks/resource.yml     |   2 +-
 .../worker/tasks/ubuntu-docker.yml            |  33 --
 .../worker/tasks/ubuntu-nvidia.yml            |  51 ---
 .../roles/requirement/worker/tasks/ubuntu.yml |   4 -
 contrib/kubespray/script/environment.sh       |   3 +
 .../kubespray/script/pre-check-generator.py   |  28 +-
 19 files changed, 199 insertions(+), 549 deletions(-)
 delete mode 100644 contrib/kubespray/example/config.yml
 delete mode 100644 contrib/kubespray/example/openpai.yml
 create mode 100644 contrib/kubespray/roles/requirement/basic-resource/defaults/main.yml
 create mode 100644 contrib/kubespray/roles/requirement/basic-resource/tasks/main.yml
 create mode 100644 contrib/kubespray/roles/requirement/computing-devices/amd.com_gpu/defaults/main.yml
 create mode 100644 contrib/kubespray/roles/requirement/computing-devices/amd.com_gpu/tasks/main.yml
 create mode 100644 contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/defaults/main.yml
 create mode 100644 contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/tasks/main.yml
 delete mode 100644 contrib/kubespray/roles/requirement/worker/tasks/ubuntu-nvidia.yml

diff --git a/contrib/kubespray/config/config.yaml b/contrib/kubespray/config/config.yaml
index 3647b306e5..0c86b3cae1 100644
--- a/contrib/kubespray/config/config.yaml
+++ b/contrib/kubespray/config/config.yaml
@@ -28,16 +28,10 @@ docker_image_tag: v1.5.0
 
 ###########################################################################################
 #                         Pre-check setting                                               #
-# By default, we assume your gpu environment is nvidia. So your runtime should be nvidia. #
-# If you are using AMD or other environment, you should modify it.                        #
 ###########################################################################################
-# worker_default_docker_runtime: nvidia
 # docker_check: true
-
 # resource_check: true
 
-# gpu_type: nvidia
-
 ########################################################################################
 # Advanced docker configuration. If you are not familiar with them, don't change them. #
 ########################################################################################
diff --git a/contrib/kubespray/config/layout.yaml b/contrib/kubespray/config/layout.yaml
index eaa1fc2814..cd72cc2181 100644
--- a/contrib/kubespray/config/layout.yaml
+++ b/contrib/kubespray/config/layout.yaml
@@ -4,15 +4,18 @@
 machine-sku:
   master-machine: # define a machine sku
     # the resource requirements for all the machines of this sku
-    mem: 60GB
+    # We use the same memory format as Kubernetes, e.g. Gi, Mi
+    # Reference: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
+    mem: 60Gi
     cpu:
+      # the number of CPU vcores
       vcore: 24
   gpu-machine:
     computing-device:
       type: nvidia.com/gpu
       model: K80
       count: 4
-    mem: 220GB
+    mem: 220Gi
     cpu:
       vcore: 24
 
diff --git a/contrib/kubespray/environment-check.yml b/contrib/kubespray/environment-check.yml
index 7d5e7f00ce..9f1c3d7523 100644
--- a/contrib/kubespray/environment-check.yml
+++ b/contrib/kubespray/environment-check.yml
@@ -43,6 +43,32 @@
   roles:
     - { role: requirement/worker }
 
+# basic-resource role checks whether the host's basic resource meets the user input in layout.yaml
+# For now, we only check CPU and memory in this role. 
+- hosts: all
+  become: true
+  become_user: root
+  gather_facts: true
+  roles:
+    - { role: requirement/basic-resource }
+
+# In the following, we do different checks according to different kinds of computing device.
+# To add a new kind of computing device, please add it as an ansible role in "requirement/computing-devices",
+# and include the role here.
+- hosts: "nvidia.com/gpu"
+  become: true
+  become_user: root
+  gather_facts: true
+  roles:
+    - { role: requirement/computing-devices/nvidia.com_gpu }
+
+- hosts: "amd.com/gpu"
+  become: true
+  become_user: root
+  gather_facts: true
+  roles:
+    - { role: requirement/computing-devices/amd.com_gpu }
+
 - hosts: all,localhost
   tasks:
     - name: "display unmet requirements"
diff --git a/contrib/kubespray/example/config.yml b/contrib/kubespray/example/config.yml
deleted file mode 100644
index b2d42e9b57..0000000000
--- a/contrib/kubespray/example/config.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-user: forexample
-password: forexample
-branch_name: <% latest-release %>
-docker_image_tag: <% latest-release %>
-
-# Optional
-
-#############################################
-# Ansible-playbooks' inventory hosts' vars. #
-#############################################
-# ssh_key_file_path: /path/to/you/key/file
-
-#####################################
-# OpenPAI's service image registry. #
-#####################################
-# docker_registry_domain: docker.io
-# docker_registry_namespace: openpai
-# docker_registry_username: exampleuser
-# docker_registry_password: examplepasswd
-
-################################################################
-# OpenPAI's daemon qos config.                                 #
-# By default, the QoS class for PAI daemon is BestEffort.      #
-# If you want to promote QoS class to Burstable or Guaranteed, #
-# you should set the value to true.                            #
-################################################################
-# qos-switch: "false"
-
-###########################################################################################
-#                         Pre-check setting                                               #
-# By default, we assume your gpu environment is nvidia. So your runtime should be nvidia. #
-# If you are using AMD or other environment, you should modify it.                        #
-###########################################################################################
-# worker_default_docker_runtime: nvidia
-# docker_check: true
-
-# resource_check: true
-
-# gpu_type: nvidia
-
-########################################################################################
-# Advanced docker configuration. If you are not familiar with them, don't change them. #
-########################################################################################
-# docker_data_root: /mnt/docker
-# docker_config_file_path: /etc/docker/daemon.json
-# docker_iptables_enabled: false
-
-## An obvious use case is allowing insecure-registry access to self hosted registries.
-## Can be ipaddress and domain_name.
-## example define 172.19.16.11 or mirror.registry.io
-# openpai_docker_insecure_registries:
-#   - mirror.registry.io
-#   - 172.19.16.11
-
-## Add other registry,example China registry mirror.
-# openpai_docker_registry_mirrors:
-#   - https://registry.docker-cn.com
-#   - https://mirror.aliyuncs.com
-
-#######################################################################
-#                       kubespray setting                             #
-#######################################################################
-
-# If you couldn't access to gcr.io or docker.io, please configure it.
-# gcr_image_repo: "gcr.io"
-# kube_image_repo: "gcr.io/google-containers"
-# quay_image_repo: "quay.io"
-# docker_image_repo: "docker.io"
-# etcd_image_repo: "quay.io/coreos/etcd"
-# pod_infra_image_repo: "gcr.io/google_containers/pause-{{ image_arch }}"
-# kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/hyperkube"
-
-# openpai_kube_network_plugin: calico
-
-# openpai_kubespray_extra_var:
-#   kay: value
-#   key: value
-
-#######################################################################
-#                     host daemon port setting                        #
-#######################################################################
-# host_daemon_port_start: 40000
-# host_daemon_port_end: 65535
diff --git a/contrib/kubespray/example/openpai.yml b/contrib/kubespray/example/openpai.yml
deleted file mode 100644
index c940718d75..0000000000
--- a/contrib/kubespray/example/openpai.yml
+++ /dev/null
@@ -1,362 +0,0 @@
-############## etcd configuration ##############
-
-# etcd_deployment_type: docker
-
-# etcd_version: v3.3.10
-
-## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
-# etcd_metrics: basic
-
-## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
-## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
-etcd_memory_limit: "0"
-
-## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
-## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
-## etcd documentation for more information.
-## https://etcd.io/docs/v3.4.0/op-guide/configuration/
-etcd_quota_backend_bytes: "8589934592"
-
-############## nginx configuration ##############
-
-# Requests for load balancer app
-loadbalancer_apiserver_memory_requests: 500M
-loadbalancer_apiserver_cpu_requests: 500m
-
-loadbalancer_apiserver_keepalive_timeout: 15m
-
-## Internal loadbalancers for apiservers
-# loadbalancer_apiserver_localhost: true
-
-## valid options are "nginx" or "haproxy"
-loadbalancer_apiserver_type: "nginx"
-## applied if only external loadbalancer_apiserver is defined, otherwise ignored
-# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
-
-## Local loadbalancer should use this port
-## And must be set port 6443
-# loadbalancer_apiserver_port: 6443
-
-## If loadbalancer_apiserver_healthcheck_port variable defined, enables proxy liveness check for nginx.
-# loadbalancer_apiserver_healthcheck_port: 8081
-
-
-############## DNS configuration ##############
-## Kubernetes cluster name, also will be used as DNS domain
-# cluster_name: cluster.local
-
-## Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
-# ndots: 2
-
-## Can be coredns, coredns_dual, manual or none
-# dns_mode: coredns
-
-## Set manual server if using a custom cluster DNS server
-## manual_dns_server: 10.x.x.x
-## Enable nodelocal dns cache
-enable_nodelocaldns: false
-# nodelocaldns_ip: 169.254.25.10
-# nodelocaldns_health_port: 9254
-
-## Enable k8s_external plugin for CoreDNS
-# enable_coredns_k8s_external: false
-# coredns_k8s_external_zone: k8s_external.local
-
-## Enable endpoint_pod_names option for kubernetes plugin
-# enable_coredns_k8s_endpoint_pod_names: false
-
-## Can be docker_dns, host_resolvconf or none
-# resolvconf_mode: docker_dns
-
-## Ip address of the kubernetes skydns service
-# skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
-# skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
-# dns_domain: "{{ cluster_name }}"
-
-# Limits for coredns
-dns_memory_limit: 1000Mi
-dns_cpu_requests: 0m
-dns_memory_requests: 0Mi
-dns_min_replicas: 1
-# dns_nodes_per_replica: 16
-# dns_cores_per_replica: 256
-# dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'false' }}"
-# coredns_ordinal_suffix: ""
-
-# nodelocaldns
-# nodelocaldns_cpu_requests: 100m
-# nodelocaldns_memory_limit: 170Mi
-# nodelocaldnsdns_memory_requests: 70Mi
-
-# Netchecker
-## Deploy netchecker app to verify DNS resolve as an HTTP service
-deploy_netchecker: false
-# netchecker_port: 31081
-# agent_report_interval: 15
-# netcheck_namespace: default
-
-# Limits for netchecker apps
-# netchecker_agent_cpu_limit: 30m
-# netchecker_agent_memory_limit: 100M
-# netchecker_agent_cpu_requests: 15m
-# netchecker_agent_memory_requests: 64M
-# netchecker_server_cpu_limit: 100m
-# netchecker_server_memory_limit: 256M
-# netchecker_server_cpu_requests: 50m
-# netchecker_server_memory_requests: 64M
-
-# SecurityContext when PodSecurityPolicy is enabled
-# netchecker_agent_user: 1000
-# netchecker_server_user: 1000
-# netchecker_agent_group: 1000
-# netchecker_server_group: 1000
-
-
-############## kubernetes configuration ##############
-
-## Change this to use another Kubernetes version, e.g. a current beta release
-# kube_version: v1.15.3
-
-## kubernetes image repo define
-# kube_image_repo: "gcr.io/google-containers"
-
-## Cluster Loglevel configuration
-kube_log_level: 2
-
-#kube_token_auth: true
-#kube_basic_auth: true
-
-## Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
-kubeconfig_localhost: true
-## Download kubectl onto the host that runs Ansible in {{ bin_dir }}
-kubectl_localhost: true
-
-# kubelet_status_update_frequency: 10s
-
-## For some things, kubelet needs to load kernel modules.  For example,
-## dynamic kernel services are needed for mounting persistent volumes into containers.  These may not be
-## loaded by preinstall kubernetes processes.  For example, ceph and rbd backed volumes.  Set this variable to
-## true to let kubelet load kernel modules.
-# kubelet_load_modules: false
-
-## Configure the amount of pods able to run on single node
-## default is equal to application default
-# kubelet_max_pods: 110
-
-## Support custom flags to be passed to kubelet
-kubelet_custom_flags:
-  - "--image-pull-progress-deadline=10m"
-
-# kube_feature_gates: []
-
-## Support custom flags to be passed to kubelet only on nodes, not masters
-kubelet_node_custom_flags:
-  - "--node-labels=pai-worker=true"
-
-k8s_image_pull_policy: Always
-
-## extra runtime config
-# kube_api_runtime_config: []
-
-##### networking
-
-# Choose network plugin (cilium, calico, contiv, weave or flannel. Use cni for generic cni plugin)
-# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
-kube_network_plugin: calico
-
-
-# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
-# kube_network_plugin_multus: false
-
-## Scale: 4096 nodes, 100 pods per node
-
-## Kubernetes internal network for services, unused block of space.
-kube_service_addresses: 10.192.0.0/13
-
-## internal network. When used, it will assign IP
-## addresses from this range to individual pods.
-## This network must be unused in your network infrastructure!
-kube_pods_subnet: 10.200.0.0/13
-
-## internal network node size allocation (optional). This is the size allocated
-## to each node on your network.  With these defaults you should have
-## room for 64 nodes with 254 pods per node.
-## Example: Up to 256 nodes, 100 pods per node (/16 network):
-##  - kube_service_addresses: 10.233.0.0/17
-##  - kube_pods_subnet: 10.233.128.0/17
-##  - kube_network_node_prefix: 25
-## Example: Up to 4096 nodes, 100 pods per node (/12 network):
-##  - kube_service_addresses: 10.192.0.0/13
-##  - kube_pods_subnet: 10.200.0.0/13
-##  - kube_network_node_prefix: 25
-kube_network_node_prefix: 25
-
-##### api-server
-
-kube_kubeadm_apiserver_extra_args:
-  cors-allowed-origins: ".*"
-  storage-media-type: "application/json"
-  max-requests-inflight: 1500
-  max-mutating-requests-inflight: 500
-  v: 2
-
-## Extra control plane host volume mounts
-## Example:
-## apiserver_extra_volumes:
-##  - name: name
-##    hostPath: /host/path
-##    mountPath: /mount/path
-##    readOnly: true
-# apiserver_extra_volumes: {}
-
-## ETCD backend for k8s data
-# kube_apiserver_storage_backend: etcd3
-
-## change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
-# kube_apiserver_insecure_bind_address: 127.0.0.1
-
-## By default the external API listens on all interfaces, this can be changed to
-## listen on a specific address/interface.
-# kube_apiserver_bind_address: 0.0.0.0
-
-## A port range to reserve for services with NodePort visibility.
-## Inclusive at both ends of the range.
-# kube_apiserver_node_port_range: "30000-32767"
-
-kube_apiserver_memory_limit: 20000M
-kube_apiserver_cpu_limit: 10000m
-kube_apiserver_memory_requests: 1024M
-kube_apiserver_cpu_requests: 1000m
-# kube_apiserver_request_timeout: "1m0s"
-
-# 1.9 and below Admission control plug-ins
-#kube_apiserver_admission_control:
-#  - NamespaceLifecycle
-#  - LimitRanger
-#  - ServiceAccount
-#  - DefaultStorageClass
-#  - PersistentVolumeClaimResize
-#  - MutatingAdmissionWebhook
-#  - ValidatingAdmissionWebhook
-#  - ResourceQuota
-#  - DefaultTolerationSeconds
-#  - DenyEscalatingExec
-
-## 1.10+ admission plugins
-# kube_apiserver_enable_admission_plugins: []
-
-## 1.10+ list of disabled admission plugins
-# kube_apiserver_disable_admission_plugins: []
-
-##### controller
-
-kube_kubeadm_controller_extra_args:
-  v: 2
-  kube-api-qps: 200
-  kube-api-burst: 300
-  concurrent-gc-syncs: 500
-  large-cluster-size-threshold: 0
-  secondary-node-eviction-rate: 0.05
-
-# controller_manager_extra_volumes: {}
-
-# kube_controller_manager_bind_address: 0.0.0.0
-
-kube_controller_memory_limit: 20000M
-kube_controller_cpu_limit: 10000m
-kube_controller_memory_requests: 1024M
-kube_controller_cpu_requests: 1000m
-# kube_controller_node_monitor_grace_period: 40s
-# kube_controller_node_monitor_period: 5s
-# kube_controller_pod_eviction_timeout: 5m0s
-# kube_controller_terminated_pod_gc_threshold: 12500
-
-##### scheduler
-
-kube_kubeadm_scheduler_extra_args:
-  v: 2
-
-# scheduler_extra_volumes: {}
-
-# kube_scheduler_bind_address: 0.0.0.0
-
-kube_scheduler_memory_limit: 200000M
-kube_scheduler_cpu_limit: 10000m
-kube_scheduler_memory_requests: 1024M
-kube_scheduler_cpu_requests: 1000m
-
-
-##### dashboard
-
-dashboard_enabled: false
-# dashboard_replicas: 1
-
-# Limits for dashboard
-# dashboard_cpu_limit: 100m
-# dashboard_memory_limit: 256M
-# dashboard_cpu_requests: 50m
-# dashboard_memory_requests: 64M
-
-# Set dashboard_use_custom_certs to true if overriding dashboard_certs_secret_name with a secret that
-# contains dashboard_tls_key_file and dashboard_tls_cert_file instead of using the initContainer provisioned certs
-# dashboard_use_custom_certs: false
-# dashboard_certs_secret_name: kubernetes-dashboard-certs
-# dashboard_tls_key_file: dashboard.key
-# dashboard_tls_cert_file: dashboard.crt
-# dashboard_master_toleration: true
-
-# Override dashboard default settings
-# dashboard_token_ttl: 900
-# dashboard_skip_login: false
-
-
-############## other configuration ##############
-
-## Optionally reserve resources for OS system daemons.
-# system_reserved: true
-## Uncomment to override default values
-# system_memory_reserved: 3072M
-# system_cpu_reserved: 500m
-## Reservation for master hosts
-# system_master_memory_reserved: 256M
-# system_master_cpu_reserved: 250m
-
-############## addon configuration ##############
-
-# Helm deployment
-# helm_enabled: true
-
-# Cert manager deployment
-# cert_manager_enabled: true
-# cert_manager_namespace: "cert-manager"
-
-############## docker configuration ##############
-
-# docker_version: latest
-
-docker_dns_servers_strict: false
-
-## Used to set docker daemon iptables options to true
-docker_iptables_enabled: false
-
-docker_log_opts: "--log-opt max-size=2g --log-opt max-file=2 --log-driver=json-file"
-
-docker_daemon_graph: "/mnt/docker"
-
-docker_options: >-
-    {%- if docker_insecure_registries is defined %}
-    {{ docker_insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }}
-    {%- endif %}
-    {% if docker_registry_mirrors is defined %}
-    {{ docker_registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }}
-    {%- endif %}
-    {%- if docker_version != "latest" and docker_version is version('17.05', '<') %}
-    --graph={{ docker_daemon_graph }} {% if ansible_os_family not in ["openSUSE Leap", "openSUSE Tumbleweed", "Suse"] %}{{ docker_log_opts }}{% endif %}
-    {%- else %}
-    --data-root={{ docker_daemon_graph }} {% if ansible_os_family not in ["openSUSE Leap", "openSUSE Tumbleweed", "Suse"] %}{{ docker_log_opts }}{% endif %}
-    {%- endif %}
-    {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %}
-    --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current
-    --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd
-    --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false
-    {%- endif -%}
\ No newline at end of file
diff --git a/contrib/kubespray/quick-start/pre-check.yml.template b/contrib/kubespray/quick-start/pre-check.yml.template
index c10b461b92..1718bd3904 100644
--- a/contrib/kubespray/quick-start/pre-check.yml.template
+++ b/contrib/kubespray/quick-start/pre-check.yml.template
@@ -8,6 +8,13 @@ all:
       ansible_ssh_user: {{ env["cfg"]['user'] }}
       ansible_ssh_pass: "{{ env["cfg"]['password'] }}"
       ansible_become_pass: "{{ env["cfg"]['password'] }}"
+      memory_mb: {{ vm['memory_mb'] }}
+      cpu_vcores: {{ vm['cpu_vcores'] }}
+      {%- if "computing_device" in vm %}
+      computing_device_type: "{{ vm['computing_device']['type'] }}"
+      computing_device_model: "{{ vm['computing_device']['model'] }}"
+      computing_device_count: {{ vm['computing_device']['count'] }}
+      {%- endif %}
 {%- if "ssh_key_file_path" in env["cfg"]%}
       ansible_ssh_private_key_file: "{{ env["cfg"]["ssh_key_file_path"] }}"
 {%- endif %}
@@ -21,6 +28,13 @@ all:
       ansible_ssh_user: {{ env["cfg"]['user'] }}
       ansible_ssh_pass: "{{ env["cfg"]['password'] }}"
       ansible_become_pass: "{{ env["cfg"]['password'] }}"
+      memory_mb: {{ vm['memory_mb'] }}
+      cpu_vcores: {{ vm['cpu_vcores'] }}
+      {%- if "computing_device" in vm %}
+      computing_device_type: "{{ vm['computing_device']['type'] }}"
+      computing_device_model: "{{ vm['computing_device']['model'] }}"
+      computing_device_count: {{ vm['computing_device']['count'] }}
+      {%- endif %}
 {%- if "ssh_key_file_path" in env["cfg"]%}
       ansible_ssh_private_key_file: "{{ env["cfg"]["ssh_key_file_path"] }}"
 {%- endif %}
@@ -37,3 +51,10 @@ all:
 {%- for vm in env["workers"] %}
         {{ vm["hostname"] }}:
 {%- endfor %}
+{%- for device_group_name, hostnames in env["computing_device_groups"].items() %}
+    {{ device_group_name }}:
+      hosts:
+        {%- for hostname in hostnames %}
+        {{ hostname }}:
+        {%- endfor %}
+{%- endfor %}
diff --git a/contrib/kubespray/roles/requirement/basic-resource/defaults/main.yml b/contrib/kubespray/roles/requirement/basic-resource/defaults/main.yml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/kubespray/roles/requirement/basic-resource/tasks/main.yml b/contrib/kubespray/roles/requirement/basic-resource/tasks/main.yml
new file mode 100644
index 0000000000..e91c20ea7d
--- /dev/null
+++ b/contrib/kubespray/roles/requirement/basic-resource/tasks/main.yml
@@ -0,0 +1,12 @@
+---
+- name: Check memory resource meets the memory specified by user
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + [\"In layout.yaml, the memory specified for this host is {{ memory_mb }}MB, but the host only have {{ ansible_memory_mb['real']['total'] }}MB in total\"] }}"
+  when:
+    - ansible_memory_mb['real']['total'] < memory_mb
+
+- name: Check cpu resource meets the cpu specified by user
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + [\"In layout.yaml, the CPU number specified for this host is {{ cpu_vcores }}, but the host only have {{ ansible_processor_vcpus }} in total\"] }}"
+  when:
+    - ansible_processor_vcpus < cpu_vcores
\ No newline at end of file
diff --git a/contrib/kubespray/roles/requirement/computing-devices/amd.com_gpu/defaults/main.yml b/contrib/kubespray/roles/requirement/computing-devices/amd.com_gpu/defaults/main.yml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/kubespray/roles/requirement/computing-devices/amd.com_gpu/tasks/main.yml b/contrib/kubespray/roles/requirement/computing-devices/amd.com_gpu/tasks/main.yml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/defaults/main.yml b/contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/defaults/main.yml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/tasks/main.yml b/contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/tasks/main.yml
new file mode 100644
index 0000000000..4a21802bba
--- /dev/null
+++ b/contrib/kubespray/roles/requirement/computing-devices/nvidia.com_gpu/tasks/main.yml
@@ -0,0 +1,106 @@
+---
+- name: "Check GPU card info with lspci"
+  raw: "lspci | grep -E '[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F].[0-9] (3D|VGA compatible) controller: NVIDIA Corporation.*'"
+  register: lspci_grep
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  environment: {}
+
+- name: "Check NVIDIA GPU exits or not"
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + ['NVIDIA GPU card is not detected'] }}"
+  changed_when: false
+  check_mode: false
+  environment: {}
+  when:
+    - lspci_grep.rc != 0
+
+- name: "Check NVIDIA GPU drivers with nvidia-smi"
+  raw: "nvidia-smi"
+  register: nvidia_smi
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  environment: {}
+
+- name: "Check NVIDIA GPU drivers is installed or not "
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + ['NVIDIA GPU drivers is not detected'] }}"
+  changed_when: false
+  check_mode: false
+  environment: {}
+  when:
+    - nvidia_smi.rc != 0
+
+- name: "Check NVIDIA GPU card number with nvidia-smi"
+  raw: "nvidia-smi --query-gpu=gpu_name --format=csv | tail --lines=+2 | wc -l"
+  register: nvidia_gpu_count
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  environment: {}
+  when:
+    # if nvidia-smi doesn't work, we can skip this step
+    - nvidia_smi.rc == 0 
+
+- name: "Check NVIDIA GPU card number is matched or not"
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + [\"NVIDIA GPU card number is not matched: {{ computing_device_count }} specified but only {{ nvidia_gpu_count.stdout_lines[0] }} found\"] }}"
+  changed_when: false
+  check_mode: false
+  environment: {}
+  when:
+    # if nvidia-smi doesn't work, we can skip this step
+    - nvidia_smi.rc == 0
+    - "nvidia_gpu_count.stdout_lines[0]|int != computing_device_count"
+    - nvidia_gpu_count.rc == 0
+
+- name: "Check NVIDIA container runtime with command"
+  raw: "command -v nvidia-container-runtime "
+  register: nvidia_container_runtime
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  environment: {}
+
+- name: "Check NVIDIA container runtime is installed or not"
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + ['NVIDIA container runtime is not detected'] }}"
+  changed_when: false
+  check_mode: false
+  environment: {}
+  when:
+    - nvidia_container_runtime.rc != 0
+
+- name: "Get default docker runtime"
+  raw: "{% raw %} docker info -f '{{json .DefaultRuntime}}'{% endraw %}"
+  register: default_runtime
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  environment: {}
+
+- name: "Init array"
+  set_fact:
+    default_runtime_processed: []
+
+- name: "Remove unnecessary char in the end of the output"
+  set_fact:
+    default_runtime_processed: "{{ default_runtime_processed + [ item | replace('\"','') ] }}"
+  loop: "{{ default_runtime.stdout_lines }}"
+
+- name: "Get Default Runtime"
+  set_fact:
+    default_runtime_fin: "{{ item }}"
+  when: item == 'nvidia'
+  loop: "{{default_runtime_processed}}"
+
+- name: "Check the default runtime is set correctly"
+  set_fact:
+    unmet_requirements: "{{ unmet_requirements + ['The default runtime is not set correctly'] }}"
+  changed_when: false
+  check_mode: false
+  environment: {}
+  when:
+    - default_runtime_fin is not defined
diff --git a/contrib/kubespray/roles/requirement/worker/defaults/main.yml b/contrib/kubespray/roles/requirement/worker/defaults/main.yml
index 63ee89e50a..82ada5a98c 100644
--- a/contrib/kubespray/roles/requirement/worker/defaults/main.yml
+++ b/contrib/kubespray/roles/requirement/worker/defaults/main.yml
@@ -1,10 +1,7 @@
 docker_check: true
-worker_default_docker_runtime: nvidia
 
 resource_check: true
 
-gpu_type: nvidia
-
 openpai_registry: docker.io
 
 gcr_image_repo: gcr.io
\ No newline at end of file
diff --git a/contrib/kubespray/roles/requirement/worker/tasks/resource.yml b/contrib/kubespray/roles/requirement/worker/tasks/resource.yml
index 0ee9a82df4..a6bbfda034 100644
--- a/contrib/kubespray/roles/requirement/worker/tasks/resource.yml
+++ b/contrib/kubespray/roles/requirement/worker/tasks/resource.yml
@@ -3,4 +3,4 @@
   set_fact:
     unmet_requirements: "{{ unmet_requirements + [\"OpenPAI's worker node should have 16 Gi free memory for service\"] }}"
   when:
-    - ansible_memory_mb["nocache"]["free"] < 16000
\ No newline at end of file
+    - ansible_memory_mb["nocache"]["free"] < 16000
diff --git a/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-docker.yml b/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-docker.yml
index c1f9d70168..d81d492277 100644
--- a/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-docker.yml
+++ b/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-docker.yml
@@ -15,36 +15,3 @@
   environment: {}
   when:
     - command_docker_worker.rc != 0
-
-- name: "Get default docker runtime"
-  raw: "{% raw %} docker info -f '{{json .DefaultRuntime}}'{% endraw %}"
-  register: default_runtime
-  failed_when: false
-  changed_when: false
-  check_mode: false
-  environment: {}
-
-- name: "Init array"
-  set_fact:
-    default_runtime_processed: []
-
-- name: "Remove unnecessary char in the end of the output"
-  set_fact:
-    default_runtime_processed: "{{ default_runtime_processed + [ item | replace('\"','') ] }}"
-  loop: "{{ default_runtime.stdout_lines }}"
-
-
-- name: "Get Default Runtime"
-  set_fact:
-    default_runtime_fin: "{{ item }}"
-  when: item == worker_default_docker_runtime
-  loop: "{{default_runtime_processed}}"
-
-- name: "Check the default runtime is set correctly"
-  set_fact:
-    unmet_requirements: "{{ unmet_requirements + ['The default runtime is not set correctly'] }}"
-  changed_when: false
-  check_mode: false
-  environment: {}
-  when:
-    - default_runtime_fin is not defined
\ No newline at end of file
diff --git a/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-nvidia.yml b/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-nvidia.yml
deleted file mode 100644
index 7e9d6ebc51..0000000000
--- a/contrib/kubespray/roles/requirement/worker/tasks/ubuntu-nvidia.yml
+++ /dev/null
@@ -1,51 +0,0 @@
----
-- name: "Check GPU card info with lspci"
-  raw: "lspci | grep -E '[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F].[0-9] (3D|VGA compatible) controller: NVIDIA Corporation.*'"
-  register: worker_lspci_grep
-  failed_when: false
-  changed_when: false
-  check_mode: false
-  environment: {}
-
-- name: "Check NVIDIA GPU exits or not"
-  set_fact:
-    unmet_requirements: "{{ unmet_requirements + ['NVIDIA GPU card is not detected in your worker machines'] }}"
-  changed_when: false
-  check_mode: false
-  environment: {}
-  when:
-    - worker_lspci_grep.rc != 0
-
-- name: "Check NVIDIA GPU drivers with nvidia-smi"
-  raw: "nvidia-smi"
-  register: worker_nvidia_smi
-  failed_when: false
-  changed_when: false
-  check_mode: false
-  environment: {}
-
-- name: "Check NVIDIA GPU drivers is installed or not "
-  set_fact:
-    unmet_requirements: "{{ unmet_requirements + ['NVIDIA GPU drivers is not detected in your worker machines'] }}"
-  changed_when: false
-  check_mode: false
-  environment: {}
-  when:
-    - worker_nvidia_smi.rc != 0
-
-- name: "Check NVIDIA container runtime with command"
-  raw: "command -v nvidia-container-runtime "
-  register: worker_nvidia_container_runtime
-  failed_when: false
-  changed_when: false
-  check_mode: false
-  environment: {}
-
-- name: "Check NVIDIA container runtime is installed or not"
-  set_fact:
-    unmet_requirements: "{{ unmet_requirements + ['NVIDIA container runtime is not detected in your worker machines'] }}"
-  changed_when: false
-  check_mode: false
-  environment: {}
-  when:
-    - worker_nvidia_container_runtime.rc != 0
diff --git a/contrib/kubespray/roles/requirement/worker/tasks/ubuntu.yml b/contrib/kubespray/roles/requirement/worker/tasks/ubuntu.yml
index d08f5cafb6..00032e5311 100644
--- a/contrib/kubespray/roles/requirement/worker/tasks/ubuntu.yml
+++ b/contrib/kubespray/roles/requirement/worker/tasks/ubuntu.yml
@@ -56,10 +56,6 @@
   when:
     - worker_nslookup_quay_io.rc != 0
 
-- name: check worker machines' NVIDIA gpu card, drivers and runtime
-  include_tasks: ubuntu-nvidia.yml
-  when: gpu_type == 'nvidia'
-
 - name: check worker machines' resource requirement
   include_tasks: resource.yml
   when: resource_check
diff --git a/contrib/kubespray/script/environment.sh b/contrib/kubespray/script/environment.sh
index 17a41d4abf..fa6840b9d8 100644
--- a/contrib/kubespray/script/environment.sh
+++ b/contrib/kubespray/script/environment.sh
@@ -36,6 +36,9 @@ sudo python3 get-pip.py
 echo "Install paramiko"
 sudo pip3 install paramiko
 
+echo "Install kubernetes package"
+sudo pip3 install kubernetes
+
 echo "Install sshpass"
 sudo apt-get -y install sshpass
 
diff --git a/contrib/kubespray/script/pre-check-generator.py b/contrib/kubespray/script/pre-check-generator.py
index 9ef5603939..bbfb191f31 100644
--- a/contrib/kubespray/script/pre-check-generator.py
+++ b/contrib/kubespray/script/pre-check-generator.py
@@ -4,7 +4,9 @@
 import logging.config
 import yaml
 import jinja2
-
+from kubernetes.utils import parse_quantity
+import math
+from collections import defaultdict
 
 def setup_logger_config(logger):
     """
@@ -53,7 +55,6 @@ def generate_template_file(template_file_path, output_path, map_table):
     generated_template = generate_from_template_dict(template, map_table)
     write_generated_file(output_path, generated_template)
 
-
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-l', '--layout', dest="layout", required=True,
@@ -73,11 +74,32 @@ def main():
     workers = list(filter(lambda elem: 'pai-worker' in elem and elem["pai-worker"] == 'true', layout['machine-list']))
     head_node = masters[0]
 
+    # fill in cpu, memory, computing_device information in both masters and workers
+    # we assume the layout file the user gives is correct
+    # TO DO: check layout file before this step
+    all_machines = masters + workers
+    for machine in all_machines:
+        sku_info = layout['machine-sku'][machine['machine-type']]
+        # use math.ceil to guarantee the memory volume 
+        # e.g. if use set 999.1MB, we ensure there is 1000MB to avoid scheduling issues
+        machine['memory_mb'] = math.ceil(parse_quantity(sku_info['mem']) / 1024 / 1024)
+        machine['cpu_vcores'] = sku_info['cpu']['vcore']
+        if 'computing-device' in sku_info:
+            machine['computing_device'] = sku_info['computing-device']
+
+    # add machine to different comupting device group
+    computing_device_groups = defaultdict(list)
+    for machine in all_machines:
+        sku_info = layout['machine-sku'][machine['machine-type']]
+        if 'computing-device' in sku_info:
+            computing_device_groups[sku_info['computing-device']['type']].append(machine['hostname'])
+
     environment = {
         'masters': masters,
         'workers': workers,
         'cfg': config,
-        'head_node': head_node
+        'head_node': head_node,
+        'computing_device_groups': computing_device_groups,
     }
 
     map_table = {