Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
735 lines (613 sloc) 22.2 KB
heat_template_version: 2014-10-16
description: >
This is a nested stack that defines a single Kubernetes master, This stack is
included by an ResourceGroup resource in the parent template
(kubecluster.yaml).
parameters:
name:
type: string
description: server name
server_image:
type: string
description: glance image used to boot the server
master_flavor:
type: string
description: flavor to use when booting the server
ssh_key_name:
type: string
description: name of ssh key to be provisioned on our server
external_network:
type: string
description: uuid/name of a network to use for floating ip addresses
portal_network_cidr:
type: string
description: >
address range used by kubernetes for service portals
kube_allow_priv:
type: string
description: >
whether or not kubernetes should permit privileged containers.
constraints:
- allowed_values: ["true", "false"]
etcd_volume_size:
type: number
description: >
size of a cinder volume to allocate for etcd storage
docker_volume_size:
type: number
description: >
size of a cinder volume to allocate to docker for container/image
storage
docker_volume_type:
type: string
description: >
type of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
default: "devicemapper"
cgroup_driver:
type: string
description: >
cgroup driver name that kubelet should use, ideally the same as
the docker cgroup driver.
default: "cgroupfs"
volume_driver:
type: string
description: volume driver to use for container storage
region_name:
type: string
description: A logically separate section of the cluster
flannel_network_cidr:
type: string
description: network range for flannel overlay network
flannel_network_subnetlen:
type: number
description: size of subnet assigned to each master
flannel_backend:
type: string
description: >
specify the backend for flannel, default udp backend
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
admission_control_list:
type: string
description: >
List of admission control plugins to activate
discovery_url:
type: string
description: >
Discovery URL used for bootstrapping the etcd cluster.
tls_disabled:
type: boolean
description: whether or not to enable TLS
kube_dashboard_enabled:
type: boolean
description: whether or not to disable kubernetes dashboard
influx_grafana_dashboard_enabled:
type: boolean
description: Enable influxdb with grafana dashboard for data from heapster
verify_ca:
type: boolean
description: whether or not to validate certificate authority
kubernetes_port:
type: number
description: >
The port which are used by kube-apiserver to provide Kubernetes
service.
cluster_uuid:
type: string
description: identifier for the cluster this template is generating
magnum_url:
type: string
description: endpoint to retrieve TLS certs from
prometheus_monitoring:
type: boolean
description: >
whether or not to have prometheus and grafana deployed
grafana_admin_passwd:
type: string
hidden: true
description: >
admin user password for the Grafana monitoring interface
api_public_address:
type: string
description: Public IP address of the Kubernetes master server.
default: ""
api_private_address:
type: string
description: Private IP address of the Kubernetes master server.
default: ""
fixed_network:
type: string
description: Network from which to allocate fixed addresses.
fixed_subnet:
type: string
description: Subnet from which to allocate fixed addresses.
network_driver:
type: string
description: network driver to use for instantiating container networks
secgroup_kube_master_id:
type: string
description: ID of the security group for kubernetes master.
api_pool_id:
type: string
description: ID of the load balancer pool of k8s API server.
etcd_pool_id:
type: string
description: ID of the load balancer pool of etcd server.
auth_url:
type: string
description: >
url for kubernetes to authenticate
username:
type: string
description: >
user account
password:
type: string
description: >
user password
http_proxy:
type: string
description: http proxy address for docker
https_proxy:
type: string
description: https proxy address for docker
no_proxy:
type: string
description: no proxies for docker
kube_tag:
type: string
description: tag of the k8s containers used to provision the kubernetes cluster
cloud_provider_tag:
type: string
description:
tag of the kubernetes/cloud-provider-openstack
https://hub.docker.com/r/k8scloudprovider/openstack-cloud-controller-manager/tags/
cloud_provider_enabled:
type: boolean
description: Enable or disable the openstack kubernetes cloud provider
etcd_tag:
type: string
description: tag of the etcd system container
flannel_tag:
type: string
description: tag of the flannel system containers
kube_version:
type: string
description: version of kubernetes used for kubernetes cluster
kube_dashboard_version:
type: string
description: version of kubernetes dashboard used for kubernetes cluster
trustee_user_id:
type: string
description: user id of the trustee
trustee_password:
type: string
description: password of the trustee
hidden: true
trust_id:
type: string
description: id of the trust which is used by the trustee
hidden: true
insecure_registry_url:
type: string
description: insecure registry url
container_infra_prefix:
type: string
description: >
prefix of container images used in the cluster, kubernetes components,
kubernetes-dashboard, coredns etc
etcd_lb_vip:
type: string
description: >
etcd lb vip private used to generate certs on master.
default: ""
dns_service_ip:
type: string
description: >
address used by Kubernetes DNS service
dns_cluster_domain:
type: string
description: >
domain name for cluster DNS
openstack_ca:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
availability_zone:
type: string
description: >
availability zone for master and nodes
default: ""
ca_key:
type: string
description: key of internal ca for the kube certificate api manager
hidden: true
cert_manager_api:
type: boolean
description: true if the kubernetes cert api manager should be enabled
default: false
calico_tag:
type: string
description: tag of the calico containers used to provision the calico node
calico_cni_tag:
type: string
description: tag of the cni used to provision the calico node
calico_kube_controllers_tag:
type: string
description: tag of the kube_controllers used to provision the calico node
calico_ipv4pool:
type: string
description: Configure the IP pool from which Pod IPs will be chosen
pods_network_cidr:
type: string
description: Configure the IP pool/range from which pod IPs will be chosen
ingress_controller:
type: string
description: >
ingress controller backend to use
ingress_controller_role:
type: string
description: >
node role where the ingress controller should run
kubelet_options:
type: string
description: >
additional options to be passed to the kubelet
kubeapi_options:
type: string
description: >
additional options to be passed to the api
kubecontroller_options:
type: string
description: >
additional options to be passed to the controller manager
kubeproxy_options:
type: string
description: >
additional options to be passed to the kube proxy
kubescheduler_options:
type: string
description: >
additional options to be passed to the scheduler
octavia_enabled:
type: boolean
description: >
whether or not to use Octavia for LoadBalancer type service.
default: False
kube_service_account_key:
type: string
hidden: true
description: >
The signed cert will be used to verify the k8s service account tokens
during authentication.
kube_service_account_private_key:
type: string
hidden: true
description: >
The private key will be used to sign generated k8s service account
tokens.
prometheus_tag:
type: string
description: tag of prometheus container
grafana_tag:
type: string
description: tag of grafana container
heat_container_agent_tag:
type: string
description: tag of the heat_container_agent system container
keystone_auth_enabled:
type: boolean
description: >
true if the keystone authN and authZ should be enabled
default:
false
k8s_keystone_auth_tag:
type: string
description: tag of the k8s_keystone_auth container
project_id:
type: string
description: >
project id of current project
resources:
######################################################################
#
# resource that exposes the IPs of either the kube master or the API
# LBaaS pool depending on whether LBaaS is enabled for the cluster.
#
api_address_switch:
type: Magnum::ApiGatewaySwitcher
properties:
pool_public_ip: {get_param: api_public_address}
pool_private_ip: {get_param: api_private_address}
master_public_ip: {get_attr: [kube_master_floating, floating_ip_address]}
master_private_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
######################################################################
#
# software configs. these are components that are combined into
# a multipart MIME user-data archive.
#
write_heat_params:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.yaml}
params:
"$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring}
"$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]}
"$KUBE_API_PRIVATE_ADDRESS": {get_attr: [api_address_switch, private_ip]}
"$KUBE_API_PORT": {get_param: kubernetes_port}
"$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
"$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$ETCD_VOLUME": {get_resource: etcd_volume}
"$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size}
"$DOCKER_VOLUME": {get_resource: docker_volume}
"$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size}
"$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
"$CGROUP_DRIVER": {get_param: cgroup_driver}
"$NETWORK_DRIVER": {get_param: network_driver}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_BACKEND": {get_param: flannel_backend}
"$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay}
"$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout}
"$PODS_NETWORK_CIDR": {get_param: pods_network_cidr}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$ADMISSION_CONTROL_LIST": {get_param: admission_control_list}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
"$AUTH_URL": {get_param: auth_url}
"$USERNAME": {get_param: username}
"$PASSWORD": {get_param: password}
"$CLUSTER_SUBNET": {get_param: fixed_subnet}
"$TLS_DISABLED": {get_param: tls_disabled}
"$KUBE_DASHBOARD_ENABLED": {get_param: kube_dashboard_enabled}
"$INFLUX_GRAFANA_DASHBOARD_ENABLED": {get_param: influx_grafana_dashboard_enabled}
"$VERIFY_CA": {get_param: verify_ca}
"$CLUSTER_UUID": {get_param: cluster_uuid}
"$MAGNUM_URL": {get_param: magnum_url}
"$VOLUME_DRIVER": {get_param: volume_driver}
"$REGION_NAME": {get_param: region_name}
"$HTTP_PROXY": {get_param: http_proxy}
"$HTTPS_PROXY": {get_param: https_proxy}
"$NO_PROXY": {get_param: no_proxy}
"$KUBE_TAG": {get_param: kube_tag}
"$CLOUD_PROVIDER_TAG": {get_param: cloud_provider_tag}
"$CLOUD_PROVIDER_ENABLED": {get_param: cloud_provider_enabled}
"$ETCD_TAG": {get_param: etcd_tag}
"$FLANNEL_TAG": {get_param: flannel_tag}
"$KUBE_VERSION": {get_param: kube_version}
"$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version}
"$TRUSTEE_USER_ID": {get_param: trustee_user_id}
"$TRUSTEE_PASSWORD": {get_param: trustee_password}
"$TRUST_ID": {get_param: trust_id}
"$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
"$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix}
"$ETCD_LB_VIP": {get_param: etcd_lb_vip}
"$DNS_SERVICE_IP": {get_param: dns_service_ip}
"$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
"$CERT_MANAGER_API": {get_param: cert_manager_api}
"$CA_KEY": {get_param: ca_key}
"$CALICO_TAG": {get_param: calico_tag}
"$CALICO_CNI_TAG": {get_param: calico_cni_tag}
"$CALICO_KUBE_CONTROLLERS_TAG": {get_param: calico_kube_controllers_tag}
"$CALICO_IPV4POOL": {get_param: calico_ipv4pool}
"$INGRESS_CONTROLLER": {get_param: ingress_controller}
"$INGRESS_CONTROLLER_ROLE": {get_param: ingress_controller_role}
"$KUBELET_OPTIONS": {get_param: kubelet_options}
"$KUBEAPI_OPTIONS": {get_param: kubeapi_options}
"$KUBECONTROLLER_OPTIONS": {get_param: kubecontroller_options}
"$KUBEPROXY_OPTIONS": {get_param: kubeproxy_options}
"$KUBESCHEDULER_OPTIONS": {get_param: kubescheduler_options}
"$OCTAVIA_ENABLED": {get_param: octavia_enabled}
"$KUBE_SERVICE_ACCOUNT_KEY": {get_param: kube_service_account_key}
"$KUBE_SERVICE_ACCOUNT_PRIVATE_KEY": {get_param: kube_service_account_private_key}
"$PROMETHEUS_TAG": {get_param: prometheus_tag}
"$GRAFANA_TAG": {get_param: grafana_tag}
"$HEAT_CONTAINER_AGENT_TAG": {get_param: heat_container_agent_tag}
"$KEYSTONE_AUTH_ENABLED": {get_param: keystone_auth_enabled}
"$K8S_KEYSTONE_AUTH_TAG": {get_param: k8s_keystone_auth_tag}
"$PROJECT_ID": {get_param: project_id}
install_openstack_ca:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
params:
$OPENSTACK_CA: {get_param: openstack_ca}
template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh}
make_cert:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/make-cert.sh}
configure_docker_storage:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config:
str_replace:
params:
$configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh}
template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh}
configure_etcd:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh}
write_kube_os_config:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh}
configure_kubernetes:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-master.sh}
write_flannel_config:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/write-flannel-config.sh}
flannel_config_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/flannel-config-service.sh}
flannel_service:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh}
enable_services:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/enable-services-master.sh}
disable_selinux:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh}
add_proxy:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh}
start_container_agent:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
kube_master_init:
type: OS::Heat::MultipartMime
properties:
parts:
- config: {get_resource: install_openstack_ca}
- config: {get_resource: disable_selinux}
- config: {get_resource: write_heat_params}
- config: {get_resource: configure_etcd}
- config: {get_resource: write_kube_os_config}
- config: {get_resource: configure_docker_storage}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: make_cert}
- config: {get_resource: add_proxy}
- config: {get_resource: start_container_agent}
- config: {get_resource: enable_services}
- config: {get_resource: write_flannel_config}
- config: {get_resource: flannel_config_service}
- config: {get_resource: flannel_service}
######################################################################
#
# a single kubernetes master.
#
# do NOT use "_" (underscore) in the Nova server name
# it creates a mismatch between the generated Nova name and its hostname
# which can lead to weird problems
kube-master:
type: OS::Nova::Server
properties:
name: {get_param: name}
image: {get_param: server_image}
flavor: {get_param: master_flavor}
key_name: {get_param: ssh_key_name}
user_data_format: SOFTWARE_CONFIG
software_config_transport: POLL_SERVER_HEAT
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
availability_zone: {get_param: availability_zone}
kube_master_eth0:
type: OS::Neutron::Port
properties:
network: {get_param: fixed_network}
security_groups:
- {get_param: secgroup_kube_master_id}
fixed_ips:
- subnet: {get_param: fixed_subnet}
allowed_address_pairs:
- ip_address: {get_param: pods_network_cidr}
replacement_policy: AUTO
kube_master_floating:
type: Magnum::Optional::KubeMaster::Neutron::FloatingIP
properties:
floating_network: {get_param: external_network}
port_id: {get_resource: kube_master_eth0}
api_pool_member:
type: Magnum::Optional::Neutron::LBaaS::PoolMember
properties:
pool: {get_param: api_pool_id}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
subnet: { get_param: fixed_subnet }
protocol_port: {get_param: kubernetes_port}
etcd_pool_member:
type: Magnum::Optional::Neutron::LBaaS::PoolMember
properties:
pool: {get_param: etcd_pool_id}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
subnet: { get_param: fixed_subnet }
protocol_port: 2379
######################################################################
#
# etcd storage. This allocates a cinder volume and attaches it
# to the master.
#
etcd_volume:
type: Magnum::Optional::Etcd::Volume
properties:
size: {get_param: etcd_volume_size}
etcd_volume_attach:
type: Magnum::Optional::Etcd::VolumeAttachment
properties:
instance_uuid: {get_resource: kube-master}
volume_id: {get_resource: etcd_volume}
mountpoint: /dev/vdc
######################################################################
#
# docker storage. This allocates a cinder volume and attaches it
# to the minion.
#
docker_volume:
type: Magnum::Optional::Cinder::Volume
properties:
size: {get_param: docker_volume_size}
volume_type: {get_param: docker_volume_type}
docker_volume_attach:
type: Magnum::Optional::Cinder::VolumeAttachment
properties:
instance_uuid: {get_resource: kube-master}
volume_id: {get_resource: docker_volume}
mountpoint: /dev/vdb
outputs:
OS::stack_id:
value: { get_resource: kube-master }
kube_master_ip:
value: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
description: >
This is the "private" IP address of the Kubernetes master node.
kube_master_external_ip:
value: {get_attr: [kube_master_floating, floating_ip_address]}
description: >
This is the "public" IP address of the Kubernetes master node.