From c7b447b9fc705e6db56f477dfae676ed456ef037 Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Wed, 24 Feb 2016 20:49:56 +0900 Subject: [PATCH 01/31] Add Heat template to create a Kubernetes stack on OpenStack --- .../fragments/configure-salt.yaml | 47 +++ .../deploy-kube-auth-files-master.yaml | 43 +++ .../deploy-kube-auth-files-node.yaml | 44 +++ .../kubernetes-heat/fragments/kube-user.yaml | 10 + .../fragments/provision-network-master.sh | 67 ++++ .../fragments/provision-network-node.sh | 43 +++ .../kubernetes-heat/fragments/run-salt.sh | 48 +++ .../fragments/write-heat-params.yaml | 11 + .../kubernetes-heat/kubecluster.yaml | 326 ++++++++++++++++++ .../openstack/kubernetes-heat/kubeminion.yaml | 201 +++++++++++ 10 files changed, 840 insertions(+) create mode 100644 cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml create mode 100644 cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml create mode 100644 cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml create mode 100644 cluster/openstack/kubernetes-heat/fragments/kube-user.yaml create mode 100644 cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh create mode 100644 cluster/openstack/kubernetes-heat/fragments/provision-network-node.sh create mode 100644 cluster/openstack/kubernetes-heat/fragments/run-salt.sh create mode 100644 cluster/openstack/kubernetes-heat/fragments/write-heat-params.yaml create mode 100644 cluster/openstack/kubernetes-heat/kubecluster.yaml create mode 100644 cluster/openstack/kubernetes-heat/kubeminion.yaml diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml new file mode 100644 index 0000000000000..e35001466b76b --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml @@ -0,0 +1,47 @@ +#cloud-config +merge_how: dict(recurse_array)+list(append) +bootcmd: + - mkdir -p /etc/salt/minion.d + - mkdir -p /srv/salt-overlay/pillar +write_files: + - path: /etc/salt/minion.d/log-level-debug.conf + content: | + log_level: warning + log_level_logfile: warning + - path: /etc/salt/minion.d/grains.conf + content: | + grains: + node_ip: $MASTER_IP + publicAddressOverride: $MASTER_IP + network_mode: openvswitch + networkInterfaceName: eth0 + api_servers: $MASTER_IP + cloud: vagrant # It's not vagrant, but required to install Docker + roles: + - $role + runtime_config: "" + docker_opts: "" + master_extra_sans: "DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:kubernetes-master" + keep_host_etcd: true + - path: /srv/salt-overlay/pillar/cluster-params.sls + content: | + service_cluster_ip_range: 10.246.0.0/16 + cert_ip: $MASTER_IP + enable_cluster_monitoring: none + enable_cluster_logging: "false" + enable_cluster_ui: "false" + enable_node_logging: "false" + logging_destination: elasticsearch + elasticsearch_replicas: "1" + enable_cluster_dns: "true" + dns_replicas: "1" + dns_server: 10.246.0.10 + dns_domain: cluster.local + instance_prefix: kubernetes + admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota + enable_cpu_cfs_quota: "true" + network_provider: none + opencontrail_tag: R2.20 + opencontrail_kubernetes_tag: master + opencontrail_public_subnet: 10.1.0.0/16 + e2e_storage_test_environment: "false" diff --git a/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml b/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml new file mode 100644 index 0000000000000..10ba9ae886166 --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml @@ -0,0 +1,43 @@ +#cloud-config +merge_how: dict(recurse_array)+list(append) +bootcmd: + - mkdir -p /srv/salt-overlay/salt/kube-apiserver + - mkdir -p /srv/salt-overlay/salt/kubelet +write_files: + - path: /srv/salt-overlay/salt/kube-apiserver/basic_auth.csv + permissions: "0600" + content: | + $apiserver_user,$apiserver_password,admin + - path: /srv/salt-overlay/salt/kube-apiserver/known_tokens.csv + permissions: "0600" + content: | + $token_kubelet,kubelet,kubelet + $token_kube_proxy,kube_proxy,kube_proxy + TokenSystemScheduler,system:scheduler,system:scheduler + TokenSystemControllerManager,system:controller_manager,system:controller_manager + TokenSystemLogging,system:logging,system:logging + TokenSystemMonitoring,system:monitoring,system:monitoring + TokenSystemDns,system:dns,system:dns + - path: /srv/salt-overlay/salt/kubelet/kubernetes_auth + permissions: "0600" + content: | + {"BearerToken": "$token_kubelet", "Insecure": true } + - path: /srv/salt-overlay/salt/kubelet/kubeconfig + permissions: "0600" + content: | + apiVersion: v1 + kind: Config + users: + - name: kubelet + user: + token: $token_kubelet + clusters: + - name: local + cluster: + insecure-skip-tls-verify: true + contexts: + - context: + cluster: local + user: kubelet + name: service-account-context + current-context: service-account-context diff --git a/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml b/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml new file mode 100644 index 0000000000000..7f265d2b8a6ad --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml @@ -0,0 +1,44 @@ +#cloud-config +merge_how: dict(recurse_array)+list(append) +bootcmd: + - mkdir -p /srv/salt-overlay/salt/kubelet + - mkdir -p /srv/salt-overlay/salt/kube-proxy +write_files: + - path: /srv/salt-overlay/salt/kubelet/kubeconfig + permissions: "0600" + content: | + apiVersion: v1 + kind: Config + users: + - name: kubelet + user: + token: $token_kubelet + clusters: + - name: local + cluster: + insecure-skip-tls-verify: true + contexts: + - context: + cluster: local + user: kubelet + name: service-account-context + current-context: service-account-context + - path: /srv/salt-overlay/salt/kube-proxy/kubeconfig + permissions: "0600" + content: | + apiVersion: v1 + kind: Config + users: + - name: kube-proxy + user: + token: $token_kube_proxy + clusters: + - name: local + cluster: + insecure-skip-tls-verify: true + contexts: + - context: + cluster: local + user: kube-proxy + name: service-account-context + current-context: service-account-context diff --git a/cluster/openstack/kubernetes-heat/fragments/kube-user.yaml b/cluster/openstack/kubernetes-heat/fragments/kube-user.yaml new file mode 100644 index 0000000000000..4e7477d64019a --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/kube-user.yaml @@ -0,0 +1,10 @@ +#cloud-config +system_info: + default_user: + name: minion + lock_passwd: true + gecos: Kubernetes Interactive User + groups: [wheel, adm, systemd-journal] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + diff --git a/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh b/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh new file mode 100644 index 0000000000000..3cdf9b8b2bc58 --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +. /etc/sysconfig/heat-params + +FLANNEL_ETCD_URL="http://${MASTER_IP}:4379" + +# Install etcd for flannel data +if ! which etcd > /dev/null 2>&1; then + yum install -y etcd +fi + +cat < /etc/etcd/etcd.conf +ETCD_NAME=flannel +ETCD_DATA_DIR="/var/lib/etcd/flannel.etcd" +ETCD_LISTEN_PEER_URLS="http://${MASTER_IP}:4380" +ETCD_LISTEN_CLIENT_URLS="http://${MASTER_IP}:4379" +ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${MASTER_IP}:4380" +ETCD_INITIAL_CLUSTER="flannel=http://${MASTER_IP}:4380" +ETCD_ADVERTISE_CLIENT_URLS="${FLANNEL_ETCD_URL}" +EOF +systemctl enable etcd +systemctl restart etcd + +# Install flannel for overlay +if ! which flanneld > /dev/null 2>&1; then + yum install -y flannel +fi + +cat < /etc/flannel-config.json +{ + "Network": "${CONTAINER_SUBNET}", + "SubnetLen": 24, + "Backend": { + "Type": "udp", + "Port": 8285 + } +} +EOF + +etcdctl -C ${FLANNEL_ETCD_URL} set /coreos.com/network/config < /etc/flannel-config.json + +cat < /etc/sysconfig/flanneld +FLANNEL_ETCD="${FLANNEL_ETCD_URL}" +FLANNEL_ETCD_KEY="/coreos.com/network" +FLANNEL_OPTIONS="-iface=eth0 --ip-masq" +EOF + +systemctl enable flanneld +systemctl restart flanneld diff --git a/cluster/openstack/kubernetes-heat/fragments/provision-network-node.sh b/cluster/openstack/kubernetes-heat/fragments/provision-network-node.sh new file mode 100644 index 0000000000000..7fd78598e2096 --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/provision-network-node.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +. /etc/sysconfig/heat-params + +FLANNEL_ETCD_URL="http://${MASTER_IP}:4379" + +# Install flannel for overlay +if ! which flanneld >/dev/null 2>&1; then + yum install -y flannel +fi + +cat </etc/sysconfig/flanneld +FLANNEL_ETCD="${FLANNEL_ETCD_URL}" +FLANNEL_ETCD_KEY="/coreos.com/network" +FLANNEL_OPTIONS="-iface=eth0 --ip-masq" +EOF + +systemctl enable flanneld +systemctl restart flanneld + +# Kubernetes node shoud be able to resolve its hostname. +# In some cloud providers, myhostname is not enabled by default. +grep '^hosts:.*myhostname' /etc/nsswitch.conf || ( + sed -e 's/^hosts:\(.*\)/hosts:\1 myhostname/' -i /etc/nsswitch.conf +) diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh new file mode 100644 index 0000000000000..97d521eac744a --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +. /etc/sysconfig/heat-params + +rm -rf /kube-install +mkdir -p /kube-install +cd /kube-install + +curl "$KUBERNETES_SERVER_URL" -o kubernetes-server.tar.gz +curl "$KUBERNETES_SALT_URL" -o kubernetes-salt.tar.gz + +tar xzf kubernetes-salt.tar.gz +./kubernetes/saltbase/install.sh kubernetes-server.tar.gz + +if ! which salt-call >/dev/null 2>&1; then + # Install salt binaries + curl -sS -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s +fi + +# Salt server runs at locahost +echo "127.0.0.1 salt" >> /etc/hosts + +# Currently heat template tells a lie that the target is Vagrant. If Vagrant cloud provider is enabled, "Unable to construct api.Node object for kubelet" error will occur. +sed -e 's/{{cloud_provider}}//' -i /srv/salt/kubelet/default + +# Run salt-call +# salt-call wants to start docker daemon but is unable to. +# See . +# Run salt-call in background and make cloud-final finished. +salt-call --local state.highstate && $$wc_notify --data-binary '{"status": "SUCCESS"}' || $$wc_notify --data-binary '{"status": "FAILURE"}' & diff --git a/cluster/openstack/kubernetes-heat/fragments/write-heat-params.yaml b/cluster/openstack/kubernetes-heat/fragments/write-heat-params.yaml new file mode 100644 index 0000000000000..d8ee9da0f9f32 --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/write-heat-params.yaml @@ -0,0 +1,11 @@ +#cloud-config +merge_how: dict(recurse_array)+list(append) +write_files: + - path: /etc/sysconfig/heat-params + owner: "root:root" + permissions: "0644" + content: | + KUBERNETES_SERVER_URL="$KUBERNETES_SERVER_URL" + KUBERNETES_SALT_URL="$KUBERNETES_SALT_URL" + MASTER_IP=$MASTER_IP + CONTAINER_SUBNET=10.246.0.0/16 diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml new file mode 100644 index 0000000000000..3832f4c4fda4a --- /dev/null +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -0,0 +1,326 @@ +heat_template_version: 2014-10-16 + +description: > + Kubernetes cluster with one master and one or more worker nodes + (as specified by the number_of_minions parameter, which defaults to 2). + +parameters: + plugin_version: + type: string + description: version of CLC plugin. It is used when you show cluster list view. + default: 1.0 + constraints: + - allowed_values: + - 1.0 + + ssh_key_name: + type: string + description: name of ssh key to be provisioned on our server + + external_network: + type: string + description: uuid/name of a network to use for floating ip addresses + default: public + + server_image: + type: string + description: glance image used to boot the server + + master_flavor: + type: string + default: m1.small + description: flavor to use when booting the server + + minion_flavor: + type: string + default: m1.small + description: flavor to use when booting the server + + dns_nameserver: + type: string + description: address of a dns nameserver reachable in your environment + default: 8.8.8.8 + + number_of_minions: + type: number + description: how many kubernetes minions to spawn initially + default: 1 + + max_number_of_minions: + type: number + description: maximum number of kubernetes minions to spawn + default: 10 + + fixed_network_cidr: + type: string + description: network range for fixed ip network + default: 10.0.0.0/24 + + kubernetes_server_url: + type: string + description: URL of kubernetes server binary. Must be tar.gz. + + kubernetes_salt_url: + type: string + description: URL of kubernetes salt scripts. Must be tar.gz. + + apiserver_user: + type: string + description: User name used for api-server + default: user + + apiserver_password: + type: string + description: Password used for api-server + default: password + + token_kubelet: + type: string + description: Token used by kubelet + default: TokenKubelet + + token_kube_proxy: + type: string + description: Token used by kube-proxy + default: TokenKubeproxy + + wait_condition_timeout: + type: number + description : > + timeout for the Wait Conditions + default: 6000 + +resources: + + master_wait_handle: + type: OS::Heat::WaitConditionHandle + + master_wait_condition: + type: OS::Heat::WaitCondition + depends_on: kube_master + properties: + handle: {get_resource: master_wait_handle} + timeout: {get_param: wait_condition_timeout} + + ###################################################################### + # + # network resources. allocate a network and router for our server. + # + + fixed_network: + type: OS::Neutron::Net + + fixed_subnet: + type: OS::Neutron::Subnet + properties: + cidr: {get_param: fixed_network_cidr} + network: {get_resource: fixed_network} + dns_nameservers: + - {get_param: dns_nameserver} + + extrouter: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: {get_param: external_network} + + extrouter_inside: + type: OS::Neutron::RouterInterface + properties: + router_id: {get_resource: extrouter} + subnet: {get_resource: fixed_subnet} + + ###################################################################### + # + # security groups. we need to permit network traffic of various + # sorts. + # + + secgroup_base: + type: OS::Neutron::SecurityGroup + properties: + rules: + - protocol: icmp + - protocol: tcp + port_range_min: 22 + port_range_max: 22 + + secgroup_kubernetes: + type: OS::Neutron::SecurityGroup + properties: + rules: + - protocol: tcp # etcd for flanneld + port_range_min: 4379 + port_range_max: 4380 + - protocol: udp # flannel backend + port_range_min: 8285 + port_range_max: 8285 + - protocol: tcp # api-server + port_range_min: 443 + port_range_max: 443 + + + ###################################################################### + # + # software configs. these are components that are combined into + # a multipart MIME user-data archive. + # + + write_heat_params: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/write-heat-params.yaml} + params: + "$KUBERNETES_SERVER_URL": {get_param: kubernetes_server_url} + "$KUBERNETES_SALT_URL": {get_param: kubernetes_salt_url} + "$MASTER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + + kube_user: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/kube-user.yaml} + + provision_network_master: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/provision-network-master.sh} + + deploy_kube_auth_files_master: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/deploy-kube-auth-files-master.yaml} + params: + "$apiserver_user": {get_param: apiserver_user} + "$apiserver_password": {get_param: apiserver_password} + "$token_kubelet": {get_param: token_kubelet} + "$token_kube_proxy": {get_param: token_kube_proxy} + + configure_salt_master: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/configure-salt.yaml} + params: + "$MASTER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + "$role": "kubernetes-master" + + run_salt: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/run-salt.sh} + params: + "$$wc_notify": {get_attr: [master_wait_handle, curl_cli]} + + kube_master_init: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: write_heat_params} + - config: {get_resource: kube_user} + - config: {get_resource: provision_network_master} + - config: {get_resource: deploy_kube_auth_files_master} + - config: {get_resource: configure_salt_master} + - config: {get_resource: run_salt} + + ###################################################################### + # + # kubernetes master server. + # + + kube_master: + type: OS::Nova::Server + depends_on: + - extrouter_inside + properties: + image: {get_param: server_image} + flavor: {get_param: master_flavor} + key_name: {get_param: ssh_key_name} + user_data_format: RAW + user_data: {get_resource: kube_master_init} + networks: + - port: {get_resource: kube_master_eth0} + name: + list_join: [-, [{get_param: "OS::stack_name"}, master]] + + kube_master_eth0: + type: OS::Neutron::Port + properties: + network: {get_resource: fixed_network} + security_groups: + - {get_resource: secgroup_base} + - {get_resource: secgroup_kubernetes} + fixed_ips: + - subnet: {get_resource: fixed_subnet} + replacement_policy: AUTO + + kube_master_floating: + type: OS::Neutron::FloatingIP + properties: + floating_network: {get_param: external_network} + port_id: {get_resource: kube_master_eth0} + + ###################################################################### + # + # kubernetes minions. This is an autoscaling group that will initially + # create minions, and will scale up to + # based on CPU utilization. + # + + kube_minions: + type: OS::Heat::AutoScalingGroup + depends_on: + - extrouter_inside + - master_wait_condition + properties: + resource: + type: kubeminion.yaml + properties: + kubernetes_server_url: {get_param: kubernetes_server_url} + kubernetes_salt_url: {get_param: kubernetes_salt_url} + ssh_key_name: {get_param: ssh_key_name} + server_image: {get_param: server_image} + minion_flavor: {get_param: minion_flavor} + token_kubelet: {get_param: token_kubelet} + token_kube_proxy: {get_param: token_kube_proxy} + fixed_network: {get_resource: fixed_network} + fixed_subnet: {get_resource: fixed_subnet} + kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + external_network: {get_param: external_network} + wait_condition_timeout: {get_param: wait_condition_timeout} + metadata: {"metering.stack": {get_param: "OS::stack_id"}} + cluster_name: {get_param: "OS::stack_name"} + min_size: {get_param: number_of_minions} + desired_capacity: {get_param: number_of_minions} + max_size: {get_param: max_number_of_minions} + +outputs: + + kube_master: + value: {get_attr: [kube_master_floating, floating_ip_address]} + description: > + This is the "public" IP address of the Kubernetes master node. Use this IP address + to log in to the Kubernetes master via ssh or to access the Kubernetes API + from outside the cluster. + + kube_minions: + value: {get_attr: [kube_minions, outputs_list, kube_minion_ip]} + description: > + Here is the list of the "private" addresses of all Kubernetes worker nodes. + + kube_minions_external: + value: {get_attr: [kube_minions, outputs_list, kube_minion_external_ip]} + description: > + Here is the list of the "public" addresses of all Kubernetes worker nodes. diff --git a/cluster/openstack/kubernetes-heat/kubeminion.yaml b/cluster/openstack/kubernetes-heat/kubeminion.yaml new file mode 100644 index 0000000000000..76075c76079bf --- /dev/null +++ b/cluster/openstack/kubernetes-heat/kubeminion.yaml @@ -0,0 +1,201 @@ +heat_template_version: 2014-10-16 + +description: > + This is a nested stack that defines a single Kubernetes minion, This stack is + included by an AutoScalingGroup resource in the parent template + (kubecluster.yaml). + +parameters: + + server_image: + type: string + description: glance image used to boot the server + + minion_flavor: + type: string + default: m1.small + description: flavor to use when booting the server + + ssh_key_name: + type: string + description: name of ssh key to be provisioned on our server + default: lars + + external_network: + type: string + description: uuid/name of a network to use for floating ip addresses + + kubernetes_server_url: + type: string + description: URL of kubernetes server binary. Must be tar.gz. + + kubernetes_salt_url: + type: string + description: URL of kubernetes salt scripts. Must be tar.gz. + + token_kubelet: + type: string + description: Token used by kubelet + + token_kube_proxy: + type: string + description: Token used by kube-proxy + + # The following are all generated in the parent template. + kube_master_ip: + type: string + description: IP address of the Kubernetes master server. + fixed_network: + type: string + description: Network from which to allocate fixed addresses. + fixed_subnet: + type: string + description: Subnet from which to allocate fixed addresses. + wait_condition_timeout: + type: number + description : > + timeout for the Wait Conditions + metadata: + type: json + description: metadata for ceilometer query + cluster_name: + type: string + +resources: + + minion_wait_handle: + type: OS::Heat::WaitConditionHandle + + minion_wait_condition: + type: OS::Heat::WaitCondition + depends_on: kube_minion + properties: + handle: {get_resource: minion_wait_handle} + timeout: {get_param: wait_condition_timeout} + + secgroup_all_open: + type: OS::Neutron::SecurityGroup + properties: + rules: + - protocol: icmp + - protocol: tcp + - protocol: udp + + ###################################################################### + # + # software configs. these are components that are combined into + # a multipart MIME user-data archive. + # + + write_heat_params: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/write-heat-params.yaml} + params: + "$KUBERNETES_SERVER_URL": {get_param: kubernetes_server_url} + "$KUBERNETES_SALT_URL": {get_param: kubernetes_salt_url} + "$MASTER_IP": {get_param: kube_master_ip} + + kube_user: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/kube-user.yaml} + + provision_network_node: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/provision-network-node.sh} + + deploy_kube_auth_files_node: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/deploy-kube-auth-files-node.yaml} + params: + "$token_kubelet": {get_param: token_kubelet} + "$token_kube_proxy": {get_param: token_kube_proxy} + + configure_salt_node: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/configure-salt.yaml} + params: + "$MASTER_IP": {get_param: kube_master_ip} + "$role": "kubernetes-pool" + + run_salt: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/run-salt.sh} + params: + "$$wc_notify": {get_attr: [minion_wait_handle, curl_cli]} + + kube_minion_init: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: write_heat_params} + - config: {get_resource: kube_user} + - config: {get_resource: provision_network_node} + - config: {get_resource: deploy_kube_auth_files_node} + - config: {get_resource: configure_salt_node} + - config: {get_resource: run_salt} + + ###################################################################### + # + # a single kubernetes minion. + # + server_name_post_fix: + type: OS::Heat::RandomString + properties: + length: 8 + + kube_minion: + type: OS::Nova::Server + properties: + image: {get_param: server_image} + flavor: {get_param: minion_flavor} + key_name: {get_param: ssh_key_name} + metadata: {get_param: metadata} + user_data_format: RAW + user_data: {get_resource: kube_minion_init} + networks: + - port: {get_resource: kube_minion_eth0} + name: + list_join: [-, [{get_param: cluster_name}, node, {get_resource: server_name_post_fix}]] + + kube_minion_eth0: + type: OS::Neutron::Port + properties: + network: {get_param: fixed_network} + security_groups: + - get_resource: secgroup_all_open + fixed_ips: + - subnet: {get_param: fixed_subnet} + replacement_policy: AUTO + + kube_minion_floating: + type: OS::Neutron::FloatingIP + properties: + floating_network: {get_param: external_network} + port_id: {get_resource: kube_minion_eth0} + +outputs: + + kube_minion_ip: + value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} + kube_minion_external_ip: + value: {get_attr: [kube_minion_floating, floating_ip_address]} From 840aa9c1867e922967ab70ed7310b52bbf22d82e Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Wed, 24 Feb 2016 20:51:45 +0900 Subject: [PATCH 02/31] OpenStack provider - Create Kubernetes cluster on the top of OpenStack --- cluster/openstack/config-default.sh | 54 +++++++ cluster/openstack/config-image.sh | 32 ++++ cluster/openstack/openrc-default.sh | 24 +++ cluster/openstack/openrc-swift.sh | 26 ++++ cluster/openstack/util.sh | 232 ++++++++++++++++++++++++++++ 5 files changed, 368 insertions(+) create mode 100644 cluster/openstack/config-default.sh create mode 100644 cluster/openstack/config-image.sh create mode 100644 cluster/openstack/openrc-default.sh create mode 100644 cluster/openstack/openrc-swift.sh create mode 100644 cluster/openstack/util.sh diff --git a/cluster/openstack/config-default.sh b/cluster/openstack/config-default.sh new file mode 100644 index 0000000000000..c806b96d5eb40 --- /dev/null +++ b/cluster/openstack/config-default.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for the Openstack cluster + +# Stack name +STACK_NAME="KubernetesStack" + +# Keypair for kubernetes stack +KUBERNETES_KEYPAIR_NAME="kubernetes_keypair" + +# Kubernetes release tar file +KUBERNETES_RELEASE_TAR="kubernetes-server-linux-amd64.tar.gz" + +NUMBER_OF_MINIONS="1" + +MAX_NUMBER_OF_MINIONS="1" + +MASTER_FLAVOR="m1.small" + +MINION_FLAVOR="m1.small" + +EXTERNAL_NETWORK="public" + +SWIFT_SERVER_URL="http://192.168.123.100:8080" + +# Flag indicates if new image must be created. If 'false' then image with IMAGE_ID will be used. +# If 'true' then new image will be created from file config-image.sh +CREATE_IMAGE="true" # use "true" for devstack + +# Image id which will be used for kubernetes stack +IMAGE_ID="f0f394b1-5546-4b68-b2bc-8abe8a7e6b8b" + +# DNS server address +DNS_SERVER="8.8.8.8" + +# Public RSA key path +CLIENT_PUBLIC_KEY_PATH="~/.ssh/id_rsa.pub" + +# Max time period for stack provisioning. Time in minutes. +STACK_CREATE_TIMEOUT=60 diff --git a/cluster/openstack/config-image.sh b/cluster/openstack/config-image.sh new file mode 100644 index 0000000000000..d62e80d75c100 --- /dev/null +++ b/cluster/openstack/config-image.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for new image. It is skip when CREATE_IMAGE=false + +# Image name which will be displayed in OpenStack +OPENSTACK_IMAGE_NAME="CentOS7" + +# Downloaded image name for Openstack project +IMAGE_FILE="CentOS-7-x86_64-GenericCloud-1510.qcow2" + +# Absolute path where image file is stored. +IMAGE_PATH="/home/openstack/openstack_temp" + +# The disk format of the image. Acceptable formats are ami, ari, aki, vhd, vmdk, raw, qcow2, vdi, and iso. +IMAGE_FORMAT="qcow2" + +# The container format of the image. Acceptable formats are ami, ari, aki, bare, docker, and ovf. +CONTAINER_FORMAT="bare" diff --git a/cluster/openstack/openrc-default.sh b/cluster/openstack/openrc-default.sh new file mode 100644 index 0000000000000..6e9fe67f237f7 --- /dev/null +++ b/cluster/openstack/openrc-default.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Enviroment variables for the OpenStack command-line client + +export OS_IDENTITY_API_VERSION=2.0 +export OS_USERNAME=admin +export OS_PASSWORD=secretsecret +export OS_AUTH_URL=http://192.168.123.100:5000/v2.0 +export OS_TENANT_NAME=admin +export OS_TENANT_ID=ed51b98b40944d89a449592eb67431eb diff --git a/cluster/openstack/openrc-swift.sh b/cluster/openstack/openrc-swift.sh new file mode 100644 index 0000000000000..e8e69504df073 --- /dev/null +++ b/cluster/openstack/openrc-swift.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Enviroment variables for the OpenStack Swift command-line client. This is required for CityCloud +## provider where Swift has different credentials. When Swift is part of your OpenStack use the same +## settings as in openrc-default.sh + +export OS_IDENTITY_API_VERSION=2.0 +export OS_USERNAME=admin +export OS_PASSWORD=secretsecret +export OS_AUTH_URL=http://192.168.123.100:5000/v2.0 +export OS_TENANT_NAME=admin +export OS_TENANT_ID=ed51b98b40944d89a449592eb67431eb diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh new file mode 100644 index 0000000000000..cacf86004c592 --- /dev/null +++ b/cluster/openstack/util.sh @@ -0,0 +1,232 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts. + +# exit on any error +set -e + +# Use the config file specified in $KUBE_CONFIG_FILE, or default to +# config-default.sh. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +readonly ROOT=$(dirname "${BASH_SOURCE}") +source "${ROOT}/${KUBE_CONFIG_FILE:-"config-default.sh"}" +source "$KUBE_ROOT/cluster/common.sh" +if [ $CREATE_IMAGE = true ]; then +source "${ROOT}/config-image.sh" +fi + +# Verify prereqs on host machine +function verify-prereqs() { + # Check the OpenStack command-line clients + for client in swift glance nova heat; + do + if which $client >/dev/null 2>&1; then + echo "$client client installed" + else + echo "$client client does not exist" + echo "Please install $client client, and retry." + exit 1 + fi + done +} + +# Instantiate a kubernetes cluster +# +# Assumed vars: +# KUBERNETES_PROVIDER +function kube-up() { + echo "kube-up for provider $KUBERNETES_PROVIDER" + create-stack +} + +# Periodically checks if cluster is created +# +# Assumed vars: +# STACK_CREATE_TIMEOUT +# STACK_NAME +function validate-cluster() { + + while (( --$STACK_CREATE_TIMEOUT >= 0)) ;do + local status=$(heat stack-show "${STACK_NAME}" | awk '$2=="stack_status" {print $4}') + if [[ $status ]]; then + echo "Cluster status ${status}" + if [ $status = "CREATE_COMPLETE" ]; then + configure-kubectl + break + elif [ $status = "CREATE_FAILED" ]; then + echo "Cluster not created. Please check stack logs to find the problem" + break + fi + else + echo "Cluster not created. Please verify if process started correctly" + break + fi + sleep 60 + done +} + +# Create stack +# +# Assumed vars: +# OPENSTACK +# OPENSTACK_TEMP +# DNS_SERVER +# OPENSTACK_IP +# OPENRC_FILE +function create-stack() { + echo "[INFO] Execute commands to create Kubernetes cluster" + # It is required for some cloud provider like CityCloud where swift client has different credentials + source "${ROOT}/openrc-swift.sh" + if [[ ! -v OS_PROJECT_ID ]]; then + SWIFT_PROJECT_ID="${OS_TENANT_ID}" + else + SWIFT_PROJECT_ID="${OS_PROJECT_ID}" + fi + upload-resources + source "${ROOT}/openrc-default.sh" + + create-glance-image + + add-keypair + run-heat-script +} + +# Upload kubernetes release tars and heat templates. +# +# Assumed vars: +# ROOT +# KUBERNETES_RELEASE_TAR +function upload-resources() { + swift post kubernetes --read-acl '.r:*,.rlistings' + + echo "[INFO] Upload ${KUBERNETES_RELEASE_TAR}" + swift upload kubernetes ${ROOT}/../../_output/release-tars/${KUBERNETES_RELEASE_TAR} \ + --object-name kubernetes-server.tar.gz + + echo "[INFO] Upload kubernetes-salt.tar.gz" + swift upload kubernetes ${ROOT}/../../_output/release-tars/kubernetes-salt.tar.gz \ + --object-name kubernetes-salt.tar.gz +} + +# Create a new key pair for use with servers. +# +# Assumed vars: +# KUBERNETES_KEYPAIR_NAME +# CLIENT_PUBLIC_KEY_PATH +function add-keypair() { + local status=$(nova keypair-show ${KUBERNETES_KEYPAIR_NAME}) + if [[ ! $status ]]; then + nova keypair-add ${KUBERNETES_KEYPAIR_NAME} --pub-key ${CLIENT_PUBLIC_KEY_PATH} + echo "[INFO] Key pair created" + else + echo "[INFO] Key pair already exists" + fi +} + +# Create a new glance image. +# +# Assumed vars: +# IMAGE_FILE +# IMAGE_PATH +# OPENSTACK_IMAGE_NAME +function create-glance-image() { + if [ $CREATE_IMAGE = true ]; then + local image_status=$(nova image-show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') + + if [[ ! $image_status ]]; then + echo "[INFO] Create image ${OPENSTACK_IMAGE_NAME}" + glance image-create --name ${OPENSTACK_IMAGE_NAME} --disk-format ${IMAGE_FORMAT} \ + --container-format ${CONTAINER_FORMAT} --file ${IMAGE_PATH}/${IMAGE_FILE} + else + echo "[INFO] Image ${OPENSTACK_IMAGE_NAME} already exists" + fi + fi +} + +# Create a new kubernetes stack. +# +# Assumed vars: +# STACK_NAME +# KUBERNETES_KEYPAIR_NAME +# DNS_SERVER +# SWIFT_SERVER_URL +# SWIFT_TENANT_ID +# OPENSTACK_IMAGE_NAME +# EXTERNAL_NETWORK +# IMAGE_ID +# MASTER_FLAVOR +# MINION_FLAVOR +# NUMBER_OF_MINIONS +# MAX_NUMBER_OF_MINIONS +# DNS_SERVER +# STACK_NAME +function run-heat-script() { + + local stack_status=$(heat stack-show ${STACK_NAME}) + local swift_repo_url="${SWIFT_SERVER_URL}/v1/AUTH_${SWIFT_PROJECT_ID}/kubernetes" + + if [ $CREATE_IMAGE = true ]; then + echo "[INFO] Retrieve new image ID" + IMAGE_ID=$(nova image-show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') + echo "[INFO] Image Id $IMAGE_ID" + fi + + if [[ ! $stack_status ]]; then + echo "[INFO] Create stack ${STACK_NAME}" + ( + cd ${ROOT}/kubernetes-heat + heat --api-timeout 60 stack-create \ + -P external_network=${EXTERNAL_NETWORK} \ + -P ssh_key_name=${KUBERNETES_KEYPAIR_NAME} \ + -P server_image=${IMAGE_ID} \ + -P master_flavor=${MASTER_FLAVOR} \ + -P minion_flavor=${MINION_FLAVOR} \ + -P number_of_minions=${NUMBER_OF_MINIONS} \ + -P max_number_of_minions=${MAX_NUMBER_OF_MINIONS} \ + -P dns_nameserver=${DNS_SERVER} \ + -P kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \ + -P kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \ + --template-file kubecluster.yaml \ + ${STACK_NAME} + ) + else + echo "[INFO] Stack ${STACK_NAME} already exists" + heat stack-show ${STACK_NAME} + fi +} + +# Configure kubectl. +# +# Assumed vars: +# STACK_NAME +function configure-kubectl() { + + export KUBE_MASTER_IP=$(nova show "${STACK_NAME}"-master | awk '$3=="network" {print $6}') + export CONTEXT="openstack" + export KUBE_BEARER_TOKEN="TokenKubelet" + create-kubeconfig +} + + +# Delete a kubernetes cluster +# +# Assumed vars: +# STACK_NAME +function kube-down { + source "${ROOT}/openrc-default.sh" + heat stack-delete ${STACK_NAME} +} From 188b303dd023a79bf14e586359053ba1e30d7f29 Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Wed, 16 Mar 2016 17:58:07 +0900 Subject: [PATCH 03/31] Execute salt-call twice --- cluster/openstack/kubernetes-heat/fragments/run-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh index 97d521eac744a..1fe355789e19e 100644 --- a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh +++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh @@ -45,4 +45,5 @@ sed -e 's/{{cloud_provider}}//' -i /srv/salt/kubelet/default # salt-call wants to start docker daemon but is unable to. # See . # Run salt-call in background and make cloud-final finished. -salt-call --local state.highstate && $$wc_notify --data-binary '{"status": "SUCCESS"}' || $$wc_notify --data-binary '{"status": "FAILURE"}' & +# Salt-call might be unstable in some environments, execute it twice. +salt-call --local state.highstate && salt-call --local state.highstate && $$wc_notify --data-binary '{"status": "SUCCESS"}' || $$wc_notify --data-binary '{"status": "FAILURE"}' & From 249a5c9f0f79885451dc86234098cbc6c2a335d2 Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Tue, 15 Mar 2016 23:49:11 +0900 Subject: [PATCH 04/31] Use host-gw flannel for better performance --- .../fragments/provision-network-master.sh | 3 +-- .../kubernetes-heat/kubecluster.yaml | 22 ++++++++++++------- .../openstack/kubernetes-heat/kubeminion.yaml | 17 +++++++------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh b/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh index 3cdf9b8b2bc58..901734e827306 100644 --- a/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh +++ b/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh @@ -49,8 +49,7 @@ cat < /etc/flannel-config.json "Network": "${CONTAINER_SUBNET}", "SubnetLen": 24, "Backend": { - "Type": "udp", - "Port": 8285 + "Type": "host-gw" } } EOF diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index 3832f4c4fda4a..bd1cc4643b16f 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -144,21 +144,23 @@ resources: - protocol: tcp port_range_min: 22 port_range_max: 22 + - remote_mode: remote_group_id - secgroup_kubernetes: + secgroup_master: type: OS::Neutron::SecurityGroup properties: rules: - - protocol: tcp # etcd for flanneld - port_range_min: 4379 - port_range_max: 4380 - - protocol: udp # flannel backend - port_range_min: 8285 - port_range_max: 8285 - protocol: tcp # api-server port_range_min: 443 port_range_max: 443 + secgroup_node: + type: OS::Neutron::SecurityGroup + properties: + rules: + - protocol: icmp + - protocol: tcp + - protocol: udp ###################################################################### # @@ -261,9 +263,11 @@ resources: network: {get_resource: fixed_network} security_groups: - {get_resource: secgroup_base} - - {get_resource: secgroup_kubernetes} + - {get_resource: secgroup_master} fixed_ips: - subnet: {get_resource: fixed_subnet} + allowed_address_pairs: + - ip_address: 10.246.0.0/16 replacement_policy: AUTO kube_master_floating: @@ -302,6 +306,8 @@ resources: wait_condition_timeout: {get_param: wait_condition_timeout} metadata: {"metering.stack": {get_param: "OS::stack_id"}} cluster_name: {get_param: "OS::stack_name"} + secgroup_base: {get_resource: secgroup_base} + secgroup_node: {get_resource: secgroup_node} min_size: {get_param: number_of_minions} desired_capacity: {get_param: number_of_minions} max_size: {get_param: max_number_of_minions} diff --git a/cluster/openstack/kubernetes-heat/kubeminion.yaml b/cluster/openstack/kubernetes-heat/kubeminion.yaml index 76075c76079bf..f33dd2008fd00 100644 --- a/cluster/openstack/kubernetes-heat/kubeminion.yaml +++ b/cluster/openstack/kubernetes-heat/kubeminion.yaml @@ -60,6 +60,10 @@ parameters: description: metadata for ceilometer query cluster_name: type: string + secgroup_base: + type: string + secgroup_node: + type: string resources: @@ -73,14 +77,6 @@ resources: handle: {get_resource: minion_wait_handle} timeout: {get_param: wait_condition_timeout} - secgroup_all_open: - type: OS::Neutron::SecurityGroup - properties: - rules: - - protocol: icmp - - protocol: tcp - - protocol: udp - ###################################################################### # # software configs. these are components that are combined into @@ -182,9 +178,12 @@ resources: properties: network: {get_param: fixed_network} security_groups: - - get_resource: secgroup_all_open + - {get_param: secgroup_base} + - {get_param: secgroup_node} fixed_ips: - subnet: {get_param: fixed_subnet} + allowed_address_pairs: + - ip_address: 10.246.0.0/16 replacement_policy: AUTO kube_minion_floating: From 730b7ef0973f5cbb6cd7331f4658fa96c802ebe9 Mon Sep 17 00:00:00 2001 From: urcan Date: Tue, 22 Mar 2016 15:22:20 +0100 Subject: [PATCH 05/31] certificate should be created for the virtual kubernetes ip --- cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml index e35001466b76b..aa20df28fb8ba 100644 --- a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml @@ -26,7 +26,7 @@ write_files: - path: /srv/salt-overlay/pillar/cluster-params.sls content: | service_cluster_ip_range: 10.246.0.0/16 - cert_ip: $MASTER_IP + cert_ip: 10.246.0.1 enable_cluster_monitoring: none enable_cluster_logging: "false" enable_cluster_ui: "false" From b5913ab43e05f01fcf4e473f79af8b38d26b4442 Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Fri, 8 Apr 2016 18:27:08 +0900 Subject: [PATCH 06/31] Make shell script compatible with older bash --- cluster/openstack/util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index cacf86004c592..266ff11149397 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -91,7 +91,7 @@ function create-stack() { echo "[INFO] Execute commands to create Kubernetes cluster" # It is required for some cloud provider like CityCloud where swift client has different credentials source "${ROOT}/openrc-swift.sh" - if [[ ! -v OS_PROJECT_ID ]]; then + if [[ -z "${OS_PROJECT_ID}" ]]; then SWIFT_PROJECT_ID="${OS_TENANT_ID}" else SWIFT_PROJECT_ID="${OS_PROJECT_ID}" From b4333adeac4ef389627f2809046b09c9c00ede63 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Tue, 12 Apr 2016 16:11:35 -0700 Subject: [PATCH 07/31] Adding support for provisioning behind a Proxy. This assumes you have your environement variables set correctly. When ENABLE_PROXY is set to true, it takes the current proxy settings and applies them to the heat configuration. Also modified the defaults system in config-default.sh --- cluster/openstack/config-default.sh | 30 ++++---- .../fragments/configure-proxy.sh | 68 +++++++++++++++++++ .../kubernetes-heat/fragments/run-salt.sh | 5 ++ .../kubernetes-heat/kubecluster.yaml | 46 +++++++++++++ .../openstack/kubernetes-heat/kubeminion.yaml | 46 +++++++++++++ cluster/openstack/util.sh | 6 ++ 6 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh diff --git a/cluster/openstack/config-default.sh b/cluster/openstack/config-default.sh index c806b96d5eb40..daad8d161d7f2 100644 --- a/cluster/openstack/config-default.sh +++ b/cluster/openstack/config-default.sh @@ -17,38 +17,40 @@ ## Contains configuration values for the Openstack cluster # Stack name -STACK_NAME="KubernetesStack" +STACK_NAME=${STACK_NAME:-KubernetesStack} # Keypair for kubernetes stack -KUBERNETES_KEYPAIR_NAME="kubernetes_keypair" +KUBERNETES_KEYPAIR_NAME=${KUBERNETES_KEYPAIR_NAME:-kubernetes_keypair} # Kubernetes release tar file -KUBERNETES_RELEASE_TAR="kubernetes-server-linux-amd64.tar.gz" +KUBERNETES_RELEASE_TAR=${KUBERNETES_RELEASE_TAR:-kubernetes-server-linux-amd64.tar.gz} -NUMBER_OF_MINIONS="1" +NUMBER_OF_MINIONS=${NUMBER_OF_MINIONS-1} -MAX_NUMBER_OF_MINIONS="1" +MAX_NUMBER_OF_MINIONS=${MAX_NUMBER_OF_MINIONS:-1} -MASTER_FLAVOR="m1.small" +MASTER_FLAVOR=${MASTER_FLAVOR:-m1.small} -MINION_FLAVOR="m1.small" +MINION_FLAVOR=${MINION_FLAVOR:-m1.small} -EXTERNAL_NETWORK="public" +EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-public} -SWIFT_SERVER_URL="http://192.168.123.100:8080" +SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-http://192.168.123.100:8080} # Flag indicates if new image must be created. If 'false' then image with IMAGE_ID will be used. # If 'true' then new image will be created from file config-image.sh -CREATE_IMAGE="true" # use "true" for devstack +CREATE_IMAGE=${CREATE_IMAGE:-true} # use "true" for devstack # Image id which will be used for kubernetes stack -IMAGE_ID="f0f394b1-5546-4b68-b2bc-8abe8a7e6b8b" +IMAGE_ID=${IMAGE_ID:-f0f394b1-5546-4b68-b2bc-8abe8a7e6b8b} # DNS server address -DNS_SERVER="8.8.8.8" +DNS_SERVER=${DNS_SERVER:-8.8.8.8} # Public RSA key path -CLIENT_PUBLIC_KEY_PATH="~/.ssh/id_rsa.pub" +CLIENT_PUBLIC_KEY_PATH=${CLIENT_PUBLIC_KEY_PATH:-~/.ssh/id_rsa.pub} # Max time period for stack provisioning. Time in minutes. -STACK_CREATE_TIMEOUT=60 +STACK_CREATE_TIMEOUT=${STACK_CREATE_TIMEOUT:-60} + +ENABLE_PROXY=${ENABLE_PROXY:-false} diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh b/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh new file mode 100644 index 0000000000000..60fba2f8efe8f --- /dev/null +++ b/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +#These values are meant to be swapped in by heat + +export ETC_ENVIRONMENT='FTP_PROXY=$FTP_PROXY +HTTP_PROXY=$HTTP_PROXY +HTTPS_PROXY=$HTTPS_PROXY +SOCKS_PROXY=$SOCKS_PROXY +NO_PROXY=$NO_PROXY +ftp_proxy=$FTP_PROXY +http_proxy=$HTTP_PROXY +https_proxy=$HTTPS_PROXY +socks_proxy=$SOCKS_PROXY +no_proxy=$NO_PROXY +' + +export ETC_PROFILE_D='export FTP_PROXY=$FTP_PROXY +export HTTP_PROXY=$HTTP_PROXY +export HTTPS_PROXY=$HTTPS_PROXY +export SOCKS_PROXY=$SOCKS_PROXY +export NO_PROXY=$NO_PROXY +export ftp_proxy=$FTP_PROXY +export http_proxy=$HTTP_PROXY +export https_proxy=$HTTPS_PROXY +export socks_proxy=$SOCKS_PROXY +export no_proxy=$NO_PROXY +' + +export DOCKER_PROXY='[Service] + Environment="HTTP_PROXY=$HTTP_PROXY" + Environment="HTTPS_PROXY=$HTTPS_PROXY" + Environment="SOCKS_PROXY=$SOCKS_PROXY" + Environment="NO_PROXY=$NO_PROXY" + Environment="ftp_proxy=$FTP_PROXY" + Environment="http_proxy=$HTTP_PROXY" + Environment="https_proxy=$HTTPS_PROXY" + Environment="socks_proxy=$SOCKS_PROXY" + Environment="no_proxy=$NO_PROXY" +' + +ENABLE_PROXY='$ENABLE_PROXY' + +if [[ "${ENABLE_PROXY}" == "true" ]]; then + mkdir -p /etc/systemd/system/docker.service.d/ + + echo "$ETC_ENVIRONMENT" >> /etc/environment + echo "$ETC_PROFILE_D" > /etc/profile.d/proxy_config.sh + echo "$DOCKER_PROXY" > etc/systemd/system/docker.service.d/http-proxy.conf + echo "proxy=$HTTP_PROXY" >> /etc/yum.conf +fi diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh index 1fe355789e19e..9f92e58d22d8b 100644 --- a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh +++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh @@ -20,6 +20,11 @@ set -o pipefail . /etc/sysconfig/heat-params +#Reads in profile, need to relax restrictions for some OSes. +set +o nounset +. /etc/profile +set -o nounset + rm -rf /kube-install mkdir -p /kube-install cd /kube-install diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index bd1cc4643b16f..0ee51535b79af 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -90,6 +90,36 @@ parameters: timeout for the Wait Conditions default: 6000 + enable_proxy: + type: string + description: Whether or not to enable proxy settings + default: false + + ftp_proxy: + type: string + description: FTP Proxy URL + default: localhost + + http_proxy: + type: string + description: HTTP Proxy URL + default: localhost + + https_proxy: + type: string + description: HTTPS Proxy URL + default: localhost + + socks_proxy: + type: string + description: SOCKS Proxy URL + default: localhost + + no_proxy: + type: string + description: Comma seperated list of domains/addresses that bypass proxying. + default: localhost + resources: master_wait_handle: @@ -180,6 +210,21 @@ resources: "$KUBERNETES_SALT_URL": {get_param: kubernetes_salt_url} "$MASTER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + proxy_config: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/configure-proxy.sh} + params: + "$ENABLE_PROXY": {get_param: enable_proxy } + "$FTP_PROXY": {get_param: ftp_proxy } + "$HTTP_PROXY": {get_param: http_proxy } + "$HTTPS_PROXY": {get_param: https_proxy } + "$SOCKS_PROXY": {get_param: socks_proxy } + "$NO_PROXY": {get_param: no_proxy } + kube_user: type: OS::Heat::SoftwareConfig properties: @@ -231,6 +276,7 @@ resources: properties: parts: - config: {get_resource: write_heat_params} + - config: {get_resource: proxy_config} - config: {get_resource: kube_user} - config: {get_resource: provision_network_master} - config: {get_resource: deploy_kube_auth_files_master} diff --git a/cluster/openstack/kubernetes-heat/kubeminion.yaml b/cluster/openstack/kubernetes-heat/kubeminion.yaml index f33dd2008fd00..83888ce0334da 100644 --- a/cluster/openstack/kubernetes-heat/kubeminion.yaml +++ b/cluster/openstack/kubernetes-heat/kubeminion.yaml @@ -41,6 +41,36 @@ parameters: type: string description: Token used by kube-proxy + enable_proxy: + type: string + description: Whether or not to enable proxy settings + default: false + + ftp_proxy: + type: string + description: FTP Proxy URL + default: localhost + + http_proxy: + type: string + description: HTTP Proxy URL + default: localhost + + https_proxy: + type: string + description: HTTPS Proxy URL + default: localhost + + socks_proxy: + type: string + description: SOCKS Proxy URL + default: localhost + + no_proxy: + type: string + description: Comma seperated list of domains/addresses that bypass proxying. + default: localhost + # The following are all generated in the parent template. kube_master_ip: type: string @@ -95,6 +125,21 @@ resources: "$KUBERNETES_SALT_URL": {get_param: kubernetes_salt_url} "$MASTER_IP": {get_param: kube_master_ip} + proxy_config: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/configure-proxy.sh} + params: + "$ENABLE_PROXY": {get_param: enable_proxy } + "$FTP_PROXY": {get_param: ftp_proxy } + "$HTTP_PROXY": {get_param: http_proxy } + "$HTTPS_PROXY": {get_param: https_proxy } + "$SOCKS_PROXY": {get_param: socks_proxy } + "$NO_PROXY": {get_param: no_proxy } + kube_user: type: OS::Heat::SoftwareConfig properties: @@ -144,6 +189,7 @@ resources: properties: parts: - config: {get_resource: write_heat_params} + - config: {get_resource: proxy_config} - config: {get_resource: kube_user} - config: {get_resource: provision_network_node} - config: {get_resource: deploy_kube_auth_files_node} diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index 266ff11149397..a93048e17893a 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -200,6 +200,12 @@ function run-heat-script() { -P dns_nameserver=${DNS_SERVER} \ -P kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \ -P kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \ + -P enable_proxy=${ENABLE_PROXY} \ + -P ftp_proxy="${FTP_PROXY}" \ + -P http_proxy="${HTTP_PROXY}" \ + -P https_proxy="${HTTPS_PROXY}" \ + -P socks_proxy="${SOCKS_PROXY}" \ + -P no_proxy="${NO_PROXY}" \ --template-file kubecluster.yaml \ ${STACK_NAME} ) From 126b6c0950f22110f3dc554673a7ba1096302bb2 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Wed, 13 Apr 2016 12:22:31 -0700 Subject: [PATCH 08/31] Fixed proxy settings for minion, added comments. --- cluster/openstack/config-default.sh | 1 + .../openstack/kubernetes-heat/fragments/configure-proxy.sh | 4 +++- cluster/openstack/kubernetes-heat/kubecluster.yaml | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cluster/openstack/config-default.sh b/cluster/openstack/config-default.sh index daad8d161d7f2..d2cc00c015145 100644 --- a/cluster/openstack/config-default.sh +++ b/cluster/openstack/config-default.sh @@ -53,4 +53,5 @@ CLIENT_PUBLIC_KEY_PATH=${CLIENT_PUBLIC_KEY_PATH:-~/.ssh/id_rsa.pub} # Max time period for stack provisioning. Time in minutes. STACK_CREATE_TIMEOUT=${STACK_CREATE_TIMEOUT:-60} +# Enable Proxy, if true kube-up will apply your current proxy settings(defined by *_PROXY environment variables) to the deployment. ENABLE_PROXY=${ENABLE_PROXY:-false} diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh b/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh index 60fba2f8efe8f..c6489239d80be 100644 --- a/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh +++ b/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh @@ -18,7 +18,7 @@ set -o errexit set -o nounset set -o pipefail -#These values are meant to be swapped in by heat +# The contents of these variables swapped in by heat via environments presented to kube-up.sh export ETC_ENVIRONMENT='FTP_PROXY=$FTP_PROXY HTTP_PROXY=$HTTP_PROXY @@ -56,8 +56,10 @@ export DOCKER_PROXY='[Service] Environment="no_proxy=$NO_PROXY" ' +# This again is set by heat ENABLE_PROXY='$ENABLE_PROXY' +# Heat itself doesn't have conditionals, so this is how we set up our proxy without breaking non-proxy setups. if [[ "${ENABLE_PROXY}" == "true" ]]; then mkdir -p /etc/systemd/system/docker.service.d/ diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index 0ee51535b79af..af385cd44a08c 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -354,6 +354,12 @@ resources: cluster_name: {get_param: "OS::stack_name"} secgroup_base: {get_resource: secgroup_base} secgroup_node: {get_resource: secgroup_node} + enable_proxy: {get_param: enable_proxy } + ftp_proxy: {get_param: ftp_proxy } + http_proxy: {get_param: http_proxy } + https_proxy: {get_param: https_proxy } + socks_proxy: {get_param: socks_proxy } + no_proxy: {get_param: no_proxy } min_size: {get_param: number_of_minions} desired_capacity: {get_param: number_of_minions} max_size: {get_param: max_number_of_minions} From 21627f5b7edd479f53e3ba4ad71ecc7939b6f1b2 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Wed, 13 Apr 2016 12:30:29 -0700 Subject: [PATCH 09/31] Made it so that existing Openstack environment variables are used. If someone has an openrc as part of their profile, this will make kube-up work automatically. The only things that have to be modified are in config-default.sh, either by editing the file or setting environment variables. --- cluster/openstack/openrc-default.sh | 13 +++++++------ cluster/openstack/openrc-swift.sh | 16 ++++++++-------- cluster/openstack/util.sh | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cluster/openstack/openrc-default.sh b/cluster/openstack/openrc-default.sh index 6e9fe67f237f7..9071e258f25bc 100644 --- a/cluster/openstack/openrc-default.sh +++ b/cluster/openstack/openrc-default.sh @@ -15,10 +15,11 @@ # limitations under the License. ## Enviroment variables for the OpenStack command-line client +## Values set via an openrc will override these defaults. -export OS_IDENTITY_API_VERSION=2.0 -export OS_USERNAME=admin -export OS_PASSWORD=secretsecret -export OS_AUTH_URL=http://192.168.123.100:5000/v2.0 -export OS_TENANT_NAME=admin -export OS_TENANT_ID=ed51b98b40944d89a449592eb67431eb +export OS_IDENTITY_API_VERSION=${OS_IDENTITY_API_VERSION:-2.0} +export OS_USERNAME=${OS_USERNAME:-admin} +export OS_PASSWORD=${OS_PASSWORD:-secretsecret} +export OS_AUTH_URL=${OS_AUTH_URL:-http://192.168.123.100:5000/v2.0} +export OS_TENANT_NAME=${OS_TENANT_NAME:-admin} +export OS_TENANT_ID=${OS_TENANT_ID:-ed51b98b40944d89a449592eb67431eb} diff --git a/cluster/openstack/openrc-swift.sh b/cluster/openstack/openrc-swift.sh index e8e69504df073..daad5704347d9 100644 --- a/cluster/openstack/openrc-swift.sh +++ b/cluster/openstack/openrc-swift.sh @@ -15,12 +15,12 @@ # limitations under the License. ## Enviroment variables for the OpenStack Swift command-line client. This is required for CityCloud -## provider where Swift has different credentials. When Swift is part of your OpenStack use the same -## settings as in openrc-default.sh +## provider where Swift has different credentials. When Swift is part of your OpenStack do not +## modify these settings. -export OS_IDENTITY_API_VERSION=2.0 -export OS_USERNAME=admin -export OS_PASSWORD=secretsecret -export OS_AUTH_URL=http://192.168.123.100:5000/v2.0 -export OS_TENANT_NAME=admin -export OS_TENANT_ID=ed51b98b40944d89a449592eb67431eb +export OS_IDENTITY_API_VERSION=${OS_IDENTITY_API_VERSION:-2.0} +export OS_USERNAME=${OS_USERNAME:-admin} +export OS_PASSWORD=${OS_PASSWORD:-secretsecret} +export OS_AUTH_URL=${OS_AUTH_URL:-http://192.168.123.100:5000/v2.0} +export OS_TENANT_NAME=${OS_TENANT_NAME:-admin} +export OS_TENANT_ID=${OS_TENANT_ID:-ed51b98b40944d89a449592eb67431eb} diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index a93048e17893a..b1a102cff0c25 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -91,7 +91,7 @@ function create-stack() { echo "[INFO] Execute commands to create Kubernetes cluster" # It is required for some cloud provider like CityCloud where swift client has different credentials source "${ROOT}/openrc-swift.sh" - if [[ -z "${OS_PROJECT_ID}" ]]; then + if [[ -z ${OS_PROJECT_ID+x} ]]; then SWIFT_PROJECT_ID="${OS_TENANT_ID}" else SWIFT_PROJECT_ID="${OS_PROJECT_ID}" From 5ae161e0862ec3911f0aac1b8f40967ae0131d25 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Wed, 13 Apr 2016 16:40:38 -0700 Subject: [PATCH 10/31] Adding cloudprovider support for Openstack kube-up provider. Making the assumption that the person running kube-up has their Openstack environment setup, those same variables are being passed into heat, and then into openstack.conf. The salt codebase was modified to add openstack as well. --- .../fragments/configure-salt.yaml | 11 +++++- .../kubernetes-heat/fragments/run-salt.sh | 3 -- .../kubernetes-heat/kubecluster.yaml | 35 +++++++++++++++++++ .../openstack/kubernetes-heat/kubeminion.yaml | 30 ++++++++++++++++ cluster/openstack/util.sh | 5 +++ cluster/saltbase/salt/kube-apiserver/init.sls | 2 +- .../kube-apiserver/kube-apiserver.manifest | 8 +++-- .../kube-controller-manager.manifest | 6 +++- .../salt/kube-proxy/kube-proxy.manifest | 2 +- cluster/saltbase/salt/kubelet/default | 11 ++++-- cluster/saltbase/salt/top.sls | 6 +++- 11 files changed, 106 insertions(+), 13 deletions(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml index aa20df28fb8ba..f137e6c73a678 100644 --- a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml @@ -16,13 +16,22 @@ write_files: network_mode: openvswitch networkInterfaceName: eth0 api_servers: $MASTER_IP - cloud: vagrant # It's not vagrant, but required to install Docker + cloud: openstack + cloud_config: /srv/kubernetes/openstack.conf roles: - $role runtime_config: "" docker_opts: "" master_extra_sans: "DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:kubernetes-master" keep_host_etcd: true + - path: /srv/kubernetes/openstack.conf + content: | + [Global] + auth-url=$OS_AUTH_URL + username=$OS_USERNAME + password=$OS_PASSWORD + region=$OS_REGION_NAME + tenant-id=$OS_TENANT_ID - path: /srv/salt-overlay/pillar/cluster-params.sls content: | service_cluster_ip_range: 10.246.0.0/16 diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh index 9f92e58d22d8b..65f018f76538b 100644 --- a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh +++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh @@ -43,9 +43,6 @@ fi # Salt server runs at locahost echo "127.0.0.1 salt" >> /etc/hosts -# Currently heat template tells a lie that the target is Vagrant. If Vagrant cloud provider is enabled, "Unable to construct api.Node object for kubelet" error will occur. -sed -e 's/{{cloud_provider}}//' -i /srv/salt/kubelet/default - # Run salt-call # salt-call wants to start docker daemon but is unable to. # See . diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index af385cd44a08c..47497606ed39f 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -90,6 +90,31 @@ parameters: timeout for the Wait Conditions default: 6000 + os_auth_url: + type: string + description: OpenStack Auth URL + default: false + + os_username: + type: string + description: OpenStack Username + default: false + + os_password: + type: string + description: OpenStack Password + default: false + + os_region_name: + type: string + description: OpenStack Region Name + default: false + + os_tenant_id: + type: string + description: OpenStack Tenant ID + default: false + enable_proxy: type: string description: Whether or not to enable proxy settings @@ -259,6 +284,11 @@ resources: template: {get_file: fragments/configure-salt.yaml} params: "$MASTER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + "$OS_AUTH_URL": {get_param: os_auth_url} + "$OS_USERNAME": {get_param: os_username} + "$OS_PASSWORD": {get_param: os_password} + "$OS_REGION_NAME": {get_param: os_region_name} + "$OS_TENANT_ID": {get_param: os_tenant_id} "$role": "kubernetes-master" run_salt: @@ -354,6 +384,11 @@ resources: cluster_name: {get_param: "OS::stack_name"} secgroup_base: {get_resource: secgroup_base} secgroup_node: {get_resource: secgroup_node} + os_auth_url: {get_param: os_auth_url} + os_username: {get_param: os_username} + os_password: {get_param: os_password} + os_region_name: {get_param: os_region_name} + os_tenant_id: {get_param: os_tenant_id} enable_proxy: {get_param: enable_proxy } ftp_proxy: {get_param: ftp_proxy } http_proxy: {get_param: http_proxy } diff --git a/cluster/openstack/kubernetes-heat/kubeminion.yaml b/cluster/openstack/kubernetes-heat/kubeminion.yaml index 83888ce0334da..0e34e9b51b4f2 100644 --- a/cluster/openstack/kubernetes-heat/kubeminion.yaml +++ b/cluster/openstack/kubernetes-heat/kubeminion.yaml @@ -41,6 +41,31 @@ parameters: type: string description: Token used by kube-proxy + os_auth_url: + type: string + description: OpenStack Auth URL + default: false + + os_username: + type: string + description: OpenStack Username + default: false + + os_password: + type: string + description: OpenStack Password + default: false + + os_region_name: + type: string + description: OpenStack Region Name + default: false + + os_tenant_id: + type: string + description: OpenStack Tenant ID + default: false + enable_proxy: type: string description: Whether or not to enable proxy settings @@ -172,6 +197,11 @@ resources: template: {get_file: fragments/configure-salt.yaml} params: "$MASTER_IP": {get_param: kube_master_ip} + "$OS_AUTH_URL": {get_param: os_auth_url} + "$OS_USERNAME": {get_param: os_username} + "$OS_PASSWORD": {get_param: os_password} + "$OS_REGION_NAME": {get_param: os_region_name} + "$OS_TENANT_ID": {get_param: os_tenant_id} "$role": "kubernetes-pool" run_salt: diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index b1a102cff0c25..dd8014fd2c162 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -200,6 +200,11 @@ function run-heat-script() { -P dns_nameserver=${DNS_SERVER} \ -P kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \ -P kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \ + -P os_auth_url=${OS_AUTH_URL} \ + -P os_username=${OS_USERNAME} \ + -P os_password=${OS_PASSWORD} \ + -P os_region_name=${OS_REGION_NAME} \ + -P os_tenant_id=${OS_TENANT_ID} \ -P enable_proxy=${ENABLE_PROXY} \ -P ftp_proxy="${FTP_PROXY}" \ -P http_proxy="${HTTP_PROXY}" \ diff --git a/cluster/saltbase/salt/kube-apiserver/init.sls b/cluster/saltbase/salt/kube-apiserver/init.sls index fff8464887704..2329437dbd1be 100644 --- a/cluster/saltbase/salt/kube-apiserver/init.sls +++ b/cluster/saltbase/salt/kube-apiserver/init.sls @@ -1,4 +1,4 @@ -{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] %} +{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack] %} # TODO: generate and distribute tokens on other cloud providers. /srv/kubernetes/known_tokens.csv: file.managed: diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index a8ac7239e3866..5dd4eaf8e22e6 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -18,6 +18,10 @@ {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} + {% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} + {% endif -%} + {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} @@ -58,7 +62,7 @@ {% set client_ca_file = "" -%} {% set secure_port = "6443" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set secure_port = "443" -%} {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} @@ -72,7 +76,7 @@ {% set basic_auth_file = "" -%} {% set authz_mode = "" -%} {% set abac_policy_file = "" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere'] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack'] %} {% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%} {% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%} {% set authz_mode = " --authorization-mode=ABAC" -%} diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index cbf650d25659c..e7fc51803bf88 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -37,6 +37,10 @@ {% endif -%} {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%} + {% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} + {% endif -%} + {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} @@ -46,7 +50,7 @@ {% set root_ca_file = "" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index c5f8f2d809451..e49491b67c727 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -5,7 +5,7 @@ {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} {% set api_servers = "--master=https://" + ips[0][0] -%} {% endif -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack' ] %} {% set api_servers_with_port = api_servers -%} {% else -%} {% set api_servers_with_port = api_servers + ":6443" -%} diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 5462c761a0374..255faeb59dfaf 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -16,7 +16,7 @@ {% endif -%} # TODO: remove nginx for other cloud providers. -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set api_servers_with_port = api_servers -%} {% else -%} {% set api_servers_with_port = api_servers + ":6443" -%} @@ -28,7 +28,7 @@ {% set reconcile_cidr_args = "" -%} {% if grains['roles'][0] == 'kubernetes-master' -%} - {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller'] -%} + {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] -%} # Unless given a specific directive, disable registration for the kubelet # running on the master. @@ -52,6 +52,11 @@ {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} +{% set cloud_config = "" -%} +{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} +{% endif -%} + {% set config = "--config=/etc/kubernetes/manifests" -%} {% set manifest_url = "" -%} @@ -192,4 +197,4 @@ {% endif -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}" diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 4b84cefccf5da..2bb99b7b084a9 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -58,6 +58,10 @@ base: - kube-controller-manager - kube-scheduler - supervisor +{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack' ] %} + - nginx +{% endif %} + - cadvisor - kube-client-tools - kube-master-addons - kube-admission-controls @@ -72,7 +76,7 @@ base: - logrotate {% endif %} - kube-addons -{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller', 'openstack'] %} - docker - kubelet {% endif %} From 2ffc86bc1dfcc28fb5ab1ad76de0e41e0b269d62 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Fri, 15 Apr 2016 12:45:57 -0700 Subject: [PATCH 11/31] Adding automatic OS image download for Openstack provider. This makes it so that we download the OS image automatically. Also contains other usability improvements: - kubectl context created with heat stack name - Bumped default minions to 3 --- cluster/openstack/config-default.sh | 7 +++++-- cluster/openstack/config-image.sh | 13 ++++++++----- cluster/openstack/kubernetes-heat/kubecluster.yaml | 4 ++-- cluster/openstack/util.sh | 8 ++++++-- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cluster/openstack/config-default.sh b/cluster/openstack/config-default.sh index d2cc00c015145..034f73ca613ec 100644 --- a/cluster/openstack/config-default.sh +++ b/cluster/openstack/config-default.sh @@ -25,9 +25,9 @@ KUBERNETES_KEYPAIR_NAME=${KUBERNETES_KEYPAIR_NAME:-kubernetes_keypair} # Kubernetes release tar file KUBERNETES_RELEASE_TAR=${KUBERNETES_RELEASE_TAR:-kubernetes-server-linux-amd64.tar.gz} -NUMBER_OF_MINIONS=${NUMBER_OF_MINIONS-1} +NUMBER_OF_MINIONS=${NUMBER_OF_MINIONS-3} -MAX_NUMBER_OF_MINIONS=${MAX_NUMBER_OF_MINIONS:-1} +MAX_NUMBER_OF_MINIONS=${MAX_NUMBER_OF_MINIONS:-3} MASTER_FLAVOR=${MASTER_FLAVOR:-m1.small} @@ -41,6 +41,9 @@ SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-http://192.168.123.100:8080} # If 'true' then new image will be created from file config-image.sh CREATE_IMAGE=${CREATE_IMAGE:-true} # use "true" for devstack +# Flag indicates if image should be downloaded +DOWNLOAD_IMAGE=${DOWNLOAD_IMAGE:-true} + # Image id which will be used for kubernetes stack IMAGE_ID=${IMAGE_ID:-f0f394b1-5546-4b68-b2bc-8abe8a7e6b8b} diff --git a/cluster/openstack/config-image.sh b/cluster/openstack/config-image.sh index d62e80d75c100..41524bf17bcb9 100644 --- a/cluster/openstack/config-image.sh +++ b/cluster/openstack/config-image.sh @@ -17,16 +17,19 @@ ## Contains configuration values for new image. It is skip when CREATE_IMAGE=false # Image name which will be displayed in OpenStack -OPENSTACK_IMAGE_NAME="CentOS7" +OPENSTACK_IMAGE_NAME=${OPENSTACK_IMAGE_NAME:-CentOS7} # Downloaded image name for Openstack project -IMAGE_FILE="CentOS-7-x86_64-GenericCloud-1510.qcow2" +IMAGE_FILE=${IMAGE_FILE:-CentOS-7-x86_64-GenericCloud-1510.qcow2} # Absolute path where image file is stored. -IMAGE_PATH="/home/openstack/openstack_temp" +IMAGE_PATH=${IMAGE_PATH:-~/Downloads/openstack} + +# The URL basepath for downloading the image +IMAGE_URL_PATH=${IMAGE_URL_PATH:-http://cloud.centos.org/centos/7/images} # The disk format of the image. Acceptable formats are ami, ari, aki, vhd, vmdk, raw, qcow2, vdi, and iso. -IMAGE_FORMAT="qcow2" +IMAGE_FORMAT=${IMAGE_FORMAT:-qcow2} # The container format of the image. Acceptable formats are ami, ari, aki, bare, docker, and ovf. -CONTAINER_FORMAT="bare" +CONTAINER_FORMAT=${CONTAINER_FORMAT:-bare} diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index 47497606ed39f..e57df48272760 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -2,7 +2,7 @@ heat_template_version: 2014-10-16 description: > Kubernetes cluster with one master and one or more worker nodes - (as specified by the number_of_minions parameter, which defaults to 2). + (as specified by the number_of_minions parameter, which defaults to 3). parameters: plugin_version: @@ -44,7 +44,7 @@ parameters: number_of_minions: type: number description: how many kubernetes minions to spawn initially - default: 1 + default: 3 max_number_of_minions: type: number diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index dd8014fd2c162..f689720005a8f 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -144,10 +144,14 @@ function add-keypair() { # IMAGE_PATH # OPENSTACK_IMAGE_NAME function create-glance-image() { - if [ $CREATE_IMAGE = true ]; then + if [[ ${CREATE_IMAGE} == "true" ]]; then local image_status=$(nova image-show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') if [[ ! $image_status ]]; then + if [[ "${DOWNLOAD_IMAGE}" == "true" ]]; then + mkdir -p ${IMAGE_PATH} + curl -L ${IMAGE_URL_PATH}/${IMAGE_FILE} -o ${IMAGE_PATH}/${IMAGE_FILE} -z ${IMAGE_PATH}/${IMAGE_FILE} + fi echo "[INFO] Create image ${OPENSTACK_IMAGE_NAME}" glance image-create --name ${OPENSTACK_IMAGE_NAME} --disk-format ${IMAGE_FORMAT} \ --container-format ${CONTAINER_FORMAT} --file ${IMAGE_PATH}/${IMAGE_FILE} @@ -227,7 +231,7 @@ function run-heat-script() { function configure-kubectl() { export KUBE_MASTER_IP=$(nova show "${STACK_NAME}"-master | awk '$3=="network" {print $6}') - export CONTEXT="openstack" + export CONTEXT="openstack-${STACK_NAME}" export KUBE_BEARER_TOKEN="TokenKubelet" create-kubeconfig } From 707ef2ce762e25699ff5356a80e3ba99b7d47a46 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Mon, 18 Apr 2016 09:16:31 -0700 Subject: [PATCH 12/31] Adding missing variable defaults for OpenStack provider. --- cluster/openstack/config-default.sh | 9 +++++++++ cluster/openstack/openrc-default.sh | 1 + cluster/openstack/openrc-swift.sh | 1 + 3 files changed, 11 insertions(+) diff --git a/cluster/openstack/config-default.sh b/cluster/openstack/config-default.sh index 034f73ca613ec..98f4bc5b662cc 100644 --- a/cluster/openstack/config-default.sh +++ b/cluster/openstack/config-default.sh @@ -58,3 +58,12 @@ STACK_CREATE_TIMEOUT=${STACK_CREATE_TIMEOUT:-60} # Enable Proxy, if true kube-up will apply your current proxy settings(defined by *_PROXY environment variables) to the deployment. ENABLE_PROXY=${ENABLE_PROXY:-false} + +# Per-protocol proxy settings. +FTP_PROXY=${FTP_PROXY:-} +HTTP_PROXY=${HTTP_PROXY:-} +HTTPS_PROXY=${HTTPS_PROXY:-} +SOCKS_PROXY=${SOCKS_PROXY:-} + +# IPs and Domains that bypass the proxy. +NO_PROXY=${NO_PROXY:-} diff --git a/cluster/openstack/openrc-default.sh b/cluster/openstack/openrc-default.sh index 9071e258f25bc..2583d5c2cd10d 100644 --- a/cluster/openstack/openrc-default.sh +++ b/cluster/openstack/openrc-default.sh @@ -23,3 +23,4 @@ export OS_PASSWORD=${OS_PASSWORD:-secretsecret} export OS_AUTH_URL=${OS_AUTH_URL:-http://192.168.123.100:5000/v2.0} export OS_TENANT_NAME=${OS_TENANT_NAME:-admin} export OS_TENANT_ID=${OS_TENANT_ID:-ed51b98b40944d89a449592eb67431eb} +export OS_REGION_NAME=${OS_REGION_NAME:-RegionOne} diff --git a/cluster/openstack/openrc-swift.sh b/cluster/openstack/openrc-swift.sh index daad5704347d9..8ba761f0d1a62 100644 --- a/cluster/openstack/openrc-swift.sh +++ b/cluster/openstack/openrc-swift.sh @@ -24,3 +24,4 @@ export OS_PASSWORD=${OS_PASSWORD:-secretsecret} export OS_AUTH_URL=${OS_AUTH_URL:-http://192.168.123.100:5000/v2.0} export OS_TENANT_NAME=${OS_TENANT_NAME:-admin} export OS_TENANT_ID=${OS_TENANT_ID:-ed51b98b40944d89a449592eb67431eb} +export OS_REGION_NAME=${OS_REGION_NAME:-RegionOne} From 8315734babf33c3b297c9cfcfc743b15371f56ff Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Mon, 18 Apr 2016 09:28:21 -0700 Subject: [PATCH 13/31] Adding openstack example to get-kube.sh --- cluster/get-kube.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/get-kube.sh b/cluster/get-kube.sh index f74b752c1d877..5b7b3c696543e 100755 --- a/cluster/get-kube.sh +++ b/cluster/get-kube.sh @@ -38,6 +38,8 @@ # * export KUBERNETES_PROVIDER=photon-controller; wget -q -O - https://get.k8s.io | bash # Rackspace # * export KUBERNETES_PROVIDER=rackspace; wget -q -O - https://get.k8s.io | bash +# OpenStack +# * export KUBERNETES_PROVIDER=openstack; wget -q -O - https://get.k8s.io | bash # # Set KUBERNETES_SKIP_DOWNLOAD to non-empty to skip downloading a release. # Set KUBERNETES_SKIP_CONFIRM to skip the installation confirmation prompt. From 4773808767d005b4a0f03bddbfae23b2ea5b83bc Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Wed, 20 Apr 2016 17:00:42 -0700 Subject: [PATCH 14/31] Improved user experience for users of OpenStack provider behind a proxy. --- cluster/kube-up.sh | 9 +++++++++ cluster/openstack/util.sh | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/cluster/kube-up.sh b/cluster/kube-up.sh index 4e179c361778b..12862d5f1d877 100755 --- a/cluster/kube-up.sh +++ b/cluster/kube-up.sh @@ -64,6 +64,15 @@ elif [[ "${validate_result}" == "2" ]]; then echo "...ignoring non-fatal errors in validate-cluster" >&2 fi +if [[ "${ENABLE_PROXY:-}" == "true" ]]; then + . /tmp/kube-proxy-env + echo "" + echo "*** Please run the following to add the kube-apiserver endpoint to your proxy white-list ***" + cat /tmp/kube-proxy-env + echo "*** ***" + echo "" +fi + echo -e "Done, listing cluster services:\n" >&2 "${KUBE_ROOT}/cluster/kubectl.sh" cluster-info echo diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index f689720005a8f..a0652b9203bd5 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -233,6 +233,13 @@ function configure-kubectl() { export KUBE_MASTER_IP=$(nova show "${STACK_NAME}"-master | awk '$3=="network" {print $6}') export CONTEXT="openstack-${STACK_NAME}" export KUBE_BEARER_TOKEN="TokenKubelet" + + if [[ "${ENABLE_PROXY:-}" == "true" ]]; then + echo 'export NO_PROXY=$NO_PROXY,'"${KUBE_MASTER_IP}" > /tmp/kube-proxy-env + echo 'export no_proxy=$NO_PROXY,'"${KUBE_MASTER_IP}" >> /tmp/kube-proxy-env + . /tmp/kube-proxy-env + fi + create-kubeconfig } From d28fd393cf02905c58aaeb53f441ee33c9843905 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Thu, 21 Apr 2016 10:07:19 -0700 Subject: [PATCH 15/31] Enabled Addons for Openstack kube-up provider. Hard-coded the enabling of the common addons: - logging - kube-dashboard - monitoring Will make it configureable in a subsequent PR. Also need to enable configuration of basic_auth.csv --- .../kubernetes-heat/fragments/configure-salt.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml index f137e6c73a678..c996d39482cb7 100644 --- a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml @@ -36,10 +36,10 @@ write_files: content: | service_cluster_ip_range: 10.246.0.0/16 cert_ip: 10.246.0.1 - enable_cluster_monitoring: none - enable_cluster_logging: "false" - enable_cluster_ui: "false" - enable_node_logging: "false" + enable_cluster_monitoring: influxdb + enable_cluster_logging: "true" + enable_cluster_ui: "true" + enable_node_logging: "true" logging_destination: elasticsearch elasticsearch_replicas: "1" enable_cluster_dns: "true" From 35bbc186aa83d96a4524dd42524140d12fade600 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Thu, 21 Apr 2016 18:23:01 -0700 Subject: [PATCH 16/31] Increasing default node size for kube-up Openstack provider. The default pod sizes are too small for some Java apps. Also the logging/monitoring pods were taking up too many resources. --- cluster/openstack/config-default.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster/openstack/config-default.sh b/cluster/openstack/config-default.sh index 98f4bc5b662cc..643fb5e0a49e0 100644 --- a/cluster/openstack/config-default.sh +++ b/cluster/openstack/config-default.sh @@ -29,9 +29,9 @@ NUMBER_OF_MINIONS=${NUMBER_OF_MINIONS-3} MAX_NUMBER_OF_MINIONS=${MAX_NUMBER_OF_MINIONS:-3} -MASTER_FLAVOR=${MASTER_FLAVOR:-m1.small} +MASTER_FLAVOR=${MASTER_FLAVOR:-m1.medium} -MINION_FLAVOR=${MINION_FLAVOR:-m1.small} +MINION_FLAVOR=${MINION_FLAVOR:-m1.medium} EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-public} From a4e6e7fa47d92683fe201a25c2835fce3f08aa65 Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Fri, 22 Apr 2016 13:42:39 +0200 Subject: [PATCH 17/31] Add configuration for e2e tests --- cluster/openstack/config-test.sh | 19 +++++++++++++++++++ cluster/openstack/util.sh | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 cluster/openstack/config-test.sh diff --git a/cluster/openstack/config-test.sh b/cluster/openstack/config-test.sh new file mode 100644 index 0000000000000..1ed3e5eedf4ca --- /dev/null +++ b/cluster/openstack/config-test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Contains configuration values for interacting with the Ubuntu cluster in test mode +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +source "${KUBE_ROOT}/cluster/openstack/config-default.sh" diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index a0652b9203bd5..7afb4f22019c6 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -252,3 +252,23 @@ function kube-down { source "${ROOT}/openrc-default.sh" heat stack-delete ${STACK_NAME} } + +# Perform preparations required to run e2e tests +function prepare-e2e { + echo "TODO: prepare-e2e" 1>&2 +} + +function test-build-release { + echo "test-build-release() " 1>&2 +} + +# Must ensure that the following ENV vars are set +function detect-master { + + source "${ROOT}/${KUBE_CONFIG_FILE:-"config-default.sh"}" + source "${ROOT}/openrc-default.sh" + + export KUBE_MASTER_IP=$(nova show "${STACK_NAME}"-master | awk '$3=="network" {print $6}') + + echo "KUBE_MASTER_IP: $KUBE_MASTER_IP" 1>&2 +} From 7f37ab50341fea284b3fd05132a438a663e50d92 Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Mon, 25 Apr 2016 10:40:23 +0900 Subject: [PATCH 18/31] Remove unused code --- cluster/openstack/kubernetes-heat/kubecluster.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index e57df48272760..fdd3eaee77ed7 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -5,14 +5,6 @@ description: > (as specified by the number_of_minions parameter, which defaults to 3). parameters: - plugin_version: - type: string - description: version of CLC plugin. It is used when you show cluster list view. - default: 1.0 - constraints: - - allowed_values: - - 1.0 - ssh_key_name: type: string description: name of ssh key to be provisioned on our server From 23bcbbfacab3a7c5374d785b057fd4f29f6c42ae Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Mon, 25 Apr 2016 10:40:57 +0900 Subject: [PATCH 19/31] Add braces around environment variable names Parameter names of Heat str_replace remain unchanged --- .../kubernetes-heat/fragments/configure-proxy.sh | 6 +++--- .../kubernetes-heat/fragments/run-salt.sh | 4 ++-- cluster/openstack/util.sh | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh b/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh index c6489239d80be..3b4ddcc3e301c 100644 --- a/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh +++ b/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh @@ -63,8 +63,8 @@ ENABLE_PROXY='$ENABLE_PROXY' if [[ "${ENABLE_PROXY}" == "true" ]]; then mkdir -p /etc/systemd/system/docker.service.d/ - echo "$ETC_ENVIRONMENT" >> /etc/environment - echo "$ETC_PROFILE_D" > /etc/profile.d/proxy_config.sh - echo "$DOCKER_PROXY" > etc/systemd/system/docker.service.d/http-proxy.conf + echo "${ETC_ENVIRONMENT}" >> /etc/environment + echo "${ETC_PROFILE_D}" > /etc/profile.d/proxy_config.sh + echo "${DOCKER_PROXY}" > etc/systemd/system/docker.service.d/http-proxy.conf echo "proxy=$HTTP_PROXY" >> /etc/yum.conf fi diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh index 65f018f76538b..e879c0384ab35 100644 --- a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh +++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh @@ -29,8 +29,8 @@ rm -rf /kube-install mkdir -p /kube-install cd /kube-install -curl "$KUBERNETES_SERVER_URL" -o kubernetes-server.tar.gz -curl "$KUBERNETES_SALT_URL" -o kubernetes-salt.tar.gz +curl "${KUBERNETES_SERVER_URL}" -o kubernetes-server.tar.gz +curl "${KUBERNETES_SALT_URL}" -o kubernetes-salt.tar.gz tar xzf kubernetes-salt.tar.gz ./kubernetes/saltbase/install.sh kubernetes-server.tar.gz diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index 7afb4f22019c6..6aa8082b01a4a 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -24,7 +24,7 @@ set -e KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. readonly ROOT=$(dirname "${BASH_SOURCE}") source "${ROOT}/${KUBE_CONFIG_FILE:-"config-default.sh"}" -source "$KUBE_ROOT/cluster/common.sh" +source "${KUBE_ROOT}/cluster/common.sh" if [ $CREATE_IMAGE = true ]; then source "${ROOT}/config-image.sh" fi @@ -35,10 +35,10 @@ function verify-prereqs() { for client in swift glance nova heat; do if which $client >/dev/null 2>&1; then - echo "$client client installed" + echo "${client} client installed" else - echo "$client client does not exist" - echo "Please install $client client, and retry." + echo "${client} client does not exist" + echo "Please install ${client} client, and retry." exit 1 fi done @@ -49,7 +49,7 @@ function verify-prereqs() { # Assumed vars: # KUBERNETES_PROVIDER function kube-up() { - echo "kube-up for provider $KUBERNETES_PROVIDER" + echo "kube-up for provider ${KUBERNETES_PROVIDER}" create-stack } @@ -186,7 +186,7 @@ function run-heat-script() { if [ $CREATE_IMAGE = true ]; then echo "[INFO] Retrieve new image ID" IMAGE_ID=$(nova image-show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') - echo "[INFO] Image Id $IMAGE_ID" + echo "[INFO] Image Id ${IMAGE_ID}" fi if [[ ! $stack_status ]]; then @@ -270,5 +270,5 @@ function detect-master { export KUBE_MASTER_IP=$(nova show "${STACK_NAME}"-master | awk '$3=="network" {print $6}') - echo "KUBE_MASTER_IP: $KUBE_MASTER_IP" 1>&2 + echo "KUBE_MASTER_IP: ${KUBE_MASTER_IP}" 1>&2 } From 5ea2940cfe722b369f175c36d4dfe3a39a6dc96a Mon Sep 17 00:00:00 2001 From: OHTAKE Tomohiro Date: Mon, 25 Apr 2016 10:43:30 +0900 Subject: [PATCH 20/31] Rename the provider to openstack-heat There might be several methods to deploy a Kubernetes cluster on OpenStack. It is better to name the provider more specific. --- cluster/{openstack => openstack-heat}/config-default.sh | 0 cluster/{openstack => openstack-heat}/config-image.sh | 0 cluster/{openstack => openstack-heat}/config-test.sh | 2 +- .../kubernetes-heat/fragments/configure-proxy.sh | 0 .../kubernetes-heat/fragments/configure-salt.yaml | 0 .../fragments/deploy-kube-auth-files-master.yaml | 0 .../kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml | 0 .../kubernetes-heat/fragments/kube-user.yaml | 0 .../kubernetes-heat/fragments/provision-network-master.sh | 0 .../kubernetes-heat/fragments/provision-network-node.sh | 0 .../kubernetes-heat/fragments/run-salt.sh | 0 .../kubernetes-heat/fragments/write-heat-params.yaml | 0 .../kubernetes-heat/kubecluster.yaml | 0 .../kubernetes-heat/kubeminion.yaml | 0 cluster/{openstack => openstack-heat}/openrc-default.sh | 0 cluster/{openstack => openstack-heat}/openrc-swift.sh | 0 cluster/{openstack => openstack-heat}/util.sh | 0 17 files changed, 1 insertion(+), 1 deletion(-) rename cluster/{openstack => openstack-heat}/config-default.sh (100%) rename cluster/{openstack => openstack-heat}/config-image.sh (100%) rename cluster/{openstack => openstack-heat}/config-test.sh (92%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/configure-proxy.sh (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/configure-salt.yaml (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/kube-user.yaml (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/provision-network-master.sh (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/provision-network-node.sh (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/run-salt.sh (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/fragments/write-heat-params.yaml (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/kubecluster.yaml (100%) rename cluster/{openstack => openstack-heat}/kubernetes-heat/kubeminion.yaml (100%) rename cluster/{openstack => openstack-heat}/openrc-default.sh (100%) rename cluster/{openstack => openstack-heat}/openrc-swift.sh (100%) rename cluster/{openstack => openstack-heat}/util.sh (100%) diff --git a/cluster/openstack/config-default.sh b/cluster/openstack-heat/config-default.sh similarity index 100% rename from cluster/openstack/config-default.sh rename to cluster/openstack-heat/config-default.sh diff --git a/cluster/openstack/config-image.sh b/cluster/openstack-heat/config-image.sh similarity index 100% rename from cluster/openstack/config-image.sh rename to cluster/openstack-heat/config-image.sh diff --git a/cluster/openstack/config-test.sh b/cluster/openstack-heat/config-test.sh similarity index 92% rename from cluster/openstack/config-test.sh rename to cluster/openstack-heat/config-test.sh index 1ed3e5eedf4ca..0787d55e7bb3c 100644 --- a/cluster/openstack/config-test.sh +++ b/cluster/openstack-heat/config-test.sh @@ -16,4 +16,4 @@ ## Contains configuration values for interacting with the Ubuntu cluster in test mode KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -source "${KUBE_ROOT}/cluster/openstack/config-default.sh" +source "${KUBE_ROOT}/cluster/openstack-heat/config-default.sh" diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh b/cluster/openstack-heat/kubernetes-heat/fragments/configure-proxy.sh similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/configure-proxy.sh rename to cluster/openstack-heat/kubernetes-heat/fragments/configure-proxy.sh diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml rename to cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml diff --git a/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml rename to cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml diff --git a/cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml rename to cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml diff --git a/cluster/openstack/kubernetes-heat/fragments/kube-user.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/kube-user.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/kube-user.yaml rename to cluster/openstack-heat/kubernetes-heat/fragments/kube-user.yaml diff --git a/cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh b/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/provision-network-master.sh rename to cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh diff --git a/cluster/openstack/kubernetes-heat/fragments/provision-network-node.sh b/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/provision-network-node.sh rename to cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack-heat/kubernetes-heat/fragments/run-salt.sh similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/run-salt.sh rename to cluster/openstack-heat/kubernetes-heat/fragments/run-salt.sh diff --git a/cluster/openstack/kubernetes-heat/fragments/write-heat-params.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/write-heat-params.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/fragments/write-heat-params.yaml rename to cluster/openstack-heat/kubernetes-heat/fragments/write-heat-params.yaml diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/kubecluster.yaml rename to cluster/openstack-heat/kubernetes-heat/kubecluster.yaml diff --git a/cluster/openstack/kubernetes-heat/kubeminion.yaml b/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml similarity index 100% rename from cluster/openstack/kubernetes-heat/kubeminion.yaml rename to cluster/openstack-heat/kubernetes-heat/kubeminion.yaml diff --git a/cluster/openstack/openrc-default.sh b/cluster/openstack-heat/openrc-default.sh similarity index 100% rename from cluster/openstack/openrc-default.sh rename to cluster/openstack-heat/openrc-default.sh diff --git a/cluster/openstack/openrc-swift.sh b/cluster/openstack-heat/openrc-swift.sh similarity index 100% rename from cluster/openstack/openrc-swift.sh rename to cluster/openstack-heat/openrc-swift.sh diff --git a/cluster/openstack/util.sh b/cluster/openstack-heat/util.sh similarity index 100% rename from cluster/openstack/util.sh rename to cluster/openstack-heat/util.sh From 001678991be1974e403829eec49def7380cc82de Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Fri, 22 Apr 2016 11:30:10 -0700 Subject: [PATCH 21/31] Removing an invalid reference to cadvisor from salt. It looks like this was left over from https://github.com/kubernetes/kubernetes/pull/23316 --- cluster/saltbase/salt/top.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 2bb99b7b084a9..cacf5fd5fc907 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -61,7 +61,6 @@ base: {% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack' ] %} - nginx {% endif %} - - cadvisor - kube-client-tools - kube-master-addons - kube-admission-controls From 545b230851b2a64d6e7a7a75c9d9197a2332e8d3 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Fri, 22 Apr 2016 19:43:04 -0700 Subject: [PATCH 22/31] Adding socat package for Redhat systems in Salt --- cluster/saltbase/salt/base.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/saltbase/salt/base.sls b/cluster/saltbase/salt/base.sls index a8eac1eb89b9f..295e46b689ff9 100644 --- a/cluster/saltbase/salt/base.sls +++ b/cluster/saltbase/salt/base.sls @@ -5,6 +5,7 @@ pkg-core: {% if grains['os_family'] == 'RedHat' %} - python - git + - socat {% else %} - apt-transport-https - python-apt From 9226cf7460b15854351d476e87445281461fd311 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Fri, 22 Apr 2016 21:48:28 -0700 Subject: [PATCH 23/31] Undoing regression of #23311 --- cluster/saltbase/salt/top.sls | 3 --- 1 file changed, 3 deletions(-) diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index cacf5fd5fc907..1717b4d3ea3f6 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -58,9 +58,6 @@ base: - kube-controller-manager - kube-scheduler - supervisor -{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack' ] %} - - nginx -{% endif %} - kube-client-tools - kube-master-addons - kube-admission-controls From 7b3b24c4f5975f6919fbdcba756bfaa40b703665 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Sat, 23 Apr 2016 18:08:45 -0700 Subject: [PATCH 24/31] Triages #7 for OpenStack provider, still needs work. Fixed the order of fields for basic_auth. This provider still needs to leverage common.sh for generating proper credentials though. Also documented a pattern for how to get the SWIFT_SERVER_URL automatically --- cluster/openstack-heat/config-default.sh | 1 + .../fragments/deploy-kube-auth-files-master.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/openstack-heat/config-default.sh b/cluster/openstack-heat/config-default.sh index 643fb5e0a49e0..480e6f43059e8 100644 --- a/cluster/openstack-heat/config-default.sh +++ b/cluster/openstack-heat/config-default.sh @@ -35,6 +35,7 @@ MINION_FLAVOR=${MINION_FLAVOR:-m1.medium} EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-public} +#FIXME This can get set by evaulating the output of `swift auth` and stripping the base URL from OS_STORAGE_URL, but will require a refactor. Till then... SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-http://192.168.123.100:8080} # Flag indicates if new image must be created. If 'false' then image with IMAGE_ID will be used. diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml index 10ba9ae886166..a791292764257 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml @@ -7,7 +7,7 @@ write_files: - path: /srv/salt-overlay/salt/kube-apiserver/basic_auth.csv permissions: "0600" content: | - $apiserver_user,$apiserver_password,admin + $apiserver_password,$apiserver_user,admin - path: /srv/salt-overlay/salt/kube-apiserver/known_tokens.csv permissions: "0600" content: | From c5ca66b2e19c5d6de9e2c2b69d80a896499ee3ad Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Sun, 24 Apr 2016 21:59:12 -0700 Subject: [PATCH 25/31] Changing openstack to openstack-heat in get-kube.sh --- cluster/get-kube.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/get-kube.sh b/cluster/get-kube.sh index 5b7b3c696543e..e8e3cb0593bdb 100755 --- a/cluster/get-kube.sh +++ b/cluster/get-kube.sh @@ -39,7 +39,7 @@ # Rackspace # * export KUBERNETES_PROVIDER=rackspace; wget -q -O - https://get.k8s.io | bash # OpenStack -# * export KUBERNETES_PROVIDER=openstack; wget -q -O - https://get.k8s.io | bash +# * export KUBERNETES_PROVIDER=openstack-heat; wget -q -O - https://get.k8s.io | bash # # Set KUBERNETES_SKIP_DOWNLOAD to non-empty to skip downloading a release. # Set KUBERNETES_SKIP_CONFIRM to skip the installation confirmation prompt. From 5f929d2a4c179a73ae2dc2337f608a379b0f495f Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Wed, 27 Apr 2016 13:06:46 +0200 Subject: [PATCH 26/31] Replace heat client to openstack to avoid warnings --- cluster/openstack-heat/util.sh | 60 +++++++++++++++++----------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cluster/openstack-heat/util.sh b/cluster/openstack-heat/util.sh index 6aa8082b01a4a..80a0e70376ba3 100644 --- a/cluster/openstack-heat/util.sh +++ b/cluster/openstack-heat/util.sh @@ -32,7 +32,7 @@ fi # Verify prereqs on host machine function verify-prereqs() { # Check the OpenStack command-line clients - for client in swift glance nova heat; + for client in swift glance nova openstack; do if which $client >/dev/null 2>&1; then echo "${client} client installed" @@ -61,7 +61,7 @@ function kube-up() { function validate-cluster() { while (( --$STACK_CREATE_TIMEOUT >= 0)) ;do - local status=$(heat stack-show "${STACK_NAME}" | awk '$2=="stack_status" {print $4}') + local status=$(openstack stack show "${STACK_NAME}" | awk '$2=="stack_status" {print $4}') if [[ $status ]]; then echo "Cluster status ${status}" if [ $status = "CREATE_COMPLETE" ]; then @@ -145,7 +145,7 @@ function add-keypair() { # OPENSTACK_IMAGE_NAME function create-glance-image() { if [[ ${CREATE_IMAGE} == "true" ]]; then - local image_status=$(nova image-show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') + local image_status=$(openstack image show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') if [[ ! $image_status ]]; then if [[ "${DOWNLOAD_IMAGE}" == "true" ]]; then @@ -180,12 +180,12 @@ function create-glance-image() { # STACK_NAME function run-heat-script() { - local stack_status=$(heat stack-show ${STACK_NAME}) + local stack_status=$(openstack stack show ${STACK_NAME}) local swift_repo_url="${SWIFT_SERVER_URL}/v1/AUTH_${SWIFT_PROJECT_ID}/kubernetes" if [ $CREATE_IMAGE = true ]; then echo "[INFO] Retrieve new image ID" - IMAGE_ID=$(nova image-show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') + IMAGE_ID=$(openstack image show ${OPENSTACK_IMAGE_NAME} | awk '$2=="id" {print $4}') echo "[INFO] Image Id ${IMAGE_ID}" fi @@ -193,34 +193,34 @@ function run-heat-script() { echo "[INFO] Create stack ${STACK_NAME}" ( cd ${ROOT}/kubernetes-heat - heat --api-timeout 60 stack-create \ - -P external_network=${EXTERNAL_NETWORK} \ - -P ssh_key_name=${KUBERNETES_KEYPAIR_NAME} \ - -P server_image=${IMAGE_ID} \ - -P master_flavor=${MASTER_FLAVOR} \ - -P minion_flavor=${MINION_FLAVOR} \ - -P number_of_minions=${NUMBER_OF_MINIONS} \ - -P max_number_of_minions=${MAX_NUMBER_OF_MINIONS} \ - -P dns_nameserver=${DNS_SERVER} \ - -P kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \ - -P kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \ - -P os_auth_url=${OS_AUTH_URL} \ - -P os_username=${OS_USERNAME} \ - -P os_password=${OS_PASSWORD} \ - -P os_region_name=${OS_REGION_NAME} \ - -P os_tenant_id=${OS_TENANT_ID} \ - -P enable_proxy=${ENABLE_PROXY} \ - -P ftp_proxy="${FTP_PROXY}" \ - -P http_proxy="${HTTP_PROXY}" \ - -P https_proxy="${HTTPS_PROXY}" \ - -P socks_proxy="${SOCKS_PROXY}" \ - -P no_proxy="${NO_PROXY}" \ - --template-file kubecluster.yaml \ + openstack stack create --timeout 60 \ + --parameter external_network=${EXTERNAL_NETWORK} \ + --parameter ssh_key_name=${KUBERNETES_KEYPAIR_NAME} \ + --parameter server_image=${IMAGE_ID} \ + --parameter master_flavor=${MASTER_FLAVOR} \ + --parameter minion_flavor=${MINION_FLAVOR} \ + --parameter number_of_minions=${NUMBER_OF_MINIONS} \ + --parameter max_number_of_minions=${MAX_NUMBER_OF_MINIONS} \ + --parameter dns_nameserver=${DNS_SERVER} \ + --parameter kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \ + --parameter kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \ + --parameter os_auth_url=${OS_AUTH_URL} \ + --parameter os_username=${OS_USERNAME} \ + --parameter os_password=${OS_PASSWORD} \ + --parameter os_region_name=${OS_REGION_NAME} \ + --parameter os_tenant_id=${OS_TENANT_ID} \ + --parameter enable_proxy=${ENABLE_PROXY} \ + --parameter ftp_proxy="${FTP_PROXY}" \ + --parameter http_proxy="${HTTP_PROXY}" \ + --parameter https_proxy="${HTTPS_PROXY}" \ + --parameter socks_proxy="${SOCKS_PROXY}" \ + --parameter no_proxy="${NO_PROXY}" \ + --template kubecluster.yaml \ ${STACK_NAME} ) else echo "[INFO] Stack ${STACK_NAME} already exists" - heat stack-show ${STACK_NAME} + openstack stack show ${STACK_NAME} fi } @@ -250,7 +250,7 @@ function configure-kubectl() { # STACK_NAME function kube-down { source "${ROOT}/openrc-default.sh" - heat stack-delete ${STACK_NAME} + openstack stack delete ${STACK_NAME} } # Perform preparations required to run e2e tests From dd786794e94ece9f683577711509897fff8d0e82 Mon Sep 17 00:00:00 2001 From: Francois Deppierraz Date: Thu, 28 Apr 2016 09:20:13 +0200 Subject: [PATCH 27/31] Expose /etc/pki from host to kube-apiserver and controller-manager CentOS 7 Core nodes running on OpenStack with an SSL-enabled API endpoint results in the following error without this patch: F0425 19:00:58.124520 5 server.go:100] Cloud provider could not be initialized: could not init cloud provider "openstack": Post https://my.openstack.cloud:5000/v2.0/tokens: x509: failed to load system roots and no roots provided The root cause is that the ca-bundle.crt file is actually a symlink which points to a directory which wasn't previously exposed. [root@kubernetesstack-master ~]# ls -l /etc/ssl/certs/ca-bundle.crt lrwxrwxrwx. 1 root root 49 18 nov 11:02 /etc/ssl/certs/ca-bundle.crt -> /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem [root@kubernetesstack-master ~]# --- .../saltbase/salt/kube-apiserver/kube-apiserver.manifest | 8 ++++---- .../kube-controller-manager.manifest | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 5dd4eaf8e22e6..81e31b291e2f2 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -168,8 +168,8 @@ { "name": "etcopenssl", "mountPath": "/etc/openssl", "readOnly": true}, - { "name": "etcpkitls", - "mountPath": "/etc/pki/tls", + { "name": "etcpki", + "mountPath": "/etc/pki", "readOnly": true}, { "name": "srvsshproxy", "mountPath": "{{srv_sshproxy_path}}", @@ -200,9 +200,9 @@ "hostPath": { "path": "/etc/openssl"} }, - { "name": "etcpkitls", + { "name": "etcpki", "hostPath": { - "path": "/etc/pki/tls"} + "path": "/etc/pki"} }, { "name": "srvsshproxy", "hostPath": { diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index e7fc51803bf88..e13048abeaa30 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -121,8 +121,8 @@ { "name": "etcopenssl", "mountPath": "/etc/openssl", "readOnly": true}, - { "name": "etcpkitls", - "mountPath": "/etc/pki/tls", + { "name": "etcpki", + "mountPath": "/etc/pki", "readOnly": true} ] } @@ -150,9 +150,9 @@ "hostPath": { "path": "/etc/openssl"} }, - { "name": "etcpkitls", + { "name": "etcpki", "hostPath": { - "path": "/etc/pki/tls"} + "path": "/etc/pki"} } ] }} From 9004dc383bdd415b77d0ad9bdd9339bae5e1de2b Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Thu, 28 Apr 2016 22:36:00 -0700 Subject: [PATCH 28/31] Adding workaround for hostname swapping issue in OpenStack kube-up provider. On reboot, minions hostname would change, causing issue querying Openstack API and registering with kube-apiserver. This is due to a bug in cloud-init which has been patched but apparently not widely distributed. Should be harmless on unaffected systems. https://bugs.launchpad.net/cloud-init/+bug/1246485 http://blog.oddbit.com/2014/12/10/cloudinit-and-the-case-of-the-changing-hostname/ --- .../fragments/hostname-hack.sh | 23 +++++++++++++++++++ .../fragments/hostname-hack.yaml | 9 ++++++++ .../kubernetes-heat/kubecluster.yaml | 14 +++++++++++ .../kubernetes-heat/kubeminion.yaml | 14 +++++++++++ 4 files changed, 60 insertions(+) create mode 100644 cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.sh create mode 100644 cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.yaml diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.sh b/cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.sh new file mode 100644 index 0000000000000..d29d5c8cd35f6 --- /dev/null +++ b/cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Workaround for this, which has been fixed but not widely distributed: https://bugs.launchpad.net/cloud-init/+bug/1246485 +# See also http://blog.oddbit.com/2014/12/10/cloudinit-and-the-case-of-the-changing-hostname/ +hostname > /etc/hostname diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.yaml new file mode 100644 index 0000000000000..8e3b2ac015893 --- /dev/null +++ b/cluster/openstack-heat/kubernetes-heat/fragments/hostname-hack.yaml @@ -0,0 +1,9 @@ +#cloud-config +merge_how: dict(recurse_array)+list(append) + +write_files: + - path: /etc/cloud/cloud.cfg.d/99_hostname.cfg + owner: "root:root" + permissions: "0644" + content: | + preserve_hostname: true diff --git a/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml b/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml index fdd3eaee77ed7..58204afe3d4a3 100644 --- a/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml @@ -242,6 +242,18 @@ resources: "$SOCKS_PROXY": {get_param: socks_proxy } "$NO_PROXY": {get_param: no_proxy } + hostname_hack: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/hostname-hack.yaml} + + hostname_hack_script: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/hostname-hack.sh} + kube_user: type: OS::Heat::SoftwareConfig properties: @@ -299,6 +311,8 @@ resources: parts: - config: {get_resource: write_heat_params} - config: {get_resource: proxy_config} + - config: {get_resource: hostname_hack} + - config: {get_resource: hostname_hack_script} - config: {get_resource: kube_user} - config: {get_resource: provision_network_master} - config: {get_resource: deploy_kube_auth_files_master} diff --git a/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml b/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml index 0e34e9b51b4f2..a51aab2a22bf3 100644 --- a/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml +++ b/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml @@ -165,6 +165,18 @@ resources: "$SOCKS_PROXY": {get_param: socks_proxy } "$NO_PROXY": {get_param: no_proxy } + hostname_hack: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/hostname-hack.yaml} + + hostname_hack_script: + type: OS::Heat::SoftwareConfig + properties: + group: ungrouped + config: {get_file: fragments/hostname-hack.sh} + kube_user: type: OS::Heat::SoftwareConfig properties: @@ -220,6 +232,8 @@ resources: parts: - config: {get_resource: write_heat_params} - config: {get_resource: proxy_config} + - config: {get_resource: hostname_hack} + - config: {get_resource: hostname_hack_script} - config: {get_resource: kube_user} - config: {get_resource: provision_network_node} - config: {get_resource: deploy_kube_auth_files_node} From 895c543c8c8db6b4877c61124e63c81c9904c555 Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Fri, 29 Apr 2016 08:16:23 +0200 Subject: [PATCH 29/31] Add missing single speech mark after rebase --- cluster/saltbase/salt/kube-apiserver/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/saltbase/salt/kube-apiserver/init.sls b/cluster/saltbase/salt/kube-apiserver/init.sls index 2329437dbd1be..8d3245013dda4 100644 --- a/cluster/saltbase/salt/kube-apiserver/init.sls +++ b/cluster/saltbase/salt/kube-apiserver/init.sls @@ -1,4 +1,4 @@ -{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack] %} +{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack'] %} # TODO: generate and distribute tokens on other cloud providers. /srv/kubernetes/known_tokens.csv: file.managed: From 1769760db0c38f64415d4fe99fc400ba93f39fcc Mon Sep 17 00:00:00 2001 From: Francois Deppierraz Date: Thu, 5 May 2016 23:45:58 +0200 Subject: [PATCH 30/31] Automatically detect swift URL if not specified --- cluster/openstack-heat/config-default.sh | 3 +-- cluster/openstack-heat/util.sh | 13 ++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cluster/openstack-heat/config-default.sh b/cluster/openstack-heat/config-default.sh index 480e6f43059e8..6c4947f86c86d 100644 --- a/cluster/openstack-heat/config-default.sh +++ b/cluster/openstack-heat/config-default.sh @@ -35,8 +35,7 @@ MINION_FLAVOR=${MINION_FLAVOR:-m1.medium} EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-public} -#FIXME This can get set by evaulating the output of `swift auth` and stripping the base URL from OS_STORAGE_URL, but will require a refactor. Till then... -SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-http://192.168.123.100:8080} +SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-} # Flag indicates if new image must be created. If 'false' then image with IMAGE_ID will be used. # If 'true' then new image will be created from file config-image.sh diff --git a/cluster/openstack-heat/util.sh b/cluster/openstack-heat/util.sh index 80a0e70376ba3..2b64a19fef7a2 100644 --- a/cluster/openstack-heat/util.sh +++ b/cluster/openstack-heat/util.sh @@ -91,11 +91,6 @@ function create-stack() { echo "[INFO] Execute commands to create Kubernetes cluster" # It is required for some cloud provider like CityCloud where swift client has different credentials source "${ROOT}/openrc-swift.sh" - if [[ -z ${OS_PROJECT_ID+x} ]]; then - SWIFT_PROJECT_ID="${OS_TENANT_ID}" - else - SWIFT_PROJECT_ID="${OS_PROJECT_ID}" - fi upload-resources source "${ROOT}/openrc-default.sh" @@ -168,7 +163,6 @@ function create-glance-image() { # KUBERNETES_KEYPAIR_NAME # DNS_SERVER # SWIFT_SERVER_URL -# SWIFT_TENANT_ID # OPENSTACK_IMAGE_NAME # EXTERNAL_NETWORK # IMAGE_ID @@ -181,7 +175,12 @@ function create-glance-image() { function run-heat-script() { local stack_status=$(openstack stack show ${STACK_NAME}) - local swift_repo_url="${SWIFT_SERVER_URL}/v1/AUTH_${SWIFT_PROJECT_ID}/kubernetes" + + # Automatically detect swift url if it wasn't specified + if [[ -z $SWIFT_SERVER_URL ]]; then + SWIFT_SERVER_URL=$(openstack catalog show object-store --format value | egrep -o "publicURL: (.+)$" | cut -d" " -f2) + fi + local swift_repo_url="${SWIFT_SERVER_URL}/kubernetes" if [ $CREATE_IMAGE = true ]; then echo "[INFO] Retrieve new image ID" From 892c2abaf525aa070cd9af6b1818a527de5a83cf Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Tue, 10 May 2016 07:25:29 +0200 Subject: [PATCH 31/31] Name adjustment to OpenStack-Heat in cluster/get-kube.sh --- cluster/get-kube.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/get-kube.sh b/cluster/get-kube.sh index e8e3cb0593bdb..6aaab0a468774 100755 --- a/cluster/get-kube.sh +++ b/cluster/get-kube.sh @@ -38,7 +38,7 @@ # * export KUBERNETES_PROVIDER=photon-controller; wget -q -O - https://get.k8s.io | bash # Rackspace # * export KUBERNETES_PROVIDER=rackspace; wget -q -O - https://get.k8s.io | bash -# OpenStack +# OpenStack-Heat # * export KUBERNETES_PROVIDER=openstack-heat; wget -q -O - https://get.k8s.io | bash # # Set KUBERNETES_SKIP_DOWNLOAD to non-empty to skip downloading a release.