Fix clusterctl workflow (openshift#184)

* Fixed rendering of userData

additionally added BatchMode to ssh, because I discovered a password
prompt during bringing up a centos machine. This must just fail.

* Fixed inserting of cloud.conf in generate-yaml.sh

otherwise, cloud.conf does not end up on the nodes, but are referenced
in config files and thus, services do not come up.

* Fixed userdata scripts and machines.yaml

Included default keyName to machines

Fixed docker systemd.service to handle DOCKER_OPTS

Included public-ipv4 as controlPlaneEndpoint to make bootstrap possible
by just editing machines.yaml

* Fixed documentation to bring up a cluster

Added additional requirement to manually upload an ssh key

* Included ssh-key and controlPlaneEndpoint to CentOS

Even though, centos does not work in my tests so far.

* Update README.md

Co-Authored-By: chrigl <christoph.glaubitz@innovo-cloud.de>

README.md

@@ -37,7 +37,7 @@ Participation in the Kubernetes community is governed by the [Kubernetes Code of
 
    ```bash
    cd examples/openstack
-   ./generate-yaml.sh --provider-os [os name]
+   ./generate-yaml.sh --provider-os [os name] [options]
    cd ../..
    ```
    [os name] is the operating system of your provider environment. 
@@ -48,12 +48,12 @@ Participation in the Kubernetes community is governed by the [Kubernetes Code of
    #### Interactively submit provider information
    By default, the generater script will give you a series of command line prompts, asking the following information about your cloud provider:
 
-   - user-name
-   - domain-name
-   - project-id
-   - region-name
-   - auth-url
-   - password
+   - `user-name`
+   - `domain-name`
+   - `project-id`
+   - `region-name`
+   - `auth-url`
+   - `password`
 
    #### Use clouds.yaml to submit provider information
    If you want to generate scripts without being prompted interactively, you can pass generate-yaml a clouds.yaml file. After downloading your clouds.yaml from your provider, make sure that it has the information listed above filled out. It is very likely that it will at lest be missing the password field. Also, note that domain-name is the same as project-name. You may reference the following sample clouds.yaml to see what yours should look like.
@@ -86,9 +86,18 @@ Participation in the Kubernetes community is governed by the [Kubernetes Code of
    ./generate-yaml.sh --provider-os [os name] --clouds clouds.yaml
    ```
 
-   You will need to make changes to the generated files to create a working cluster.
+   You **will need** to make changes to the generated files to create a working cluster.
    You can find some guidance on what needs to be edited, and how to create some of the
-   required OpenStack resources in the [Configuration documentation](docs/config.md)
+   required OpenStack resources in the [Configuration documentation](docs/config.md).
+
+   #### Special notes on ssh keys and fetching `admin.conf`
+
+   When running `generate-yaml.sh` the first time, a new ssh keypair is generated and stored as `$HOME/.ssh/openstack_tmp` and `$HOME/.ssh/openstack_tmp.pub`. In order to allow `clusterctl` to fetch Kubernetes' `admin.conf` from the master node, you **must** manually create the key pair in OpenStack. By default the generated `machine.yaml` uses `cluster-api-provider-openstack` to be the `keyName`. However, you are free to change that.
+
+   e.g.
+   ```
+   openstack keypair create --public-key ~/.ssh/openstack_tmp.pub cluster-api-provider-openstack
+   ```
 
 2. Create a cluster:
 

cmd/clusterctl/examples/openstack/centos/machines.yaml.template

@@ -13,6 +13,7 @@ items:
         flavor: m1.medium
         image: CentOS-7-x86_64-GenericCloud
         sshUserName: centos
+        keyName: cluster-api-provider-openstack
         availabilityZone: nova
         networks:
         - uuid: ab14ce0d-5e1f-4e32-bf65-00416e3cc19c
@@ -39,6 +40,7 @@ items:
         flavor: m1.medium
         image: CentOS-7-x86_64-GenericCloud
         sshUserName: centos
+        keyName: cluster-api-provider-openstack
         availabilityZone: nova
         networks:
         - uuid: ab14ce0d-5e1f-4e32-bf65-00416e3cc19c

cmd/clusterctl/examples/openstack/centos/master-user-data.sh

@@ -13,6 +13,37 @@ CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }}
 POD_CIDR={{ .PodCIDR }}
 SERVICE_CIDR={{ .ServiceCIDR }}
 ARCH=amd64
+
+# Getting master ip from the metadata of the node. By default we try the public-ipv4
+# If we don't get any, we fall back to local-ipv4 and in the worst case to localhost
+MASTER=""
+for i in $(seq 60); do
+    echo "trying to get public-ipv4 $i / 60"
+    MASTER=$(curl --fail -s http://169.254.169.254/2009-04-04/meta-data/public-ipv4)
+    if [[ $? == 0 ]] && [[ -n "$MASTER" ]]; then
+        break
+    fi
+    sleep 1
+done
+
+if [[ -z "$MASTER" ]]; then
+    echo "falling back to local-ipv4"
+    for i in $(seq 60); do
+        echo "trying to get local-ipv4 $i / 60"
+        MASTER=$(curl --fail -s http://169.254.169.254/2009-04-04/meta-data/local-ipv4)
+        if [[ $? == 0 ]] && [[ -n "$MASTER" ]]; then
+            break
+        fi
+        sleep 1
+    done
+fi
+
+if [[ -z "$MASTER" ]]; then
+    echo "falling back to localhost"
+    MASTER="localhost"
+fi
+MASTER="${MASTER}:443"
+
 cat <<EOF > /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
@@ -77,7 +108,7 @@ apiServerExtraVolumes:
 - name: cloud
   hostPath: "/etc/kubernetes/cloud.conf"
   mountPath: "/etc/kubernetes/cloud.conf"
-controlPlaneEndpoint: ""
+controlPlaneEndpoint: ${MASTER}
 controllerManagerExtraArgs:
   cluster-cidr: ${POD_CIDR}
   service-cluster-ip-range: ${SERVICE_CIDR}

cmd/clusterctl/examples/openstack/generate-yaml.sh

@@ -215,16 +215,24 @@ if [[ "$OS" =~ "Linux" ]]; then
   MACHINE_CONTROLLER_SSH_USER=$(echo -n $MACHINE_CONTROLLER_SSH_PLAIN|base64 -w0)
   MACHINE_CONTROLLER_SSH_PUBLIC=$(cat "$MACHINE_CONTROLLER_SSH_HOME$MACHINE_CONTROLLER_SSH_PUBLIC_FILE"|base64 -w0)
   MACHINE_CONTROLLER_SSH_PRIVATE=$(cat "$MACHINE_CONTROLLER_SSH_HOME$MACHINE_CONTROLLER_SSH_PRIVATE_FILE"|base64 -w0)
-  MASTER_USER_DATA=$(echo "$MASTER_USER_DATA_PLAIN"|base64 -w0)
-  WORKER_USER_DATA=$(echo "$WORKER_USER_DATA_PLAIN"|base64 -w0)
+  MASTER_USER_DATA=$(echo "$MASTER_USER_DATA_PLAIN" \
+    | sed -e "s/\$OPENSTACK_CLOUD_PROVIDER_CONF/$OPENSTACK_CLOUD_PROVIDER_CONF/" \
+    | base64 -w0)
+  WORKER_USER_DATA=$(echo "$WORKER_USER_DATA_PLAIN" \
+    | sed -e "s/\$OPENSTACK_CLOUD_PROVIDER_CONF/$OPENSTACK_CLOUD_PROVIDER_CONF/" \
+    | base64 -w0)
 elif [[ "$OS" =~ "Darwin" ]]; then
   OPENSTACK_CLOUD_CONFIG=$(echo "$OPENSTACK_CLOUD_CONFIG_PLAIN"|base64)
   OPENSTACK_CLOUD_PROVIDER_CONF=$(echo "$OPENSTACK_CLOUD_PROVIDER_CONF_PLAIN"|base64)
   MACHINE_CONTROLLER_SSH_USER=$(printf $MACHINE_CONTROLLER_SSH_PLAIN|base64)
   MACHINE_CONTROLLER_SSH_PUBLIC=$(cat "$MACHINE_CONTROLLER_SSH_HOME$MACHINE_CONTROLLER_SSH_PUBLIC_FILE"|base64)
   MACHINE_CONTROLLER_SSH_PRIVATE=$(cat "$MACHINE_CONTROLLER_SSH_HOME$MACHINE_CONTROLLER_SSH_PRIVATE_FILE"|base64)
-  MASTER_USER_DATA=$(echo "$MASTER_USER_DATA_PLAIN"|base64)
-  WORKER_USER_DATA=$(echo "$WORKER_USER_DATA_PLAIN"|base64)
+  MASTER_USER_DATA=$(echo "$MASTER_USER_DATA_PLAIN" \
+    | sed -e "s/\$OPENSTACK_CLOUD_PROVIDER_CONF/$OPENSTACK_CLOUD_PROVIDER_CONF/" \
+    | base64)
+  WORKER_USER_DATA=$(echo "$WORKER_USER_DATA_PLAIN" \
+    | sed -e "s/\$OPENSTACK_CLOUD_PROVIDER_CONF/$OPENSTACK_CLOUD_PROVIDER_CONF/" \
+    | base64)
 else
   echo "Unrecognized OS : $OS"
   exit 1
@@ -267,7 +275,6 @@ cat "$PROVIDERCOMPONENT_TEMPLATE_FILE" \
   | sed -e "s/\$MACHINE_CONTROLLER_SSH_USER/$MACHINE_CONTROLLER_SSH_USER/" \
   | sed -e "s/\$MACHINE_CONTROLLER_SSH_PUBLIC/$MACHINE_CONTROLLER_SSH_PUBLIC/" \
   | sed -e "s/\$MACHINE_CONTROLLER_SSH_PRIVATE/$MACHINE_CONTROLLER_SSH_PRIVATE/" \
-  | sed -e "s/\$OPENSTACK_CLOUD_PROVIDER_CONF/$OPENSTACK_CLOUD_PROVIDER_CONF/" \
   | sed -e "s/\$MASTER_USER_DATA/$MASTER_USER_DATA/" \
   | sed -e "s/\$WORKER_USER_DATA/$WORKER_USER_DATA/" \
   >> "$PROVIDERCOMPONENT_GENERATED_FILE"

cmd/clusterctl/examples/openstack/ubuntu/machines.yaml.template

@@ -13,6 +13,7 @@ items:
         flavor: m1.medium
         image: Ubuntu-Server-16.04-x64
         sshUserName: ubuntu
+        keyName: cluster-api-provider-openstack
         availabilityZone: nova
         networks:
         - uuid: ab14ce0d-5e1f-4e32-bf65-00416e3cc19c
@@ -39,6 +40,7 @@ items:
         flavor: m1.medium
         image: Ubuntu-Server-16.04-x64
         sshUserName: ubuntu
+        keyName: cluster-api-provider-openstack
         availabilityZone: nova
         networks:
         - uuid: ab14ce0d-5e1f-4e32-bf65-00416e3cc19c

cmd/clusterctl/examples/openstack/ubuntu/master-user-data.sh

@@ -20,13 +20,53 @@ apt-get update -y
 apt-get install -y \
     prips
 
+# Getting master ip from the metadata of the node. By default we try the public-ipv4
+# If we don't get any, we fall back to local-ipv4 and in the worst case to localhost
+MASTER=""
+for i in $(seq 60); do
+    echo "trying to get public-ipv4 $i / 60"
+    MASTER=$(curl --fail -s http://169.254.169.254/2009-04-04/meta-data/public-ipv4)
+    if [[ $? == 0 ]] && [[ -n "$MASTER" ]]; then
+        break
+    fi
+    sleep 1
+done
+
+if [[ -z "$MASTER" ]]; then
+    echo "falling back to local-ipv4"
+    for i in $(seq 60); do
+        echo "trying to get local-ipv4 $i / 60"
+        MASTER=$(curl --fail -s http://169.254.169.254/2009-04-04/meta-data/local-ipv4)
+        if [[ $? == 0 ]] && [[ -n "$MASTER" ]]; then
+            break
+        fi
+        sleep 1
+    done
+fi
+
+if [[ -z "$MASTER" ]]; then
+    echo "falling back to localhost"
+    MASTER="localhost"
+fi
+MASTER="${MASTER}:443"
+
 function install_configure_docker () {
     # prevent docker from auto-starting
     echo "exit 101" > /usr/sbin/policy-rc.d
     chmod +x /usr/sbin/policy-rc.d
     trap "rm /usr/sbin/policy-rc.d" RETURN
     apt-get install -y docker.io
     echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
+
+    # Reset iptables config
+    mkdir -p /etc/systemd/system/docker.service.d
+    cat > /etc/systemd/system/docker.service.d/10-iptables.conf <<EOF
+[Service]
+EnvironmentFile=/etc/default/docker
+ExecStart=
+ExecStart=/usr/bin/dockerd -H fd:// \$DOCKER_OPTS
+EOF
+
     systemctl daemon-reload
     systemctl enable docker
     systemctl start docker
@@ -66,6 +106,8 @@ echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf
 
 systemctl daemon-reload
 systemctl restart kubelet.service
+systemctl disable ufw
+systemctl mask ufw
 
 # Set up kubeadm config file to pass parameters to kubeadm init.
 # We're using 443 until this bug is fixed
@@ -88,7 +130,7 @@ kubernetesVersion: v${CONTROL_PLANE_VERSION}
 networking:
   serviceSubnet: ${SERVICE_CIDR}
 clusterName: kubernetes
-controlPlaneEndpoint: ""
+controlPlaneEndpoint: ${MASTER}
 apiServerExtraArgs:
   cloud-provider: "openstack"
   cloud-config: "/etc/kubernetes/cloud.conf"

cmd/clusterctl/examples/openstack/ubuntu/worker-user-data.sh

@@ -28,6 +28,16 @@ function install_configure_docker () {
     trap "rm /usr/sbin/policy-rc.d" RETURN
     apt-get install -y docker.io
     echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
+
+    # Reset iptables config
+    mkdir -p /etc/systemd/system/docker.service.d
+    cat > /etc/systemd/system/docker.service.d/10-iptables.conf <<EOF
+[Service]
+EnvironmentFile=/etc/default/docker
+ExecStart=
+ExecStart=/usr/bin/dockerd -H fd:// \$DOCKER_OPTS
+EOF
+
     systemctl daemon-reload
     systemctl enable docker
     systemctl start docker
@@ -89,17 +99,8 @@ Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-doma
 EOF
 systemctl daemon-reload
 systemctl restart kubelet.service
-
-# Reset iptables config  
-cat > /etc/systemd/system/docker.service.d/10-iptables.conf <<EOF
-[Service]
-EnvironmentFile=/etc/default/docker
-ExecStart=
-ExecStart=/usr/bin/dockerd -H fd:// $DOCKER_OPTS
-EOF
-
-systemctl daemon-reload
 systemctl disable ufw 
+systemctl mask ufw
 
 kubeadm join --ignore-preflight-errors=all --config /etc/kubernetes/kubeadm_config.yaml
 for tries in $(seq 1 60); do

pkg/cloud/openstack/deployer.go

@@ -76,6 +76,7 @@ func (d *DeploymentClient) GetKubeConfig(cluster *clusterv1.Cluster, master *clu
 		"ssh", "-i", homeDir+"/.ssh/openstack_tmp",
 		"-o", "StrictHostKeyChecking no",
 		"-o", "UserKnownHostsFile /dev/null",
+		"-o", "BatchMode=yes",
 		fmt.Sprintf("%s@%s", machineSpec.SshUserName, ip),
 		"echo STARTFILE; sudo cat /etc/kubernetes/admin.conf"))
 	parts := strings.Split(result, "STARTFILE")

pkg/cloud/openstack/machine/actuator.go

@@ -128,7 +128,7 @@ func (oc *OpenstackClient) Create(ctx context.Context, cluster *clusterv1.Cluste
 	}
 
 	var userDataRendered string
-	if len(userData) == 0 {
+	if len(userData) > 0 {
 		if util.IsMaster(machine) {
 			userDataRendered, err = masterStartupScript(cluster, machine, string(userData))
 			if err != nil {
@@ -318,6 +318,7 @@ func (oc *OpenstackClient) GetKubeConfig(cluster *clusterv1.Cluster, master *clu
 		"ssh", "-i", SshPrivateKeyPath,
 		"-o", "StrictHostKeyChecking no",
 		"-o", "UserKnownHostsFile /dev/null",
+		"-o", "BatchMode=yes",
 		fmt.Sprintf("%s@%s", machineSpec.SshUserName, ip),
 		"echo STARTFILE; sudo cat /etc/kubernetes/admin.conf"))
 	parts := strings.Split(result, "STARTFILE")