Support for ICP 3.1.0 on AWS (#10)

* Initial 3.1 updates

Signed-off-by: Rick Osowski <rosowski@gmail.com>

* Initial 3.1 updates

Signed-off-by: Rick Osowski <osowski@us.ibm.com>

* Updated variables.tf defaults for disk sizes

Signed-off-by: Rick Osowski <osowski@us.ibm.com>

* Cleanup start_install.sh

* Update start_install.sh

* Updated instance types to align with KC

Signed-off-by: Rick Osowski <osowski@us.ibm.com>

.gitignore

@@ -5,8 +5,31 @@
 *.tfstate
 *.tfstate.*
 
+# Crash log files
+crash.log
+
+# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
+# .tfvars files are managed as part of configuration and so should be included in
+# version control.
+#
+# example.tfvars
+terraform.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+awscli/*
+=======
 # .tfvars files
 *.tfvars
 
 # generated key files
-*.pem
+*.pem

efs.tf

@@ -1,4 +1,3 @@
-/*
 resource "aws_efs_file_system" "icp-registry" {
   count = "${var.master["nodes"] > 1 ? 1 : 0 }"
   creation_token = "icp-${random_id.clusterid.hex}-registry"
@@ -30,4 +29,3 @@ resource "aws_efs_mount_target" "icp-audit" {
   subnet_id     = "${element(aws_subnet.icp_private_subnet.*.id, count.index)}"
   security_groups = [ "${aws_security_group.icp-audit-mount.id}"]
 }
-*/

instances.tf

@@ -184,6 +184,13 @@ ${count.index == 0 && var.enable_autoscaling ? "
 - /tmp/icp_scripts/create_client_cert.sh -i ${var.icp_inception_image} -b ${aws_s3_bucket.icp_config_backup.id}"
   :
 "" }
+${var.master["nodes"] > 1 ? "
+mounts:
+  - ['${element(local.efs_registry_mountpoints, count.index)}:/', '/var/lib/registry', 'nfs4', 'nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2', '0', '0']
+  - ['${element(local.efs_audit_mountpoints, count.index)}:/', '/var/lib/icp/audit', 'nfs4', 'nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2', '0', '0']
+"
+:
+"" }
 users:
 - default
 - name: icpdeploy

scripts/bootstrap.sh

@@ -154,7 +154,11 @@ image_load() {
   if [[ ! -z "${image_location}" ]]; then
     if [[ "${image_location:0:2}" == "s3" ]]; then
       echo "Load docker images from ${image_location} ..."
-      ${awscli} s3 cp ${image_location} - | tar zxf - -O | docker load
+      #TODO Is this install directory parameterized?
+      IMAGE_DIR=/opt/ibm/cluster/images
+      mkdir -p ${IMAGE_DIR}
+      ${awscli} s3 cp ${image_location} ${IMAGE_DIR}/ibm-cloud-private-x86_64-3.1.0.tar.gz 
+      tar zxf ${IMAGE_DIR}/ibm-cloud-private-x86_64-3.1.0.tar.gz -O | docker load
     fi
   fi
 

scripts/start_install.sh

@@ -48,6 +48,10 @@ ${awscli} s3 cp s3://${s3_config_bucket}/cfc-certs /opt/ibm/cluster/cfc-certs
 ${awscli} s3 cp s3://${s3_config_bucket}/ssh_key /opt/ibm/cluster/ssh_key
 ${awscli} s3 cp s3://${s3_config_bucket}/icp-terraform-config.yaml /tmp/icp-terraform-config.yaml
 
+# HOTFIX for https://www.ibm.com/support/knowledgecenter/en/SSBS6K_3.1.0/troubleshoot/manifest_tool.html
+echo "Symlinking manifest-tool into /usr/bin"
+sudo ln -s /usr/local/bin/manifest-tool /usr/bin/manifest-tool
+
 # append the image repo
 if [ ! -z "${registry}${registry:+}" ]; then
   echo "image_repo: ${registry}${registry:+/}${org}" >> /tmp/icp-terraform-config.yaml
@@ -143,18 +147,19 @@ for script in ${s3_patch_scripts}; do
   rm -f /opt/ibm/cluster/${script_name}
 done
 
+# REVERTED BACK TO EBS-BASED REGISTRY SUPPORT IN ICP 3.1.0
 # patch the registry to use our S3 bucket
-region=`curl http://169.254.169.254/latest/dynamic/instance-identity/document | grep "region" | awk -F: '{print $2}' | sed -e 's/[ ",]//g'`
-sed -i '/filesystem/{$!{N;s/filesystem:\n\(.*\)rootdirectory.*/s3:\n\1bucket: '${s3_registry_bucket}'\n\1region: '${region}'/}}' /opt/ibm/cluster/cfc-components/registry-conf/registry-config.yaml
-kubectl="docker run --net=host -e KUBECONFIG=/tmp/kubeconfig.yaml -v /opt/ibm/cluster:/installer/cluster -v /tmp:/tmp --entrypoint /usr/local/bin/kubectl ${inception_image}"
-$kubectl config set-cluster local --server=https://localhost:8001 --insecure-skip-tls-verify=true
-$kubectl config set-credentials user --embed-certs=true --client-certificate=/installer/cluster/cfc-certs/kubecfg.crt --client-key=/installer/cluster/cfc-certs/kubecfg.key
-$kubectl config set-context ctx --cluster=local --user=user --namespace=kube-system
-$kubectl config use-context ctx
-$kubectl delete configmap registry-config
-$kubectl create configmap registry-config --from-file=/installer/cluster/cfc-components/registry-conf/registry-config.yaml
-$kubectl delete pods -l app=image-manager
-rm -f /tmp/kubeconfig.yaml
+#region=`curl http://169.254.169.254/latest/dynamic/instance-identity/document | grep "region" | awk -F: '{print $2}' | sed -e 's/[ ",]//g'`
+#sed -i '/filesystem/{$!{N;s/filesystem:\n\(.*\)rootdirectory.*/s3:\n\1bucket: '${s3_registry_bucket}'\n\1region: '${region}'/}}' /opt/ibm/cluster/cfc-components/registry-conf/registry-config.yaml
+#kubectl="docker run --net=host -e KUBECONFIG=/tmp/kubeconfig.yaml -v /opt/ibm/cluster:/installer/cluster -v /tmp:/tmp --entrypoint /usr/local/bin/kubectl ${inception_image}"
+#$kubectl config set-cluster local --server=https://localhost:8001 --insecure-skip-tls-verify=true
+#$kubectl config set-credentials user --embed-certs=true --client-certificate=/installer/cluster/cfc-certs/kubecfg.crt --client-key=/installer/cluster/cfc-certs/kubecfg.key
+#$kubectl config set-context ctx --cluster=local --user=user --namespace=kube-system
+#$kubectl config use-context ctx
+#$kubectl delete configmap registry-config
+#$kubectl create configmap registry-config --from-file=/installer/cluster/cfc-components/registry-conf/registry-config.yaml
+#$kubectl delete pods -l app=image-manager
+#rm -f /tmp/kubeconfig.yaml
 
 # backup the config
 ${awscli} s3 sync /opt/ibm/cluster s3://${s3_config_bucket}

security_group.tf

@@ -256,7 +256,7 @@ resource "aws_security_group_rule" "master-egress" {
   security_group_id = "${aws_security_group.master.id}"
 }
 
-/*
+
 resource "aws_security_group" "icp-registry-mount" {
   count = "${var.master["nodes"] > 1 ? 1 : 0 }"
   name = "icp_efs_registry_sg-${random_id.clusterid.hex}"
@@ -316,4 +316,4 @@ resource "aws_security_group" "icp-audit-mount" {
     map("Name", "icp-audit-mount-sg-${random_id.clusterid.hex}")
   )}"
 }
-*/
+

variables.tf

@@ -75,9 +75,9 @@ variable "master" {
   type = "map"
   default = {
     nodes     = "3"
-    type      = "m4.xlarge"
+    type      = "m4.2xlarge"
     ami       = "" // Leave blank to let terraform search for Ubuntu 16.04 ami. NOT RECOMMENDED FOR PRODUCTION
-    disk      = "100" //GB
+    disk      = "300" //GB
     docker_vol = "100" // GB
     ebs_optimized = true    // not all instance types support EBS optimized
   }
@@ -86,9 +86,9 @@ variable "proxy" {
   type = "map"
   default = {
     nodes     = "3"
-    type      = "m4.large"
+    type      = "m4.xlarge"
     ami       = "" // Leave blank to let terraform search for Ubuntu 16.04 ami. NOT RECOMMENDED FOR PRODUCTION
-    disk      = "100" //GB
+    disk      = "150" //GB
     docker_vol = "100" // GB
     ebs_optimized = true    // not all instance types support EBS optimized
   }
@@ -98,9 +98,9 @@ variable "management" {
   type = "map"
   default = {
     nodes     = "3"
-    type      = "m4.xlarge"
+    type      = "m4.2xlarge"
     ami       = "" // Leave blank to let terraform search for Ubuntu 16.04 ami. NOT RECOMMENDED FOR PRODUCTION
-    disk      = "100" //GB
+    disk      = "300" //GB
     docker_vol = "100" // GB
     ebs_optimized = true    // not all instance types support EBS optimized
   }
@@ -110,9 +110,9 @@ variable "worker" {
   type = "map"
   default = {
     nodes     = "3"
-    type      = "m4.xlarge"
+    type      = "m4.2xlarge"
     ami       = "" // Leave blank to let terraform search for Ubuntu 16.04 ami. NOT RECOMMENDED FOR PRODUCTION
-    disk      = "100" //GB
+    disk      = "150" //GB
     docker_vol = "100" // GB
     ebs_optimized = true    // not all instance types support EBS optimized
   }
@@ -122,9 +122,9 @@ variable "va" {
   type = "map"
   default = {
     nodes     = "3"
-    type      = "m4.xlarge"
+    type      = "m4.2xlarge"
     ami       = "" // Leave blank to let terraform search for Ubuntu 16.04 ami. NOT RECOMMENDED FOR PRODUCTION
-    disk      = "100" //GB
+    disk      = "300" //GB
     docker_vol = "100" // GB
     ebs_optimized = true    // not all instance types support EBS optimized
   }