From 35c9aa666f76d1e440af44bd9eb45a0ff5c84b0f Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Mon, 17 Dec 2018 23:49:34 +0100 Subject: [PATCH 1/8] Use S3 bucket for CI e2e state --- hack/ci-e2e-test.sh | 20 ++++++++++++++------ test/tools/integration/Makefile | 11 +++++++++-- test/tools/integration/provider.tf.disabled | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 test/tools/integration/provider.tf.disabled diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index e4072baf5..05ae18c08 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e +set -euo pipefail function cleanup { set +e @@ -20,18 +20,26 @@ function cleanup { } trap cleanup EXIT -export BUILD_ID="${BUILD_ID}" - # Install dependencies echo "Installing dependencies." apt update && apt install -y jq rsync unzip && -curl --retry 5 -LO https://storage.googleapis.com/kubernetes-release/release/v1.10.0/bin/linux/amd64/kubectl && +curl --retry 5 -LO \ + https://storage.googleapis.com/kubernetes-release/release/v1.12.4/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin # Generate ssh keypair -echo "Generating ssh keypairs." -ssh-keygen -f $HOME/.ssh/id_rsa -P '' +echo "Set permissions for ssh key" +chmod 0700 $HOME/.ssh + +# Initialize terraform +echo "Initalizing terraform" +cd test/tools/integration +make terraform +cp provider.tf{.disabled,} +terraform init --input=false --backend-config=key=$BUILD_ID +terraform import hcloud_ssh_key.machine-controller-e2e machine-controller-e2e +cd - for try in {1..20}; do # Create environment at cloud provider diff --git a/test/tools/integration/Makefile b/test/tools/integration/Makefile index a26e1a698..16d098f6d 100644 --- a/test/tools/integration/Makefile +++ b/test/tools/integration/Makefile @@ -1,3 +1,5 @@ +SHELL := /bin/bash + BUILD_ID ?= $(USER)-local USER ?= prow @@ -10,14 +12,19 @@ else ifeq ($(MAKECMDGOALS),destroy) EXTRA_ARG = -force endif +.PHONY: terraform terraform: @if ! which terraform; then \ - curl https://releases.hashicorp.com/terraform/0.11.7/terraform_0.11.7_linux_amd64.zip > /tmp/terraform.zip && \ + curl https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ + --retry 5 \ + -o /tmp/terraform.zip && \ unzip -n /tmp/terraform.zip terraform; \ fi .terraform: terraform - terraform init >/dev/null 2>&1 + @if ! ls .terraform &>/dev/null; then \ + terraform init &>/dev/null; \ + fi .PHONY: plan apply destroy plan apply destroy: .terraform diff --git a/test/tools/integration/provider.tf.disabled b/test/tools/integration/provider.tf.disabled new file mode 100644 index 000000000..aab2a5e0b --- /dev/null +++ b/test/tools/integration/provider.tf.disabled @@ -0,0 +1,14 @@ +terraform { + backend "s3" { + bucket = "terraform" + endpoint = "http://minio.minio:9000" + access_key = "PMIC1HMXNB2R67RNPIX8" + secret_key = "NemiWx+uY79rcJ0hXrktzHk1dm9c0k85WepbuSlK" + region = "myregion" + skip_region_validation = "true" + skip_metadata_api_check = "true" + skip_requesting_account_id = "true" + skip_credentials_validation = "true" + force_path_style = "true" + } +} From 2156b2bbd89e653bfc47f78feb9887efdf81146c Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 16:03:02 +0100 Subject: [PATCH 2/8] Use terraform output for ip address --- test/tools/integration/cleanup_machines.sh | 2 +- test/tools/integration/output.tf | 3 +++ test/tools/integration/provision_master.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 test/tools/integration/output.tf diff --git a/test/tools/integration/cleanup_machines.sh b/test/tools/integration/cleanup_machines.sh index aaf98e05b..ca6fcac0e 100755 --- a/test/tools/integration/cleanup_machines.sh +++ b/test/tools/integration/cleanup_machines.sh @@ -5,7 +5,7 @@ set -x cd $(dirname $0) -export ADDR=$(cat terraform.tfstate |jq -r '.modules[0].resources["hcloud_server.machine-controller-test"].primary.attributes.ipv4_address') +export ADDR=$(terraform output -json|jq '.ip.value' -r) ssh_exec() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$ADDR $@; } diff --git a/test/tools/integration/output.tf b/test/tools/integration/output.tf new file mode 100644 index 000000000..acaf1c4df --- /dev/null +++ b/test/tools/integration/output.tf @@ -0,0 +1,3 @@ +output "ip" { + value = "${hcloud_server.machine-controller-test.ipv4_address}" +} diff --git a/test/tools/integration/provision_master.sh b/test/tools/integration/provision_master.sh index 62f7c4d29..b14692971 100755 --- a/test/tools/integration/provision_master.sh +++ b/test/tools/integration/provision_master.sh @@ -5,7 +5,7 @@ set -x cd $(dirname $0) -export ADDR=$(cat terraform.tfstate |jq -r '.modules[0].resources["hcloud_server.machine-controller-test"].primary.attributes.ipv4_address') +export ADDR=$(terraform output -json|jq '.ip.value' -r) ssh_exec() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$ADDR $@; } From 9bd8cda6be51fc7bd8a308924db1ae7f9cfb3a93 Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 16:50:47 +0100 Subject: [PATCH 3/8] Move terraform into PATH --- test/tools/integration/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/tools/integration/Makefile b/test/tools/integration/Makefile index 16d098f6d..8271cf506 100644 --- a/test/tools/integration/Makefile +++ b/test/tools/integration/Makefile @@ -18,7 +18,8 @@ terraform: curl https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ --retry 5 \ -o /tmp/terraform.zip && \ - unzip -n /tmp/terraform.zip terraform; \ + unzip -n /tmp/terraform.zip terraform && \ + mv terraform /usr/local/bin \ fi .terraform: terraform From 818366449e3ef7658b78a72ccb252bbcb17f5692 Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 16:59:35 +0100 Subject: [PATCH 4/8] Fix bash loop --- test/tools/integration/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/integration/Makefile b/test/tools/integration/Makefile index 8271cf506..c3dde5d8d 100644 --- a/test/tools/integration/Makefile +++ b/test/tools/integration/Makefile @@ -19,7 +19,7 @@ terraform: --retry 5 \ -o /tmp/terraform.zip && \ unzip -n /tmp/terraform.zip terraform && \ - mv terraform /usr/local/bin \ + mv terraform /usr/local/bin; \ fi .terraform: terraform From 8b0dc91d41a26015526004eae5e0fb225cc770ff Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 17:08:20 +0100 Subject: [PATCH 5/8] Update bucket name --- test/tools/integration/provider.tf.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/integration/provider.tf.disabled b/test/tools/integration/provider.tf.disabled index aab2a5e0b..2176115d2 100644 --- a/test/tools/integration/provider.tf.disabled +++ b/test/tools/integration/provider.tf.disabled @@ -1,6 +1,6 @@ terraform { backend "s3" { - bucket = "terraform" + bucket = "terraform-machine-controller" endpoint = "http://minio.minio:9000" access_key = "PMIC1HMXNB2R67RNPIX8" secret_key = "NemiWx+uY79rcJ0hXrktzHk1dm9c0k85WepbuSlK" From de29ab10b6d4dece24df1d72d812b839a1a4e89e Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 17:21:12 +0100 Subject: [PATCH 6/8] Hopefully fix ssh key --- hack/ci-e2e-test.sh | 3 ++- test/tools/integration/variables.tf | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index 05ae18c08..b0dccee34 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -38,7 +38,8 @@ cd test/tools/integration make terraform cp provider.tf{.disabled,} terraform init --input=false --backend-config=key=$BUILD_ID -terraform import hcloud_ssh_key.machine-controller-e2e machine-controller-e2e +export TF_VAR_hcloud_sshkey_content="$(cat ~/.ssh/id_rsa.pub)" +terraform import hcloud_ssh_key.default machine-controller-e2e cd - for try in {1..20}; do diff --git a/test/tools/integration/variables.tf b/test/tools/integration/variables.tf index 2fcdb9b78..be1e69b2d 100644 --- a/test/tools/integration/variables.tf +++ b/test/tools/integration/variables.tf @@ -1,4 +1,6 @@ -variable "hcloud_token" {} +variable "hcloud_token" { + default = "machine-controller-e2e" +} variable "hcloud_sshkey_content" {} variable "hcloud_sshkey_name" {} variable "hcloud_test_server_name" {} From 5e6ee47a3f11dc6efbd32035806d19b3df1f2737 Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 17:31:35 +0100 Subject: [PATCH 7/8] Fix ssh key import --- hack/ci-e2e-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index b0dccee34..12b08c99c 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -38,8 +38,9 @@ cd test/tools/integration make terraform cp provider.tf{.disabled,} terraform init --input=false --backend-config=key=$BUILD_ID +export TF_VAR_hcloud_token="${HZ_E2E_TOKEN}" export TF_VAR_hcloud_sshkey_content="$(cat ~/.ssh/id_rsa.pub)" -terraform import hcloud_ssh_key.default machine-controller-e2e +terraform import hcloud_ssh_key.default 264677 cd - for try in {1..20}; do From 1e1a4b22feffb4282f5df7a72bd2c9da41b5d05a Mon Sep 17 00:00:00 2001 From: Alvaro Aleman Date: Tue, 18 Dec 2018 18:20:31 +0100 Subject: [PATCH 8/8] Hopefully final fix for s3-based state --- hack/ci-e2e-test.sh | 15 +++++++++++---- test/tools/integration/variables.tf | 8 +++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index 12b08c99c..8b989c983 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -9,10 +9,13 @@ function cleanup { echo "Cleaning up machines." ./test/tools/integration/cleanup_machines.sh + cd test/tools/integration for try in {1..20}; do # Clean up master echo "Cleaning up controller, attempt ${try}" - make -C test/tools/integration destroy + # Clean up only the server, we want to keep the key as only one key may exist + # for a given fingerprint + terraform destroy -target=hcloud_server.machine-controller-test -force if [[ $? == 0 ]]; then break; fi echo "Sleeping for $try seconds" sleep ${try}s @@ -40,18 +43,22 @@ cp provider.tf{.disabled,} terraform init --input=false --backend-config=key=$BUILD_ID export TF_VAR_hcloud_token="${HZ_E2E_TOKEN}" export TF_VAR_hcloud_sshkey_content="$(cat ~/.ssh/id_rsa.pub)" -terraform import hcloud_ssh_key.default 264677 -cd - +export TF_VAR_hcloud_test_server_name="machine-controller-test-${BUILD_ID}" for try in {1..20}; do + set +e # Create environment at cloud provider echo "Creating environment at cloud provider." - make -C test/tools/integration apply + terraform import hcloud_ssh_key.default 265119 + terraform apply -auto-approve if [[ $? == 0 ]]; then break; fi echo "Sleeping for $try seconds" sleep ${try}s done +set -e +cd - + # Build binaries echo "Building machine-controller and webhook" make machine-controller webhook diff --git a/test/tools/integration/variables.tf b/test/tools/integration/variables.tf index be1e69b2d..111646de9 100644 --- a/test/tools/integration/variables.tf +++ b/test/tools/integration/variables.tf @@ -1,6 +1,8 @@ -variable "hcloud_token" { +variable "hcloud_token" {} +variable "hcloud_sshkey_content" {} + +variable "hcloud_sshkey_name" { default = "machine-controller-e2e" } -variable "hcloud_sshkey_content" {} -variable "hcloud_sshkey_name" {} + variable "hcloud_test_server_name" {}