From f249adb2e61145230fdff84a28db57b5d8396fc2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 8 Jun 2023 18:44:54 +0200 Subject: [PATCH 01/21] Revert "Validate JSON output file before unmarshall (#1298)" This reverts commit 5ad8bd8aeab8b7fa433c213d9a9c07de6baf0a5d. --- .../system/servicedeployer/terraform.go | 6 ----- .../system/servicedeployer/terraform_test.go | 26 +------------------ 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 67b79e6fca..6a191d244e 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -64,12 +64,6 @@ func addTerraformOutputs(outCtxt ServiceContext) error { // Unmarshall the data into `terraformOutputs` logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta - - if !json.Valid(content) { - logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") - return nil - } - if err = json.Unmarshal(content, &terraformOutputs); err != nil { return fmt.Errorf("error during json Unmarshal %w", err) } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_test.go b/internal/testrunner/runners/system/servicedeployer/terraform_test.go index ce19808beb..30d822f455 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_test.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestAddTerraformOutputs(t *testing.T) { @@ -21,28 +20,6 @@ func TestAddTerraformOutputs(t *testing.T) { content []byte expectedProps map[string]interface{} }{ - { - testName: "invalid_json_output", - runId: "987987", - ctxt: ServiceContext{ - Test: struct{ RunID string }{"987987"}, - }, - content: []byte( - ``, - ), - expectedProps: map[string]interface{}{}, - }, - { - testName: "empty_json_output", - runId: "v", - ctxt: ServiceContext{ - Test: struct{ RunID string }{"9887"}, - }, - content: []byte( - `{}`, - ), - expectedProps: map[string]interface{}{}, - }, { testName: "single_value_output", runId: "99999", @@ -144,8 +121,7 @@ func TestAddTerraformOutputs(t *testing.T) { } // Test that the terraform output values are generated correctly - err := addTerraformOutputs(tc.ctxt) - require.NoError(t, err) + addTerraformOutputs(tc.ctxt) assert.Equal(t, tc.expectedProps, tc.ctxt.CustomProperties) }) } From 154ace20bc0e7cdc42eb12515935028ec6d30783 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 8 Jun 2023 18:45:46 +0200 Subject: [PATCH 02/21] Check terraform output --- .../system/servicedeployer/_static/terraform_deployer_run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index b50cd53196..968d78c8e6 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -22,7 +22,9 @@ terraform init terraform plan terraform apply -auto-approve && touch /tmp/tf-applied +set +e terraform output -json > /output/tfOutputValues.json +set -e echo "Terraform definitions applied." From ed8f113306096f0ab113aed68f2232443ad97dd9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 8 Jun 2023 19:14:29 +0200 Subject: [PATCH 03/21] Show contents --- .../system/servicedeployer/_static/terraform_deployer_run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index 968d78c8e6..4a551577ce 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -24,6 +24,10 @@ terraform apply -auto-approve && touch /tmp/tf-applied set +e terraform output -json > /output/tfOutputValues.json + +echo "Output Values content" +cat /output/tfOutputValues +echo "-----" set -e echo "Terraform definitions applied." From 198b5c3ed85f3360331b1448152486847cf1389f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 8 Jun 2023 21:49:08 +0200 Subject: [PATCH 04/21] Add extension --- .../system/servicedeployer/_static/terraform_deployer_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index 4a551577ce..afd0b531b5 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -26,7 +26,7 @@ set +e terraform output -json > /output/tfOutputValues.json echo "Output Values content" -cat /output/tfOutputValues +cat /output/tfOutputValues.json echo "-----" set -e From 6015c9f152753d94231e2f6907dc1f4b59136ea3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 09:39:06 +0200 Subject: [PATCH 05/21] Make not fail to see logs --- .../testrunner/runners/system/servicedeployer/terraform.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 6a191d244e..9102f508d7 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -65,7 +65,7 @@ func addTerraformOutputs(outCtxt ServiceContext) error { logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta if err = json.Unmarshal(content, &terraformOutputs); err != nil { - return fmt.Errorf("error during json Unmarshal %w", err) + return fmt.Errorf("error during JSON Unmarshal: %w", err) } if len(terraformOutputs) == 0 { @@ -160,6 +160,10 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic err = addTerraformOutputs(outCtxt) if err != nil { + if strings.HasPrefix(err.Error(), "error during JSON Unmarshal") { + logger.Errorf(err) + return &service, nil + } return nil, fmt.Errorf("could not handle terraform output %w", err) } service.ctxt = outCtxt From 1ddf8a8f643ff78e695c1af35cdb7926888df842 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 09:47:16 +0200 Subject: [PATCH 06/21] Fix for log message --- internal/testrunner/runners/system/servicedeployer/terraform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 9102f508d7..15f704a409 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -161,7 +161,7 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic err = addTerraformOutputs(outCtxt) if err != nil { if strings.HasPrefix(err.Error(), "error during JSON Unmarshal") { - logger.Errorf(err) + logger.Errorf(err.Error()) return &service, nil } return nil, fmt.Errorf("could not handle terraform output %w", err) From 664f5c69217ba229ccd0b4f48a8d649eb4fe8070 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 12:40:33 +0200 Subject: [PATCH 07/21] Use type bind --- .../system/servicedeployer/_static/terraform_deployer.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml index 794446d1ba..5007652a61 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml @@ -12,5 +12,9 @@ services: - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown} - TF_VAR_REPO=${REPO:-unknown} volumes: - - ${TF_DIR}:/stage - - ${TF_OUTPUT_DIR}:/output + - type: bind + source: ${TF_DIR} + target: /stage + - type: bind + source: ${TF_OUTPUT_DIR} + target: /output From cfe4c0a697171666d87ec4bb93150308f18f27c5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 12:44:53 +0200 Subject: [PATCH 08/21] Show outputs from the host --- .../system/servicedeployer/terraform.go | 6 +++++ .../system/servicedeployer/terraform_test.go | 26 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 15f704a409..3d681fec48 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -64,6 +64,12 @@ func addTerraformOutputs(outCtxt ServiceContext) error { // Unmarshall the data into `terraformOutputs` logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta + + if !json.Valid(content) { + logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") + return fmt.Errorf("error during JSON Unmarshal: %w", err) + } + if err = json.Unmarshal(content, &terraformOutputs); err != nil { return fmt.Errorf("error during JSON Unmarshal: %w", err) } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_test.go b/internal/testrunner/runners/system/servicedeployer/terraform_test.go index 30d822f455..ce19808beb 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_test.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAddTerraformOutputs(t *testing.T) { @@ -20,6 +21,28 @@ func TestAddTerraformOutputs(t *testing.T) { content []byte expectedProps map[string]interface{} }{ + { + testName: "invalid_json_output", + runId: "987987", + ctxt: ServiceContext{ + Test: struct{ RunID string }{"987987"}, + }, + content: []byte( + ``, + ), + expectedProps: map[string]interface{}{}, + }, + { + testName: "empty_json_output", + runId: "v", + ctxt: ServiceContext{ + Test: struct{ RunID string }{"9887"}, + }, + content: []byte( + `{}`, + ), + expectedProps: map[string]interface{}{}, + }, { testName: "single_value_output", runId: "99999", @@ -121,7 +144,8 @@ func TestAddTerraformOutputs(t *testing.T) { } // Test that the terraform output values are generated correctly - addTerraformOutputs(tc.ctxt) + err := addTerraformOutputs(tc.ctxt) + require.NoError(t, err) assert.Equal(t, tc.expectedProps, tc.ctxt.CustomProperties) }) } From 2bea51513b11ddce16ffff7018a31631eba4421b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 12:47:12 +0200 Subject: [PATCH 09/21] Add log message --- internal/testrunner/runners/system/servicedeployer/terraform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 3d681fec48..fc12b5d528 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -64,7 +64,7 @@ func addTerraformOutputs(outCtxt ServiceContext) error { // Unmarshall the data into `terraformOutputs` logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta - + logger.Debugf("**********JSON CONTENT****************\n%s***********************", string(content)) if !json.Valid(content) { logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") return fmt.Errorf("error during JSON Unmarshal: %w", err) From 7deba01f3355f7887bf71e9ca9bef2e61bd9e75b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 13:00:18 +0200 Subject: [PATCH 10/21] skip test case for debug --- .../system/servicedeployer/terraform_test.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_test.go b/internal/testrunner/runners/system/servicedeployer/terraform_test.go index ce19808beb..54c280f98b 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_test.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_test.go @@ -21,17 +21,17 @@ func TestAddTerraformOutputs(t *testing.T) { content []byte expectedProps map[string]interface{} }{ - { - testName: "invalid_json_output", - runId: "987987", - ctxt: ServiceContext{ - Test: struct{ RunID string }{"987987"}, - }, - content: []byte( - ``, - ), - expectedProps: map[string]interface{}{}, - }, + // { + // testName: "invalid_json_output", + // runId: "987987", + // ctxt: ServiceContext{ + // Test: struct{ RunID string }{"987987"}, + // }, + // content: []byte( + // ``, + // ), + // expectedProps: map[string]interface{}{}, + // }, { testName: "empty_json_output", runId: "v", From e5a5f4e6979d9a59f7ac60d3d44fa7787cb6100d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 13:03:16 +0200 Subject: [PATCH 11/21] Check storage driver --- .buildkite/scripts/integration_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index fb1637c117..7e562e1da5 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -111,6 +111,7 @@ echo "--- Run integration test ${TARGET}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then make install + docker info | grep "Storage Driver" # allow to fail this command, to be able to upload safe logs set +e make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} From 2d050ae237e6cf2d8081fe78980b1e749a79d320 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 13:36:37 +0200 Subject: [PATCH 12/21] check storage driver --- .buildkite/scripts/integration_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 7e562e1da5..114ee791c3 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -111,7 +111,7 @@ echo "--- Run integration test ${TARGET}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then make install - docker info | grep "Storage Driver" + docker info | grep "Driver" # allow to fail this command, to be able to upload safe logs set +e make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} From 532da8628b179dfa0655cf0ce3f26cbd8d160ae3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:05:42 +0200 Subject: [PATCH 13/21] Add some retries --- .../system/servicedeployer/terraform.go | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index fc12b5d528..608b9940b8 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/pkg/errors" @@ -20,6 +21,7 @@ import ( "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/signal" ) const ( @@ -45,15 +47,61 @@ type TerraformServiceDeployer struct { definitionsDir string } +// copied from system package +func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) { + timeoutTicker := time.NewTicker(timeout) + defer timeoutTicker.Stop() + + retryTicker := time.NewTicker(1 * time.Second) + defer retryTicker.Stop() + + for { + result, err := fn() + if err != nil { + return false, err + } + if result { + return true, nil + } + + select { + case <-retryTicker.C: + continue + case <-timeoutTicker.C: + return false, nil + } + } +} + // addTerraformOutputs method reads the terraform outputs generated in the json format and // adds them to the custom properties of ServiceContext and can be used in the handlebars template // like `{{TF_OUTPUT_queue_url}}` where `queue_url` is the output configured func addTerraformOutputs(outCtxt ServiceContext) error { // Read the `output.json` file where terraform outputs are generated outputFile := filepath.Join(outCtxt.OutputDir, terraformOutputJsonFile) - content, err := os.ReadFile(outputFile) + var content []byte + ready, err := waitUntilTrue(func() (bool, error) { + var err error + if signal.SIGINT() { + return true, errors.New("SIGINT: cancel clearing data") + } + + content, err = os.ReadFile(outputFile) + if err != nil { + return false, fmt.Errorf("failed to read terraform output file: %w", err) + } + if !json.Valid(content) { + time.Sleep(5 * time.Second) + return false, nil + } + return true, nil + }, 2*time.Minute) + if err != nil { - return fmt.Errorf("failed to read terraform output file: %w", err) + return err + } + if !ready { + logger.Debugf("Not ready output file") } // https://github.com/hashicorp/terraform/blob/v1.4.6/internal/command/views/output.go#L217-L222 @@ -64,7 +112,7 @@ func addTerraformOutputs(outCtxt ServiceContext) error { // Unmarshall the data into `terraformOutputs` logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta - logger.Debugf("**********JSON CONTENT****************\n%s***********************", string(content)) + logger.Debugf("***JSON CONTENT %s****************\n%s ***********************", outputFile, string(content)) if !json.Valid(content) { logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") return fmt.Errorf("error during JSON Unmarshal: %w", err) From ee508d762fc416faa1e5341aa335151363057239 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:08:36 +0200 Subject: [PATCH 14/21] Add more logs --- .../testrunner/runners/system/servicedeployer/terraform.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 608b9940b8..376678a959 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -86,14 +86,17 @@ func addTerraformOutputs(outCtxt ServiceContext) error { return true, errors.New("SIGINT: cancel clearing data") } + logger.Debugf("Trying to read %s", outputFile) content, err = os.ReadFile(outputFile) if err != nil { return false, fmt.Errorf("failed to read terraform output file: %w", err) } if !json.Valid(content) { + logger.Debugf("Invalid contents %s", outputFile) time.Sleep(5 * time.Second) return false, nil } + logger.Debugf("JSON valid %s", outputFile) return true, nil }, 2*time.Minute) From eb19760f447adbacebf66d37159cef41179fcc33 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:15:11 +0200 Subject: [PATCH 15/21] Move writing tf-applied after outputs --- .../servicedeployer/_static/terraform_deployer_run.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index afd0b531b5..b9a6c93cd8 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -20,15 +20,13 @@ trap cleanup EXIT INT TERM terraform init terraform plan -terraform apply -auto-approve && touch /tmp/tf-applied +terraform apply -auto-approve -set +e terraform output -json > /output/tfOutputValues.json -echo "Output Values content" cat /output/tfOutputValues.json -echo "-----" -set -e + +touch /tmp/tf-applied echo "Terraform definitions applied." From 6b8f6d7dc6433099e9480b159e902487850f0413 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:15:20 +0200 Subject: [PATCH 16/21] Revert "Add more logs" This reverts commit ee508d762fc416faa1e5341aa335151363057239. --- .../testrunner/runners/system/servicedeployer/terraform.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 376678a959..608b9940b8 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -86,17 +86,14 @@ func addTerraformOutputs(outCtxt ServiceContext) error { return true, errors.New("SIGINT: cancel clearing data") } - logger.Debugf("Trying to read %s", outputFile) content, err = os.ReadFile(outputFile) if err != nil { return false, fmt.Errorf("failed to read terraform output file: %w", err) } if !json.Valid(content) { - logger.Debugf("Invalid contents %s", outputFile) time.Sleep(5 * time.Second) return false, nil } - logger.Debugf("JSON valid %s", outputFile) return true, nil }, 2*time.Minute) From e5188ea7f6dffbb185befd8095bff6dad9b54b3c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:15:26 +0200 Subject: [PATCH 17/21] Revert "Add some retries" This reverts commit 532da8628b179dfa0655cf0ce3f26cbd8d160ae3. --- .../system/servicedeployer/terraform.go | 54 ++----------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 608b9940b8..fc12b5d528 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -11,7 +11,6 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/pkg/errors" @@ -21,7 +20,6 @@ import ( "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/signal" ) const ( @@ -47,61 +45,15 @@ type TerraformServiceDeployer struct { definitionsDir string } -// copied from system package -func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) { - timeoutTicker := time.NewTicker(timeout) - defer timeoutTicker.Stop() - - retryTicker := time.NewTicker(1 * time.Second) - defer retryTicker.Stop() - - for { - result, err := fn() - if err != nil { - return false, err - } - if result { - return true, nil - } - - select { - case <-retryTicker.C: - continue - case <-timeoutTicker.C: - return false, nil - } - } -} - // addTerraformOutputs method reads the terraform outputs generated in the json format and // adds them to the custom properties of ServiceContext and can be used in the handlebars template // like `{{TF_OUTPUT_queue_url}}` where `queue_url` is the output configured func addTerraformOutputs(outCtxt ServiceContext) error { // Read the `output.json` file where terraform outputs are generated outputFile := filepath.Join(outCtxt.OutputDir, terraformOutputJsonFile) - var content []byte - ready, err := waitUntilTrue(func() (bool, error) { - var err error - if signal.SIGINT() { - return true, errors.New("SIGINT: cancel clearing data") - } - - content, err = os.ReadFile(outputFile) - if err != nil { - return false, fmt.Errorf("failed to read terraform output file: %w", err) - } - if !json.Valid(content) { - time.Sleep(5 * time.Second) - return false, nil - } - return true, nil - }, 2*time.Minute) - + content, err := os.ReadFile(outputFile) if err != nil { - return err - } - if !ready { - logger.Debugf("Not ready output file") + return fmt.Errorf("failed to read terraform output file: %w", err) } // https://github.com/hashicorp/terraform/blob/v1.4.6/internal/command/views/output.go#L217-L222 @@ -112,7 +64,7 @@ func addTerraformOutputs(outCtxt ServiceContext) error { // Unmarshall the data into `terraformOutputs` logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta - logger.Debugf("***JSON CONTENT %s****************\n%s ***********************", outputFile, string(content)) + logger.Debugf("**********JSON CONTENT****************\n%s***********************", string(content)) if !json.Valid(content) { logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") return fmt.Errorf("error during JSON Unmarshal: %w", err) From 6c6d4e77b94bcf0a84d3898320fe7a6ceadb92ed Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:34:52 +0200 Subject: [PATCH 18/21] Revert debug --- .buildkite/scripts/integration_tests.sh | 1 - .../_static/terraform_deployer.yml | 8 ++---- .../system/servicedeployer/terraform.go | 16 ++++------- .../system/servicedeployer/terraform_test.go | 28 +++++++++++-------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 114ee791c3..fb1637c117 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -111,7 +111,6 @@ echo "--- Run integration test ${TARGET}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then make install - docker info | grep "Driver" # allow to fail this command, to be able to upload safe logs set +e make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml index 5007652a61..794446d1ba 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml @@ -12,9 +12,5 @@ services: - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown} - TF_VAR_REPO=${REPO:-unknown} volumes: - - type: bind - source: ${TF_DIR} - target: /stage - - type: bind - source: ${TF_OUTPUT_DIR} - target: /output + - ${TF_DIR}:/stage + - ${TF_OUTPUT_DIR}:/output diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index fc12b5d528..3ae49666a1 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -64,11 +64,11 @@ func addTerraformOutputs(outCtxt ServiceContext) error { // Unmarshall the data into `terraformOutputs` logger.Debug("Unmarshalling terraform output json") var terraformOutputs map[string]OutputMeta - logger.Debugf("**********JSON CONTENT****************\n%s***********************", string(content)) - if !json.Valid(content) { - logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") - return fmt.Errorf("error during JSON Unmarshal: %w", err) - } + // logger.Debugf("**********JSON CONTENT****************\n%s***********************", string(content)) + // if !json.Valid(content) { + // logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") + // return fmt.Errorf("error during JSON Unmarshal: %w", err) + // } if err = json.Unmarshal(content, &terraformOutputs); err != nil { return fmt.Errorf("error during JSON Unmarshal: %w", err) @@ -166,11 +166,7 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic err = addTerraformOutputs(outCtxt) if err != nil { - if strings.HasPrefix(err.Error(), "error during JSON Unmarshal") { - logger.Errorf(err.Error()) - return &service, nil - } - return nil, fmt.Errorf("could not handle terraform output %w", err) + return nil, fmt.Errorf("could not handle terraform output: %w", err) } service.ctxt = outCtxt return &service, nil diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_test.go b/internal/testrunner/runners/system/servicedeployer/terraform_test.go index 54c280f98b..c8ee673b0f 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_test.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_test.go @@ -20,18 +20,20 @@ func TestAddTerraformOutputs(t *testing.T) { runId string content []byte expectedProps map[string]interface{} + expectedError bool }{ - // { - // testName: "invalid_json_output", - // runId: "987987", - // ctxt: ServiceContext{ - // Test: struct{ RunID string }{"987987"}, - // }, - // content: []byte( - // ``, - // ), - // expectedProps: map[string]interface{}{}, - // }, + { + testName: "invalid_json_output", + runId: "987987", + ctxt: ServiceContext{ + Test: struct{ RunID string }{"987987"}, + }, + content: []byte( + ``, + ), + expectedProps: map[string]interface{}{}, + expectedError: true, + }, { testName: "empty_json_output", runId: "v", @@ -145,6 +147,10 @@ func TestAddTerraformOutputs(t *testing.T) { // Test that the terraform output values are generated correctly err := addTerraformOutputs(tc.ctxt) + if tc.expectedError { + require.Error(t, err) + return + } require.NoError(t, err) assert.Equal(t, tc.expectedProps, tc.ctxt.CustomProperties) }) From 61a376f4a577510821d19043444c53db92c65d13 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 9 Jun 2023 16:37:46 +0200 Subject: [PATCH 19/21] Remove debug --- .../runners/system/servicedeployer/terraform.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 3ae49666a1..d64d40e45d 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -62,14 +62,8 @@ func addTerraformOutputs(outCtxt ServiceContext) error { } // Unmarshall the data into `terraformOutputs` - logger.Debug("Unmarshalling terraform output json") + logger.Debug("Unmarshalling terraform output JSON") var terraformOutputs map[string]OutputMeta - // logger.Debugf("**********JSON CONTENT****************\n%s***********************", string(content)) - // if !json.Valid(content) { - // logger.Debug("Invalid Json content in the terraform output file, skipped creating outputs") - // return fmt.Errorf("error during JSON Unmarshal: %w", err) - // } - if err = json.Unmarshal(content, &terraformOutputs); err != nil { return fmt.Errorf("error during JSON Unmarshal: %w", err) } From 2708e4c2974e3c9704d40a95be1a98ecbfb67745 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 12 Jun 2023 10:27:17 +0200 Subject: [PATCH 20/21] Remove cat output file --- .../system/servicedeployer/_static/terraform_deployer_run.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index b9a6c93cd8..3868688f75 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -24,8 +24,6 @@ terraform apply -auto-approve terraform output -json > /output/tfOutputValues.json -cat /output/tfOutputValues.json - touch /tmp/tf-applied echo "Terraform definitions applied." From d669032b54c35e3e6e853d9e48605a6460c223dc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 12 Jun 2023 12:47:50 +0200 Subject: [PATCH 21/21] Add comment --- .../system/servicedeployer/_static/terraform_deployer_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh index 3868688f75..80054b46c2 100644 --- a/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh +++ b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh @@ -24,7 +24,7 @@ terraform apply -auto-approve terraform output -json > /output/tfOutputValues.json -touch /tmp/tf-applied +touch /tmp/tf-applied # This file is used as indicator (healthcheck) that the service is UP, and so it must be placed as the last statement in the script echo "Terraform definitions applied."