Merge pull request #711 from ystia/feature/GH-707_Add_a_new_synchrono…

…us_purge_API_endpoint

Implement a synchronous purge endpoint

CHANGELOG.md

@@ -20,6 +20,7 @@
 
 ### ENHANCEMENTS
 
+* Add a new synchronous purge API endpoint ([GH-707](https://github.com/ystia/yorc/issues/707))
 * Should be able to specify the type of volume when creating an openstack instance ([GH-703](https://github.com/ystia/yorc/issues/703))
 * Support ssh connection retries ([GH-688](https://github.com/ystia/yorc/issues/688))
 * Remove useless/cluttering logs ([GH-681](https://github.com/ystia/yorc/issues/681))

commands/deployments/dep_purge.go

@@ -0,0 +1,107 @@
+// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deployments
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/ystia/yorc/v4/commands/httputil"
+	"github.com/ystia/yorc/v4/log"
+	"github.com/ystia/yorc/v4/rest"
+)
+
+func init() {
+	var force bool
+	var purgeCmd = &cobra.Command{
+		Use:   "purge <id>",
+		Short: "purge a deployment",
+		Long: `Purge a deployment <id>. This deployment should be in UNDEPLOYED status.
+If an error is encountered the purge process is stopped and the deployment status is set
+to PURGE_FAILED.
+
+A purge may be run in force mode. In this mode Yorc does not check if the deployment is in
+UNDEPLOYED status or even if the deployment exist. Moreover, in force mode the purge process
+doesn't fail-fast and try to delete as much as it can. An report with encountered errors is
+produced at the end of the process.`,
+		Args: cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			client, err := httputil.GetClient(ClientConfig)
+			if err != nil {
+				httputil.ErrExit(err)
+			}
+			deploymentID := args[0]
+
+			err = postPurgeRequest(client, deploymentID, force)
+			if err != nil {
+				os.Exit(1)
+			}
+
+		},
+	}
+	purgeCmd.Flags().BoolVarP(&force, "force", "f", false, "Force purge of a deployment ignoring states checks and any errors. This should be use with extrem caution to cleanup environment.")
+	DeploymentsCmd.AddCommand(purgeCmd)
+}
+
+func postPurgeRequest(client httputil.HTTPClient, deploymentID string, force bool) error {
+	request, err := client.NewRequest("POST", path.Join("/deployments", deploymentID, "purge"), nil)
+	if err != nil {
+		httputil.ErrExit(errors.Wrap(err, httputil.YorcAPIDefaultErrorMsg))
+	}
+
+	query := request.URL.Query()
+	if force {
+		query.Set("force", strconv.FormatBool(force))
+	}
+	request.URL.RawQuery = query.Encode()
+	request.Header.Add("Accept", "application/json")
+	log.Debugf("POST: %s", request.URL.String())
+
+	response, err := client.Do(request)
+	if err != nil {
+		httputil.ErrExit(errors.Wrap(err, httputil.YorcAPIDefaultErrorMsg))
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode == http.StatusOK {
+		ioutil.ReadAll(response.Body)
+		return nil
+	}
+	var errs rest.Errors
+	bodyContent, _ := ioutil.ReadAll(response.Body)
+	json.Unmarshal(bodyContent, &errs)
+
+	if len(errs.Errors) > 0 {
+		var err *multierror.Error
+		fmt.Println("Got errors on purge:")
+		for _, e := range errs.Errors {
+			fmt.Printf("  * Error: %s: %s\n", e.Title, e.Detail)
+			multierror.Append(err, e)
+		}
+
+		return err
+	}
+
+	return nil
+}

commands/deployments/dep_purge_test.go

@@ -0,0 +1,131 @@
+// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deployments
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/ystia/yorc/v4/rest"
+)
+
+type mockHTTPClient struct {
+	GetFunc        func(path string) (*http.Response, error)
+	HeadFunc       func(path string) (*http.Response, error)
+	PostFunc       func(path string, contentType string, body io.Reader) (*http.Response, error)
+	PostFormFunc   func(path string, data url.Values) (*http.Response, error)
+	NewRequestFunc func(method, path string, body io.Reader) (*http.Request, error)
+	DoFunc         func(req *http.Request) (*http.Response, error)
+}
+
+func (m *mockHTTPClient) Get(path string) (*http.Response, error) {
+	if m.GetFunc == nil {
+		panic("missing mock test function GetFunc")
+	}
+	return m.GetFunc(path)
+}
+func (m *mockHTTPClient) Head(path string) (*http.Response, error) {
+	if m.HeadFunc == nil {
+		panic("missing mock test function HeadFunc")
+	}
+	return m.HeadFunc(path)
+}
+func (m *mockHTTPClient) Post(path string, contentType string, body io.Reader) (*http.Response, error) {
+	if m.PostFunc == nil {
+		panic("missing mock test function PostFunc")
+	}
+	return m.PostFunc(path, contentType, body)
+}
+func (m *mockHTTPClient) PostForm(path string, data url.Values) (*http.Response, error) {
+	if m.PostFormFunc == nil {
+		panic("missing mock test function PostFormFunc")
+	}
+	return m.PostFormFunc(path, data)
+}
+
+func (m *mockHTTPClient) NewRequest(method, path string, body io.Reader) (*http.Request, error) {
+	if m.NewRequestFunc != nil {
+		return m.NewRequestFunc(method, path, body)
+	}
+	return httptest.NewRequest(method, path, body), nil
+}
+
+func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	if m.DoFunc == nil {
+		panic("missing mock test function DoFunc")
+	}
+	return m.DoFunc(req)
+}
+
+func Test_postPurgeRequest(t *testing.T) {
+
+	type args struct {
+		doRequest func(req *http.Request) (*http.Response, error)
+		force     bool
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{"OkNoForce", args{doRequest: func(req *http.Request) (*http.Response, error) {
+			errs := new(rest.Errors)
+			b, err := json.Marshal(errs)
+			if err != nil {
+				return nil, err
+			}
+			res := &httptest.ResponseRecorder{
+				Code: 200,
+				Body: bytes.NewBuffer(b),
+			}
+
+			return res.Result(), nil
+		}, force: false}, false},
+		{"ForceWithErrors", args{doRequest: func(req *http.Request) (*http.Response, error) {
+			errs := &rest.Errors{
+				Errors: []*rest.Error{
+					{ID: "internal_server_error", Status: 500, Title: "Internal Server Error", Detail: "error 1"},
+					{ID: "internal_server_error", Status: 500, Title: "Internal Server Error", Detail: "error 2"},
+					{ID: "internal_server_error", Status: 500, Title: "Internal Server Error", Detail: "error 3"},
+				},
+			}
+			b, err := json.Marshal(errs)
+			if err != nil {
+				return nil, err
+			}
+			res := &httptest.ResponseRecorder{
+				Code: 500,
+				Body: bytes.NewBuffer(b),
+			}
+
+			return res.Result(), nil
+		}, force: true}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mockClient := &mockHTTPClient{
+				DoFunc: tt.args.doRequest,
+			}
+			if err := postPurgeRequest(mockClient, "deploymentID", tt.args.force); (err != nil) != tt.wantErr {
+				t.Errorf("postPurgeRequest() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

deployments/deployments.go

@@ -45,7 +45,7 @@ func (d deploymentNotFound) Error() string {
 // IsDeploymentNotFoundError checks if an error is a deployment not found error
 func IsDeploymentNotFoundError(err error) bool {
 	cause := errors.Cause(err)
-	_, ok := cause.(deploymentNotFound)
+	_, ok := cause.(*deploymentNotFound)
 	return ok
 }
 
@@ -97,7 +97,7 @@ func GetDeploymentStatus(ctx context.Context, deploymentID string) (DeploymentSt
 		return INITIAL, errors.Wrap(err, consulutil.ConsulGenericErrMsg)
 	}
 	if !exist || value == "" {
-		return INITIAL, deploymentNotFound{deploymentID: deploymentID}
+		return INITIAL, errors.WithStack(&deploymentNotFound{deploymentID: deploymentID})
 	}
 	return DeploymentStatusFromString(value, true)
 }

deployments/structs.go

@@ -37,6 +37,7 @@ UPDATE_IN_PROGRESS
 UPDATED
 UPDATE_FAILURE
 PURGED
+PURGE_FAILED
 )
 */
 type DeploymentStatus int

go.mod

@@ -49,7 +49,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.1
 	github.com/hashicorp/go-hclog v0.8.0
 	github.com/hashicorp/go-memdb v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-plugin v1.0.0
 	github.com/hashicorp/go-rootcerts v1.0.0
 	github.com/hashicorp/serf v0.8.3 // indirect

go.sum

@@ -221,6 +221,8 @@ github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqk
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-plugin v1.0.0 h1:/gQ1sNR8/LHpoxKRQq4PmLBuacfZb4tC93e9B30o/7c=
 github.com/hashicorp/go-plugin v1.0.0/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=

internal/operations/consul_test.go

@@ -0,0 +1,71 @@
+// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package operations
+
+import (
+	"context"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/stretchr/testify/require"
+	"github.com/ystia/yorc/v4/deployments"
+	"github.com/ystia/yorc/v4/helper/consulutil"
+	"github.com/ystia/yorc/v4/tasks"
+	"github.com/ystia/yorc/v4/testutil"
+)
+
+// The aim of this function is to run all package tests with consul server dependency with only one consul server start
+func TestConsul(t *testing.T) {
+	cfg := testutil.SetupTestConfig(t)
+	srv, client := testutil.NewTestConsulInstance(t, &cfg)
+	defer func() {
+		srv.Stop()
+		os.RemoveAll(cfg.WorkingDirectory)
+	}()
+
+	t.Run("groupInternalOperations", func(t *testing.T) {
+		testPurgeTasks(t, srv, client)
+		testPurgeDeployment(t, cfg, srv, client)
+		testPurgeDeploymentPreChecks(t, cfg, srv, client)
+		testEnsurePurgeFailedStatus(t, cfg, srv, client)
+	})
+
+}
+
+func createTaskKV(t *testing.T, taskID string) {
+	t.Helper()
+
+	var keyValue string
+	keyValue = strconv.Itoa(int(tasks.TaskStatusINITIAL))
+	_, err := consulutil.GetKV().Put(&api.KVPair{Key: path.Join(consulutil.TasksPrefix, taskID, "status"), Value: []byte(keyValue)}, nil)
+	require.NoError(t, err)
+	_, err = consulutil.GetKV().Put(&api.KVPair{Key: path.Join(consulutil.TasksPrefix, taskID, "targetId"), Value: []byte(taskID)}, nil)
+	require.NoError(t, err)
+
+	creationDate := time.Now()
+	keyValue = creationDate.Format(time.RFC3339Nano)
+	_, err = consulutil.GetKV().Put(&api.KVPair{Key: path.Join(consulutil.TasksPrefix, taskID, "creationDate"), Value: []byte(keyValue)}, nil)
+	require.NoError(t, err)
+}
+
+func loadTestYaml(t testing.TB, deploymentID string) {
+	yamlName := "testdata/topology.yaml"
+	err := deployments.StoreDeploymentDefinition(context.Background(), deploymentID, yamlName)
+	require.Nil(t, err, "Failed to parse "+yamlName+" definition")
+}