Add error codes for test failures (#5385)

* Adding error codes for test failures

* Adding error code for structure tests

* Updating Copyright year

docs/content/en/docs/references/api/grpc.md

@@ -959,6 +959,9 @@ For Cancelled Error code, use range 800 to 850.<br>
 | DEPLOY_MANIFEST_WRITE_ERR | 1020 | Error writing hydrated kubernetes manifests. |
 | DEPLOY_PARSE_MANIFEST_IMAGES_ERR | 1021 | Error getting images from a kubernetes manifest. |
 | DEPLOY_HELM_CREATE_NS_NOT_AVAILABLE | 1022 | Helm config `createNamespace` not available |
+| TEST_USER_CONFIG_ERR | 1101 | Error expanding paths |
+| TEST_CST_USER_ERR | 1102 | Error running container-structure-test |
+| TEST_IMG_PULL_ERR | 1103 | Unable to docker pull image |
 
 
 

pkg/skaffold/test/error.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import (
+	"fmt"
+
+	sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors"
+	"github.com/GoogleContainerTools/skaffold/proto"
+)
+
+func dockerPullImageErr(fqn string, err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("unable to docker pull image %s: %s", fqn, err),
+			ErrCode: proto.StatusCode_TEST_IMG_PULL_ERR,
+		},
+	)
+}
+
+func expandingFilePathsErr(err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("expanding test file paths: %s", err),
+			ErrCode: proto.StatusCode_TEST_USER_CONFIG_ERR,
+		},
+	)
+}

pkg/skaffold/test/structure/error.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package structure
+
+import (
+	"fmt"
+
+	sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors"
+	"github.com/GoogleContainerTools/skaffold/proto"
+)
+
+func containerStructureTestErr(err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("running container-structure-test: %s", err),
+			ErrCode: proto.StatusCode_TEST_CST_USER_ERR,
+		},
+	)
+}

pkg/skaffold/test/structure/structure.go

@@ -18,7 +18,6 @@ package structure
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -43,7 +42,7 @@ func (tr *Runner) Test(ctx context.Context, out io.Writer, image string) error {
 	cmd.Env = tr.env()
 
 	if err := util.RunCmd(cmd); err != nil {
-		return fmt.Errorf("running container-structure-test: %w", err)
+		return containerStructureTestErr(err)
 	}
 
 	return nil

pkg/skaffold/test/test.go

@@ -67,7 +67,7 @@ func (t FullTester) TestDependencies() ([]string, error) {
 	for _, test := range t.testCases {
 		files, err := util.ExpandPathsGlob(t.workingDir, test.StructureTests)
 		if err != nil {
-			return nil, fmt.Errorf("expanding test file paths: %w", err)
+			return nil, expandingFilePathsErr(err)
 		}
 
 		deps = append(deps, files...)
@@ -139,13 +139,13 @@ func (t FullTester) runStructureTests(ctx context.Context, out io.Writer, bRes [
 		// `container-structure-test` currently can't do it:
 		// https://github.com/GoogleContainerTools/container-structure-test/issues/253.
 		if err := t.localDaemon.Pull(ctx, out, fqn); err != nil {
-			return fmt.Errorf("unable to docker pull image %q: %w", fqn, err)
+			return dockerPullImageErr(fqn, err)
 		}
 	}
 
 	files, err := util.ExpandPathsGlob(t.workingDir, tc.StructureTests)
 	if err != nil {
-		return fmt.Errorf("expanding test file paths: %w", err)
+		return expandingFilePathsErr(err)
 	}
 
 	runner := structure.NewRunner(files, t.localDaemon.ExtraEnv())

pkg/skaffold/test/test_test.go

@@ -197,7 +197,7 @@ func TestTestFailureRemoteImage(t *testing.T) {
 			Tag:       "image:tag",
 		}})
 
-		t.CheckErrorContains(`unable to docker pull image "image:tag"`, err)
+		t.CheckErrorContains(`unable to docker pull image image:tag`, err)
 	})
 }
 

proto/skaffold.proto

@@ -681,6 +681,15 @@ enum StatusCode {
     DEPLOY_PARSE_MANIFEST_IMAGES_ERR = 1021;
     // Helm config `createNamespace` not available
     DEPLOY_HELM_CREATE_NS_NOT_AVAILABLE = 1022;
+
+    // Test errors
+
+    // Error expanding paths
+    TEST_USER_CONFIG_ERR = 1101;
+    // Error running container-structure-test
+    TEST_CST_USER_ERR = 1102;
+    // Unable to docker pull image
+    TEST_IMG_PULL_ERR = 1103;
 }
 
 // Enum for Suggestion codes