Merge pull request #1513 from k8s-infra-cherrypick-robot/cherry-pick-…

…1493-to-release-1.29

[release-1.29] [test] add mock for azcopy ut

hack/boilerplate/boilerplate.generatego.txt

@@ -0,0 +1,15 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/

hack/boilerplate/boilerplate.gomock.txt

@@ -0,0 +1,15 @@
+// /*
+// Copyright The Kubernetes Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// */

hack/boilerplate/boilerplate.py

@@ -65,6 +65,14 @@ def get_refs():
 
     return refs
 
+def is_generated_file(filename, data, regexs):
+    for d in skipped_ungenerated_files:
+        if d in filename:
+            return False
+
+    p = regexs["generated"]
+    return p.search(data)
+
 def file_passes(filename, refs, regexs):
     try:
         f = open(filename, 'r')
@@ -75,15 +83,21 @@ def file_passes(filename, refs, regexs):
     data = f.read()
     f.close()
 
+    # determine if the file is automatically generated
+    generated = is_generated_file(filename, data, regexs)
+
     basename = os.path.basename(filename)
     extension = file_extension(filename)
+    if generated:
+        if extension == "go":
+            extension = "gomock"
     if extension != "":
         ref = refs[extension]
     else:
         ref = refs[basename]
 
     # remove build tags from the top of Go files
-    if extension == "go":
+    if extension == "go" or extension == "gomock":
         p = regexs["go_build_constraints"]
         (data, found) = p.subn("", data, 1)
 
@@ -136,6 +150,10 @@ def file_extension(filename):
                 'cluster/env.sh', 'vendor', 'test/e2e/generated/bindata.go',
                 'repo-infra/verify/boilerplate/test', '.glide']
 
+ # list all the files contain 'DO NOT EDIT', but are not generated
+skipped_ungenerated_files = [
+    'hack/boilerplate/boilerplate.py']
+
 def normalize_files(files):
     newfiles = []
     for pathname in files:
@@ -183,6 +201,8 @@ def get_regexs():
     regexs["go_build_constraints"] = re.compile(r"^(//(go:build| \+build).*\n)+\n", re.MULTILINE)
     # strip #!.* from shell scripts
     regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
+    # Search for generated files
+    regexs["generated"] = re.compile('DO NOT EDIT')
     return regexs
 
 

hack/update-mock.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+REPO_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
+COPYRIGHT_FILE="${REPO_ROOT}/hack/boilerplate/boilerplate.generatego.txt"
+
+if ! type mockgen &> /dev/null; then
+    echo "mockgen not exist, install it"
+    go install github.com/golang/mock/mockgen@v1.6.0
+fi
+
+echo "Updating mocks for util.go"
+mockgen -copyright_file=$COPYRIGHT_FILE -source=pkg/util/util.go -package=util -destination=pkg/util/util_mock.go

pkg/azurefile/azurefile.go

@@ -46,6 +46,7 @@ import (
 
 	csicommon "sigs.k8s.io/azurefile-csi-driver/pkg/csi-common"
 	"sigs.k8s.io/azurefile-csi-driver/pkg/mounter"
+	fileutil "sigs.k8s.io/azurefile-csi-driver/pkg/util"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/fileclient"
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
@@ -269,6 +270,8 @@ type Driver struct {
 	volStatsCache azcache.Resource
 	// sas expiry time for azcopy in volume clone
 	sasTokenExpirationMinutes int
+	// azcopy for provide exec mock for ut
+	azcopy *fileutil.Azcopy
 }
 
 // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
@@ -300,6 +303,7 @@ func NewDriver(options *DriverOptions) *Driver {
 	driver.volLockMap = newLockMap()
 	driver.subnetLockMap = newLockMap()
 	driver.volumeLocks = newVolumeLocks()
+	driver.azcopy = &fileutil.Azcopy{}
 
 	var err error
 	getter := func(key string) (interface{}, error) { return nil, nil }
@@ -928,21 +932,21 @@ func (d *Driver) copyFileShare(ctx context.Context, req *csi.CreateVolumeRequest
 	srcPath := fmt.Sprintf("https://%s.file.%s/%s%s", accountName, storageEndpointSuffix, srcFileShareName, accountSasToken)
 	dstPath := fmt.Sprintf("https://%s.file.%s/%s%s", accountName, storageEndpointSuffix, dstFileShareName, accountSasToken)
 
-	jobState, percent, err := getAzcopyJob(dstFileShareName)
+	jobState, percent, err := d.azcopy.GetAzcopyJob(dstFileShareName)
 	klog.V(2).Infof("azcopy job status: %s, copy percent: %s%%, error: %v", jobState, percent, err)
-	if jobState == AzcopyJobError || jobState == AzcopyJobCompleted {
+	if jobState == fileutil.AzcopyJobError || jobState == fileutil.AzcopyJobCompleted {
 		return err
 	}
 	klog.V(2).Infof("begin to copy fileshare %s to %s", srcFileShareName, dstFileShareName)
 	for {
 		select {
 		case <-timeTick:
-			jobState, percent, err := getAzcopyJob(dstFileShareName)
+			jobState, percent, err := d.azcopy.GetAzcopyJob(dstFileShareName)
 			klog.V(2).Infof("azcopy job status: %s, copy percent: %s%%, error: %v", jobState, percent, err)
 			switch jobState {
-			case AzcopyJobError, AzcopyJobCompleted:
+			case fileutil.AzcopyJobError, fileutil.AzcopyJobCompleted:
 				return err
-			case AzcopyJobNotFound:
+			case fileutil.AzcopyJobNotFound:
 				klog.V(2).Infof("copy fileshare %s to %s", srcFileShareName, dstFileShareName)
 				out, copyErr := exec.Command("azcopy", "copy", srcPath, dstPath, "--recursive", "--check-length=false").CombinedOutput()
 				if copyErr != nil {

pkg/azurefile/controllerserver.go

@@ -108,7 +108,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 	if acquired := d.volumeLocks.TryAcquire(volName); !acquired {
 		// logging the job status if it's volume cloning
 		if req.GetVolumeContentSource() != nil {
-			jobState, percent, err := getAzcopyJob(volName)
+			jobState, percent, err := d.azcopy.GetAzcopyJob(volName)
 			klog.V(2).Infof("azcopy job status: %s, copy percent: %s%%, error: %v", jobState, percent, err)
 		}
 		return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volName)

pkg/azurefile/controllerserver_test.go

@@ -23,12 +23,14 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
-	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/fileclient"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"sigs.k8s.io/azurefile-csi-driver/pkg/util"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/fileclient"
+
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/subnetclient/mocksubnetclient"
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
@@ -1783,6 +1785,95 @@ func TestCopyVolume(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "azcopy job is already completed",
+			testFunc: func(t *testing.T) {
+				d := NewFakeDriver()
+				mp := map[string]string{}
+
+				volumeSource := &csi.VolumeContentSource_VolumeSource{
+					VolumeId: "vol_1#f5713de20cde511e8ba4900#fileshare#",
+				}
+				volumeContentSourceVolumeSource := &csi.VolumeContentSource_Volume{
+					Volume: volumeSource,
+				}
+				volumecontensource := csi.VolumeContentSource{
+					Type: volumeContentSourceVolumeSource,
+				}
+
+				req := &csi.CreateVolumeRequest{
+					Name:                "unit-test",
+					VolumeCapabilities:  stdVolCap,
+					Parameters:          mp,
+					VolumeContentSource: &volumecontensource,
+				}
+
+				ctrl := gomock.NewController(t)
+				defer ctrl.Finish()
+
+				m := util.NewMockEXEC(ctrl)
+				listStr := "JobId: ed1c3833-eaff-fe42-71d7-513fb065a9d9\nStart Time: Monday, 07-Aug-23 03:29:54 UTC\nStatus: Completed\nCommand: copy https://{accountName}.file.core.windows.net/{srcFileshare}{SAStoken} https://{accountName}.file.core.windows.net/{dstFileshare}{SAStoken} --recursive --check-length=false"
+				m.EXPECT().RunCommand(gomock.Eq("azcopy jobs list | grep dstFileshare -B 3")).Return(listStr, nil)
+				// if test.enableShow {
+				// 	m.EXPECT().RunCommand(gomock.Not("azcopy jobs list | grep dstContainer -B 3")).Return(test.showStr, test.showErr)
+				// }
+
+				d.azcopy.ExecCmd = m
+
+				ctx := context.Background()
+
+				var expectedErr error
+				err := d.copyVolume(ctx, req, "", &fileclient.ShareOptions{Name: "dstFileshare"}, "core.windows.net")
+				if !reflect.DeepEqual(err, expectedErr) {
+					t.Errorf("Unexpected error: %v", err)
+				}
+			},
+		},
+		{
+			name: "azcopy job is first in progress and then be completed",
+			testFunc: func(t *testing.T) {
+				d := NewFakeDriver()
+				mp := map[string]string{}
+
+				volumeSource := &csi.VolumeContentSource_VolumeSource{
+					VolumeId: "vol_1#f5713de20cde511e8ba4900#fileshare#",
+				}
+				volumeContentSourceVolumeSource := &csi.VolumeContentSource_Volume{
+					Volume: volumeSource,
+				}
+				volumecontensource := csi.VolumeContentSource{
+					Type: volumeContentSourceVolumeSource,
+				}
+
+				req := &csi.CreateVolumeRequest{
+					Name:                "unit-test",
+					VolumeCapabilities:  stdVolCap,
+					Parameters:          mp,
+					VolumeContentSource: &volumecontensource,
+				}
+
+				ctrl := gomock.NewController(t)
+				defer ctrl.Finish()
+
+				m := util.NewMockEXEC(ctrl)
+				listStr1 := "JobId: ed1c3833-eaff-fe42-71d7-513fb065a9d9\nStart Time: Monday, 07-Aug-23 03:29:54 UTC\nStatus: InProgress\nCommand: copy https://{accountName}.file.core.windows.net/{srcFileshare}{SAStoken} https://{accountName}.file.core.windows.net/{dstFileshare}{SAStoken} --recursive --check-length=false"
+				listStr2 := "JobId: ed1c3833-eaff-fe42-71d7-513fb065a9d9\nStart Time: Monday, 07-Aug-23 03:29:54 UTC\nStatus: Completed\nCommand: copy https://{accountName}.file.core.windows.net/{srcFileshare}{SAStoken} https://{accountName}.file.core.windows.net/{dstFileshare}{SAStoken} --recursive --check-length=false"
+				o1 := m.EXPECT().RunCommand(gomock.Eq("azcopy jobs list | grep dstFileshare -B 3")).Return(listStr1, nil).Times(1)
+				m.EXPECT().RunCommand(gomock.Not("azcopy jobs list | grep dstFileshare -B 3")).Return("Percent Complete (approx): 50.0", nil)
+				o2 := m.EXPECT().RunCommand(gomock.Eq("azcopy jobs list | grep dstFileshare -B 3")).Return(listStr2, nil)
+				gomock.InOrder(o1, o2)
+
+				d.azcopy.ExecCmd = m
+
+				ctx := context.Background()
+
+				var expectedErr error
+				err := d.copyVolume(ctx, req, "", &fileclient.ShareOptions{Name: "dstFileshare"}, "core.windows.net")
+				if !reflect.DeepEqual(err, expectedErr) {
+					t.Errorf("Unexpected error: %v", err)
+				}
+			},
+		},
 	}
 
 	for _, tc := range testCases {