fix: download subdirs in azure artifact. Fixes #11385 (#11394)

Signed-off-by: Roel Arents <roel.arents@kadaster.nl>

go.mod

@@ -81,13 +81,13 @@ require (
 )
 
 require (
-	cloud.google.com/go v0.110.0 // indirect
-	cloud.google.com/go/compute v1.19.3 // indirect
+	cloud.google.com/go v0.110.2 // indirect
+	cloud.google.com/go/compute v1.20.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.13.0 // indirect
+	cloud.google.com/go/iam v1.1.0 // indirect
 	github.com/Azure/azure-sdk-for-go v62.0.0+incompatible // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.24 // indirect

workflow/artifacts/azure/azure.go

@@ -155,6 +155,10 @@ func DownloadFile(containerClient *azblob.ContainerClient, blobName, path string
 		return fmt.Errorf("unable to create Azure Blob client for %s: %s", blobName, err)
 	}
 
+	err := os.MkdirAll(filepath.Dir(path), 0755)
+	if err != nil {
+		return fmt.Errorf("unable to create dir for file %s: %s", path, err)
+	}
 	outFile, err := os.Create(path)
 	if err != nil {
 		return fmt.Errorf("unable to create file %s: %s", path, err)
@@ -178,7 +182,7 @@ func (azblobDriver *ArtifactDriver) DownloadDirectory(containerClient *azblob.Co
 		return fmt.Errorf("unable to list blob %s in Azure Storage: %s", artifact.Azure.Blob, err)
 	}
 
-	err = os.Mkdir(path, 0755)
+	err = os.MkdirAll(path, 0755)
 	if err != nil {
 		return fmt.Errorf("unable to create local directory %s: %s", path, err)
 	}

workflow/artifacts/azure/azure_test.go

@@ -1,9 +1,16 @@
 package azure
 
 import (
+	"context"
+	"errors"
 	"net/url"
+	"path/filepath"
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+
+	wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,3 +44,41 @@ func TestDetermineAccountName(t *testing.T) {
 		assert.Equal(t, "", accountName)
 	}
 }
+
+func TestArtifactDriver_DownloadDirectory_Subdir(t *testing.T) {
+	t.Skipf("This test needs azurite. docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite:latest azurite-blob")
+
+	driver := ArtifactDriver{
+		AccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", // default azurite key
+		Container:  "test",
+		Endpoint:   "http://127.0.0.1:10000/devstoreaccount1",
+	}
+
+	// ensure container exists
+	containerClient, err := driver.newAzureContainerClient()
+	assert.NoError(t, err)
+	_, err = containerClient.Create(context.Background(), nil)
+	var responseError *azcore.ResponseError
+	if err != nil && !(errors.As(err, &responseError) && responseError.ErrorCode == "ContainerAlreadyExists") {
+		assert.NoError(t, err)
+	}
+
+	// put a file in a subdir on the azurite blob storage
+	blobClient := containerClient.NewBlockBlobClient("dir/subdir/file-in-subdir.txt")
+	_, err = blobClient.UploadBuffer(context.Background(), []byte("foo"), nil)
+	assert.NoError(t, err)
+
+	// download the dir, containing a subdir
+	azureArtifact := wfv1.AzureArtifact{
+		Blob: "dir",
+	}
+	argoArtifact := wfv1.Artifact{
+		ArtifactLocation: wfv1.ArtifactLocation{
+			Azure: &azureArtifact,
+		},
+	}
+	dstDir := t.TempDir()
+	err = driver.DownloadDirectory(containerClient, &argoArtifact, filepath.Join(dstDir, "dir"))
+	assert.NoError(t, err)
+	assert.FileExists(t, filepath.Join(dstDir, "dir", "subdir", "file-in-subdir.txt"))
+}