From e7104c909e871c94d88a01a3ca317e61dbbed4fe Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:24:17 -0600 Subject: [PATCH 01/18] First pass at getting size of bucket --- component/file_cache/file_cache_linux.go | 31 +-------- component/s3storage/client.go | 82 ++++++++++++++++++++++++ component/s3storage/connection.go | 1 + component/s3storage/s3storage.go | 25 ++++++++ 4 files changed, 109 insertions(+), 30 deletions(-) diff --git a/component/file_cache/file_cache_linux.go b/component/file_cache/file_cache_linux.go index 4db6572c4..e7ca9c288 100644 --- a/component/file_cache/file_cache_linux.go +++ b/component/file_cache/file_cache_linux.go @@ -29,13 +29,10 @@ package file_cache import ( "io/fs" - "math" "os" "syscall" "time" - "golang.org/x/sys/unix" - "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" "github.com/Seagate/cloudfuse/internal" @@ -149,31 +146,5 @@ func (c *FileCache) StatFs() (*common.Statfs_t, bool, error) { // cache_size - used = f_frsize * f_bavail/1024 // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 // if cache size is set to 0 then we have the root mount usage - maxCacheSize := c.maxCacheSize * MB - if maxCacheSize == 0 { - return nil, false, nil - } - usage, _ := common.GetUsage(c.tmpPath) - usage *= MB - - available := maxCacheSize - usage - statfs := &unix.Statfs_t{} - err := unix.Statfs("/", statfs) - if err != nil { - log.Debug("FileCache::StatFs : statfs err [%s].", err.Error()) - return nil, false, err - } - - stat := common.Statfs_t{ - Blocks: uint64(maxCacheSize) / uint64(statfs.Frsize), - Bavail: uint64(math.Max(0, available)) / uint64(statfs.Frsize), - Bfree: statfs.Bavail, - Bsize: statfs.Bsize, - Ffree: statfs.Ffree, - Files: statfs.Files, - Frsize: statfs.Frsize, - Namemax: 255, - } - - return &stat, true, nil + return c.NextComponent().StatFs() } diff --git a/component/s3storage/client.go b/component/s3storage/client.go index afb6f2860..62a080038 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -1126,3 +1126,85 @@ func (cl *Client) combineSmallBlocks(name string, blockList []*common.Block) ([] } return newBlockList, nil } + +func (cl *Client) GetUsedSize() uint64 { + var totalSize uint64 + var totalEntriesFetched int32 + var iteration int + var marker *string + var count int32 = maxResultsPerListCall + done := false + + path := "" + + for !done { + // prepare parameters + bucketName := cl.Config.authConfig.BucketName + if count == 0 { + count = maxResultsPerListCall + } + + // combine the configured prefix and the prefix being given to List to get a full listPath + listPath := cl.getKey(path, false) + // replace any trailing forward slash stripped by common.JoinUnixFilepath + if (path != "" && path[len(path)-1] == '/') || (path == "" && cl.Config.prefixPath != "") { + listPath += "/" + } + + var nextMarker *string + var token *string + + // using paginator from here: https://aws.github.io/aws-sdk-go-v2/docs/making-requests/#using-paginators + // List is a tricky function. Here is a great explanation of how list works: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html + + if marker != nil && *marker == "" { + token = nil + } else { + token = marker + } + params := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + MaxKeys: &count, + Prefix: aws.String(listPath), + ContinuationToken: token, + } + paginator := s3.NewListObjectsV2Paginator(cl.awsS3Client, params) + + // fetch and process a single result page + output, err := paginator.NextPage(context.Background()) + if err != nil { + log.Err("Client::List : Failed to list objects in bucket %v with path %v. Here's why: %v", listPath, bucketName, err) + return totalSize + } + + if output.IsTruncated != nil && *output.IsTruncated { + nextMarker = output.NextContinuationToken + } else { + nextMarker = nil + } + + totalEntriesFetched = *output.KeyCount + + // documentation for this S3 data structure: + // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3@v1.30.2#ListObjectsV2Output + for _, value := range output.Contents { + totalSize += uint64(*value.Size) + } + + marker = nextMarker + + log.Debug("Client::GetUsedSize : %s So far retrieved %d objects in %d iterations. Total size in bytes is %d", + listPath, totalEntriesFetched, iteration, totalSize) + if marker == nil || *marker == "" { + done = true + break + } else { + log.Debug("Client::GetUsedSize : %s List iteration %d nextMarker=\"%s\"", + listPath, iteration, *nextMarker) + } + iteration++ + } + + return totalSize +} diff --git a/component/s3storage/connection.go b/component/s3storage/connection.go index f5b36ee06..7e4499d0a 100644 --- a/component/s3storage/connection.go +++ b/component/s3storage/connection.go @@ -112,4 +112,5 @@ type S3Connection interface { StageAndCommit(name string, bol *common.BlockOffsetList) error NewCredentialKey(_, _ string) error + GetUsedSize() uint64 } diff --git a/component/s3storage/s3storage.go b/component/s3storage/s3storage.go index 34c2b78cf..857617be8 100644 --- a/component/s3storage/s3storage.go +++ b/component/s3storage/s3storage.go @@ -476,6 +476,31 @@ func (s3 *S3Storage) FlushFile(options internal.FlushFileOptions) error { return s3.storage.StageAndCommit(options.Handle.Path, options.Handle.CacheObj.BlockOffsetList) } +const blockSize = 4096 + +func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { + // cache_size = f_blocks * f_frsize/1024 + // cache_size - used = f_frsize * f_bavail/1024 + // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 + // if cache size is set to 0 then we have the root mount usage + sizeUsed := s3.storage.GetUsedSize() + + var total uint64 = 3 * common.TbToBytes + + stat := common.Statfs_t{ + Blocks: total / blockSize, + Bavail: (total - sizeUsed) / blockSize, + Bfree: (total - sizeUsed) / blockSize, + Bsize: blockSize, + Ffree: 1e9, + Files: 1e9, + Frsize: blockSize, + Namemax: 255, + } + + return &stat, true, nil +} + // TODO: decide if the TODO below is relevant and delete if not // TODO : Below methods are pending to be implemented // FlushFile(*handlemap.Handle) error From 68ac53f8ccce0bd524d98e7040f8b2bc7f2e1244 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 21 Oct 2024 09:17:29 -0600 Subject: [PATCH 02/18] Get HeadBucketOutput from HeadBucket call. --- component/s3storage/client.go | 7 ++++++- component/s3storage/s3wrappers.go | 21 +++------------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/component/s3storage/client.go b/component/s3storage/client.go index 314faad29..12dcb667a 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -203,7 +203,7 @@ func (cl *Client) Configure(cfg Config) error { } // Check that the provided bucket exists and that user has access to bucket - exists, err := cl.headBucket() + exists, err := cl.bucketExists() if err != nil || !exists { // From the aws-sdk-go-v2 documentation // If the bucket does not exist or you do not have permission to access it, @@ -277,6 +277,11 @@ func (cl *Client) SetPrefixPath(path string) error { return nil } +func (cl *Client) bucketExists() (bool, error) { + _, err := cl.headBucket() + return err != syscall.ENOENT, err +} + // CreateFile : Create a new file in the bucket/virtual directory func (cl *Client) CreateFile(name string, mode os.FileMode) error { log.Trace("Client::CreateFile : name %s", name) diff --git a/component/s3storage/s3wrappers.go b/component/s3storage/s3wrappers.go index 6ea4c4ab0..2cf5f0963 100644 --- a/component/s3storage/s3wrappers.go +++ b/component/s3storage/s3wrappers.go @@ -48,7 +48,6 @@ import ( "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" ) const symlinkStr = ".rclonelink" @@ -220,25 +219,11 @@ func (cl *Client) headObject(name string, isSymlink bool) (*internal.ObjAttr, er } // Wrapper for awsS3Client.HeadBucket -func (cl *Client) headBucket() (bool, error) { - _, err := cl.awsS3Client.HeadBucket(context.Background(), &s3.HeadBucketInput{ +func (cl *Client) headBucket() (*s3.HeadBucketOutput, error) { + headBucketOutput, err := cl.awsS3Client.HeadBucket(context.Background(), &s3.HeadBucketInput{ Bucket: aws.String(cl.Config.authConfig.BucketName), }) - exists := true - if err != nil { - var apiError smithy.APIError - if errors.As(err, &apiError) { - switch apiError.(type) { - case *types.NotFound: - log.Err("Client::headBucket : Bucket %s does not exist: ", err.Error()) - exists = false - default: - log.Err("Client::headBucket : Bucket %s exists but you do not have access to bucket or other error occurred : ", err.Error()) - } - } - } - - return exists, err + return headBucketOutput, parseS3Err(err, "HeadBucket "+cl.Config.authConfig.BucketName) } // Wrapper for awsS3Client.CopyObject From e07333b560e657b0b8290de89271f29436cce5e9 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 21 Oct 2024 12:49:08 -0600 Subject: [PATCH 03/18] Replace import lost on merge --- component/file_cache/file_cache_linux.go | 1 + 1 file changed, 1 insertion(+) diff --git a/component/file_cache/file_cache_linux.go b/component/file_cache/file_cache_linux.go index ecfa9d5ab..40beedd64 100644 --- a/component/file_cache/file_cache_linux.go +++ b/component/file_cache/file_cache_linux.go @@ -36,6 +36,7 @@ import ( "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" "github.com/Seagate/cloudfuse/internal" + "golang.org/x/sys/unix" ) // Creates a new object attribute From 3dfd985cf6c3d9f62e750a24b73813b774c66267 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 21 Oct 2024 14:53:55 -0600 Subject: [PATCH 04/18] Print the fields in the HeadBucket result --- component/s3storage/client.go | 76 ++--------------------------------- 1 file changed, 3 insertions(+), 73 deletions(-) diff --git a/component/s3storage/client.go b/component/s3storage/client.go index 9c1f29496..ac46b6d67 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -1130,81 +1130,11 @@ func (cl *Client) combineSmallBlocks(name string, blockList []*common.Block) ([] func (cl *Client) GetUsedSize() uint64 { var totalSize uint64 - var totalEntriesFetched int32 - var iteration int - var marker *string - var count int32 = maxResultsPerListCall - done := false - - path := "" - - for !done { - // prepare parameters - bucketName := cl.Config.authConfig.BucketName - if count == 0 { - count = maxResultsPerListCall - } - - // combine the configured prefix and the prefix being given to List to get a full listPath - listPath := cl.getKey(path, false) - // replace any trailing forward slash stripped by common.JoinUnixFilepath - if (path != "" && path[len(path)-1] == '/') || (path == "" && cl.Config.prefixPath != "") { - listPath += "/" - } - - var nextMarker *string - var token *string - - // using paginator from here: https://aws.github.io/aws-sdk-go-v2/docs/making-requests/#using-paginators - // List is a tricky function. Here is a great explanation of how list works: - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html - - if marker != nil && *marker == "" { - token = nil - } else { - token = marker - } - params := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucketName), - MaxKeys: &count, - Prefix: aws.String(listPath), - ContinuationToken: token, - } - paginator := s3.NewListObjectsV2Paginator(cl.awsS3Client, params) - // fetch and process a single result page - output, err := paginator.NextPage(context.Background()) - if err != nil { - log.Err("Client::List : Failed to list objects in bucket %v with path %v. Here's why: %v", listPath, bucketName, err) - return totalSize - } - - if output.IsTruncated != nil && *output.IsTruncated { - nextMarker = output.NextContinuationToken - } else { - nextMarker = nil - } - - totalEntriesFetched = *output.KeyCount - - // documentation for this S3 data structure: - // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3@v1.30.2#ListObjectsV2Output - for _, value := range output.Contents { - totalSize += uint64(*value.Size) - } - - marker = nextMarker + headBucketOutput, err := cl.headBucket() - log.Debug("Client::GetUsedSize : %s So far retrieved %d objects in %d iterations. Total size in bytes is %d", - listPath, totalEntriesFetched, iteration, totalSize) - if marker == nil || *marker == "" { - done = true - break - } else { - log.Debug("Client::GetUsedSize : %s List iteration %d nextMarker=\"%s\"", - listPath, iteration, *nextMarker) - } - iteration++ + if err != nil { + fmt.Println(headBucketOutput.ResultMetadata) } return totalSize From e2c41f942cf388388eb6ba625e3ac3598d1d9f89 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 21 Oct 2024 15:52:31 -0600 Subject: [PATCH 05/18] Pull custom header to get bucket size --- component/s3storage/client.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/component/s3storage/client.go b/component/s3storage/client.go index ac46b6d67..ae84e14c8 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -36,6 +36,7 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" "syscall" "time" @@ -47,12 +48,14 @@ import ( "github.com/Seagate/cloudfuse/internal/stats_manager" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/middleware" awsHttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" + smithyHttp "github.com/aws/smithy-go/transport/http" ) const ( @@ -1129,13 +1132,28 @@ func (cl *Client) combineSmallBlocks(name string, blockList []*common.Block) ([] } func (cl *Client) GetUsedSize() uint64 { - var totalSize uint64 - headBucketOutput, err := cl.headBucket() + if err != nil { + return 0 + } + + response, ok := middleware.GetRawResponse(headBucketOutput.ResultMetadata).(*smithyHttp.Response) + if !ok || response == nil { + return 0 + } + + headerValue, ok := response.Header["X-Rstor-Size"] + if !ok { + headerValue, ok = response.Header["X-Lyve-Size"] + } + if !ok || len(headerValue) == 0 { + return 0 + } + bucketSizeBytes, err := strconv.ParseUint(headerValue[0], 10, 64) if err != nil { - fmt.Println(headBucketOutput.ResultMetadata) + return 0 } - return totalSize + return bucketSizeBytes } From 25d04e5ceab51f7f2eefdaf295c89ba7002974b6 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Tue, 22 Oct 2024 16:17:35 -0600 Subject: [PATCH 06/18] s3storage: Statfs: Use total size to signal used size libfuse2: Always use configured display capacity --- component/file_cache/file_cache_linux.go | 8 -------- component/libfuse/libfuse2_handler.go | 18 +++++++----------- component/s3storage/s3storage.go | 8 +++----- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/component/file_cache/file_cache_linux.go b/component/file_cache/file_cache_linux.go index 40beedd64..5a0047814 100644 --- a/component/file_cache/file_cache_linux.go +++ b/component/file_cache/file_cache_linux.go @@ -142,14 +142,6 @@ func (fc *FileCache) isDownloadRequired(localPath string, blobPath string, flock return downloadRequired, fileExists, attr, err } -func (c *FileCache) StatFs() (*common.Statfs_t, bool, error) { - // cache_size = f_blocks * f_frsize/1024 - // cache_size - used = f_frsize * f_bavail/1024 - // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 - // if cache size is set to 0 then we have the root mount usage - return c.NextComponent().StatFs() -} - func (fc *FileCache) getAvailableSize() (uint64, error) { statfs := &unix.Statfs_t{} err := unix.Statfs(fc.tmpPath, statfs) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index ee130c63e..9a898e498 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -329,28 +329,24 @@ func (cf *CgofuseFS) Statfs(path string, stat *fuse.Statfs_t) int { return -fuse.EIO } + total := fuseFS.displayCapacityMb * common.MbToBytes // if populated then we need to overwrite root attributes if populated { stat.Bsize = uint64(attr.Bsize) stat.Frsize = uint64(attr.Frsize) - stat.Blocks = attr.Blocks - stat.Bavail = attr.Bavail - stat.Bfree = attr.Bfree + stat.Blocks = total / stat.Bsize + // attr.Blocks has the blocks used + stat.Bavail = stat.Blocks - attr.Blocks + stat.Bfree = stat.Blocks - attr.Blocks stat.Files = attr.Files stat.Ffree = attr.Ffree stat.Namemax = attr.Namemax } else { - var free, total, avail uint64 - // TODO: if display capacity is specified, should it overwrite populated Bavail? - total = fuseFS.displayCapacityMb * common.MbToBytes - avail = total - free = total - stat.Bsize = blockSize stat.Frsize = blockSize stat.Blocks = total / blockSize - stat.Bavail = avail / blockSize - stat.Bfree = free / blockSize + stat.Bavail = total / blockSize + stat.Bfree = total / blockSize stat.Files = 1e9 stat.Ffree = 1e9 stat.Namemax = maxNameSize diff --git a/component/s3storage/s3storage.go b/component/s3storage/s3storage.go index 857617be8..e3a3e2c9b 100644 --- a/component/s3storage/s3storage.go +++ b/component/s3storage/s3storage.go @@ -485,12 +485,10 @@ func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { // if cache size is set to 0 then we have the root mount usage sizeUsed := s3.storage.GetUsedSize() - var total uint64 = 3 * common.TbToBytes - stat := common.Statfs_t{ - Blocks: total / blockSize, - Bavail: (total - sizeUsed) / blockSize, - Bfree: (total - sizeUsed) / blockSize, + Blocks: sizeUsed / blockSize, + Bavail: sizeUsed / blockSize, + Bfree: sizeUsed / blockSize, Bsize: blockSize, Ffree: 1e9, Files: 1e9, From a9d82ab4f69658da9e4b21a946c4a03407b215e0 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 24 Oct 2024 13:29:28 -0600 Subject: [PATCH 07/18] Don't use fc for statfs --- component/file_cache/file_cache_windows.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/component/file_cache/file_cache_windows.go b/component/file_cache/file_cache_windows.go index 1f7b1c2fb..86ab6a519 100644 --- a/component/file_cache/file_cache_windows.go +++ b/component/file_cache/file_cache_windows.go @@ -145,6 +145,13 @@ func (fc *FileCache) isDownloadRequired(localPath string, blobPath string, flock } func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { + + // TODO: add flag for this + if true { + return fc.NextComponent().StatFs() + } + + // TODO: find and fix arithmetic error here // cache_size = f_blocks * f_frsize/1024 // cache_size - used = f_frsize * f_bavail/1024 // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 From e865cd00e988ba525491380679a1db38dd86db14 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 24 Oct 2024 13:29:33 -0600 Subject: [PATCH 08/18] Fix underflow --- component/libfuse/libfuse2_handler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index 9a898e498..f0a6cf994 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -334,8 +334,10 @@ func (cf *CgofuseFS) Statfs(path string, stat *fuse.Statfs_t) int { if populated { stat.Bsize = uint64(attr.Bsize) stat.Frsize = uint64(attr.Frsize) - stat.Blocks = total / stat.Bsize // attr.Blocks has the blocks used + // if used > display capacity, we still want to report useful data + // so just set the capacity to the amount used (and set free to 0) + stat.Blocks = max(total/stat.Bsize, attr.Blocks) stat.Bavail = stat.Blocks - attr.Blocks stat.Bfree = stat.Blocks - attr.Blocks stat.Files = attr.Files From 082f56a88af9f48eed50a80cf985c1aa833215c6 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 1 Nov 2024 17:49:06 -0600 Subject: [PATCH 09/18] Let Libfuse StatFs math work with both file cache and cloud data --- component/libfuse/libfuse2_handler.go | 29 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index f0a6cf994..a3cd3f27a 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -329,26 +329,35 @@ func (cf *CgofuseFS) Statfs(path string, stat *fuse.Statfs_t) int { return -fuse.EIO } - total := fuseFS.displayCapacityMb * common.MbToBytes // if populated then we need to overwrite root attributes if populated { stat.Bsize = uint64(attr.Bsize) stat.Frsize = uint64(attr.Frsize) - // attr.Blocks has the blocks used - // if used > display capacity, we still want to report useful data - // so just set the capacity to the amount used (and set free to 0) - stat.Blocks = max(total/stat.Bsize, attr.Blocks) - stat.Bavail = stat.Blocks - attr.Blocks - stat.Bfree = stat.Blocks - attr.Blocks + // cloud storage always sets free and avail to zero + statsFromCloudStorage := attr.Bfree == 0 && attr.Bavail == 0 + // calculate blocks used from attr + usedBlocks := attr.Blocks - attr.Bfree + // we only use displayCapacity to complement used size from cloud storage + if statsFromCloudStorage { + displayCapacityBlocks := fuseFS.displayCapacityMb * common.MbToBytes / uint64(attr.Bsize) + // if used > displayCapacity, then report used and show that we are out of space + stat.Blocks = max(displayCapacityBlocks, usedBlocks) + } else { + stat.Blocks = attr.Blocks + } + // adjust avail and free to make sure we display used space correctly + stat.Bavail = stat.Blocks - usedBlocks + stat.Bfree = stat.Blocks - usedBlocks stat.Files = attr.Files stat.Ffree = attr.Ffree stat.Namemax = attr.Namemax } else { stat.Bsize = blockSize stat.Frsize = blockSize - stat.Blocks = total / blockSize - stat.Bavail = total / blockSize - stat.Bfree = total / blockSize + displayCapacityBlocks := fuseFS.displayCapacityMb * common.MbToBytes / blockSize + stat.Blocks = displayCapacityBlocks + stat.Bavail = displayCapacityBlocks + stat.Bfree = displayCapacityBlocks stat.Files = 1e9 stat.Ffree = 1e9 stat.Namemax = maxNameSize From 8a70e2e91821132bb0be488d507369e582564f14 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 1 Nov 2024 21:34:47 -0600 Subject: [PATCH 10/18] Add new flag "usage-from-cache" --- component/file_cache/file_cache.go | 8 ++++++-- setup/baseConfig.yaml | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 95179eab3..8899d4f74 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -55,6 +55,7 @@ type FileCache struct { fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) policy cachePolicy + statFsFromCache bool createEmptyFile bool allowNonEmpty bool cacheTimeout float64 @@ -91,6 +92,7 @@ type FileCacheOptions struct { HighThreshold uint32 `config:"high-threshold" yaml:"high-threshold,omitempty"` LowThreshold uint32 `config:"low-threshold" yaml:"low-threshold,omitempty"` + StatFsFromCache bool `config:"usage-from-cache" yaml:"usage-from-cache,omitempty"` CreateEmptyFile bool `config:"create-empty-file" yaml:"create-empty-file,omitempty"` AllowNonEmpty bool `config:"allow-non-empty-temp" yaml:"allow-non-empty-temp,omitempty"` CleanupOnStart bool `config:"cleanup-on-start" yaml:"cleanup-on-start,omitempty"` @@ -218,6 +220,7 @@ func (c *FileCache) Configure(_ bool) error { return fmt.Errorf("config error in %s [%s]", c.Name(), err.Error()) } + c.statFsFromCache = conf.StatFsFromCache c.createEmptyFile = conf.CreateEmptyFile if config.IsSet(compName + ".file-cache-timeout-in-seconds") { c.cacheTimeout = float64(conf.V1Timeout) @@ -323,8 +326,8 @@ func (c *FileCache) Configure(_ bool) error { c.diskHighWaterMark = (((conf.MaxSizeMB * MB) * float64(cacheConfig.highThreshold)) / 100) } - log.Info("FileCache::Configure : create-empty %t, cache-timeout %d, tmp-path %s, max-size-mb %d, high-mark %d, low-mark %d, refresh-sec %v, max-eviction %v, hard-limit %v, policy %s, allow-non-empty-temp %t, cleanup-on-start %t, policy-trace %t, offload-io %t, sync-to-flush %t, ignore-sync %t, defaultPermission %v, diskHighWaterMark %v, maxCacheSize %v, mountPath %v", - c.createEmptyFile, int(c.cacheTimeout), c.tmpPath, int(cacheConfig.maxSizeMB), int(cacheConfig.highThreshold), int(cacheConfig.lowThreshold), c.refreshSec, cacheConfig.maxEviction, c.hardLimit, conf.Policy, c.allowNonEmpty, c.cleanupOnStart, c.policyTrace, c.offloadIO, c.syncToFlush, c.syncToDelete, c.defaultPermission, c.diskHighWaterMark, c.maxCacheSize, c.mountPath) + log.Info("FileCache::Configure : usage-from-cache %t, create-empty %t, cache-timeout %d, tmp-path %s, max-size-mb %d, high-mark %d, low-mark %d, refresh-sec %v, max-eviction %v, hard-limit %v, policy %s, allow-non-empty-temp %t, cleanup-on-start %t, policy-trace %t, offload-io %t, sync-to-flush %t, ignore-sync %t, defaultPermission %v, diskHighWaterMark %v, maxCacheSize %v, mountPath %v", + c.statFsFromCache, c.createEmptyFile, int(c.cacheTimeout), c.tmpPath, int(cacheConfig.maxSizeMB), int(cacheConfig.highThreshold), int(cacheConfig.lowThreshold), c.refreshSec, cacheConfig.maxEviction, c.hardLimit, conf.Policy, c.allowNonEmpty, c.cleanupOnStart, c.policyTrace, c.offloadIO, c.syncToFlush, c.syncToDelete, c.defaultPermission, c.diskHighWaterMark, c.maxCacheSize, c.mountPath) return nil } @@ -340,6 +343,7 @@ func (c *FileCache) OnConfigChange() { log.Err("FileCache: config error [invalid config attributes]") } + c.statFsFromCache = conf.StatFsFromCache c.createEmptyFile = conf.CreateEmptyFile c.cacheTimeout = float64(conf.Timeout) c.policyTrace = conf.EnablePolicyTrace diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index f9c2f4269..e664dac08 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -88,7 +88,7 @@ block_cache: disk-size-mb: disk-timeout-sec: prefetch: - parallelism: + parallelism: prefetch-on-open: true|false # Disk cache related configuration @@ -102,6 +102,7 @@ file_cache: max-size-mb: high-threshold: <% disk space consumed which triggers eviction. This parameter overrides 'timeout-sec' parameter and cached files will be removed even if they have not expired. Default - 80> low-threshold: <% disk space consumed which triggers eviction to stop when previously triggered by the high-threshold. Default - 60> + usage-from-cache: true|false create-empty-file: true|false allow-non-empty-temp: true|false cleanup-on-start: true|false From 9263e4a778b0fa002ad269bb6df319d50e662abb Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 1 Nov 2024 21:35:52 -0600 Subject: [PATCH 11/18] Use the same logic for StatFs for both platforms. Use new flag to choose which component provides StatFs info. --- component/file_cache/file_cache.go | 39 +++++++++++++++++ component/file_cache/file_cache_windows.go | 49 ---------------------- 2 files changed, 39 insertions(+), 49 deletions(-) diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 8899d4f74..e0dcd0f9e 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -380,6 +380,45 @@ func (c *FileCache) GetPolicyConfig(conf FileCacheOptions) cachePolicyConfig { return cacheConfig } +func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { + + if !fc.statFsFromCache { + return fc.NextComponent().StatFs() + } + + // cache_size = f_blocks * f_frsize/1024 + // cache_size - used = f_frsize * f_bavail/1024 + // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 + // if cache size is set to 0 then we have the root mount usage + maxCacheSize := fc.maxCacheSize * MB + if maxCacheSize == 0 { + return nil, false, nil + } + usage, _ := common.GetUsage(fc.tmpPath) + available := maxCacheSize - usage*MB + + // how much space is available on the underlying file system? + availableOnCacheFS, err := fc.getAvailableSize() + if err != nil { + return nil, false, err + } + + const blockSize = 4096 + + stat := common.Statfs_t{ + Blocks: uint64(maxCacheSize) / uint64(blockSize), + Bavail: uint64(max(0, available)) / uint64(blockSize), + Bfree: availableOnCacheFS / uint64(blockSize), + Bsize: blockSize, + Ffree: 1e9, + Files: 1e9, + Frsize: blockSize, + Namemax: 255, + } + + return &stat, true, nil +} + // isLocalDirEmpty: Whether or not the local directory is empty. func isLocalDirEmpty(path string) bool { f, _ := common.Open(path) diff --git a/component/file_cache/file_cache_windows.go b/component/file_cache/file_cache_windows.go index 86ab6a519..0ca9224f6 100644 --- a/component/file_cache/file_cache_windows.go +++ b/component/file_cache/file_cache_windows.go @@ -29,7 +29,6 @@ package file_cache import ( "io/fs" - "math" "os" "syscall" "time" @@ -144,54 +143,6 @@ func (fc *FileCache) isDownloadRequired(localPath string, blobPath string, flock return downloadRequired, fileExists, attr, err } -func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { - - // TODO: add flag for this - if true { - return fc.NextComponent().StatFs() - } - - // TODO: find and fix arithmetic error here - // cache_size = f_blocks * f_frsize/1024 - // cache_size - used = f_frsize * f_bavail/1024 - // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 - // if cache size is set to 0 then we have the root mount usage - maxCacheSize := fc.maxCacheSize * MB - if maxCacheSize == 0 { - return nil, false, nil - } - usage, _ := common.GetUsage(fc.tmpPath) - available := maxCacheSize - usage - - var free, total, avail uint64 - - // Get path to the cache - pathPtr, err := windows.UTF16PtrFromString(fc.tmpPath) - if err != nil { - return nil, false, err - } - err = windows.GetDiskFreeSpaceEx(pathPtr, &free, &total, &avail) - if err != nil { - log.Debug("FileCache::StatFs : statfs err [%s].", err.Error()) - return nil, false, err - } - - const blockSize = 4096 - - stat := common.Statfs_t{ - Blocks: uint64(maxCacheSize) / uint64(blockSize), - Bavail: uint64(math.Max(0, available)) / uint64(blockSize), - Bfree: free / uint64(blockSize), - Bsize: blockSize, - Ffree: 1e9, - Files: 1e9, - Frsize: blockSize, - Namemax: 255, - } - - return &stat, true, nil -} - func (fc *FileCache) getAvailableSize() (uint64, error) { var free, total, avail uint64 From c173e53d56b17e9cd63945702c11b14c14c61773 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 1 Nov 2024 22:03:25 -0600 Subject: [PATCH 12/18] Update tests --- component/file_cache/file_cache_test.go | 26 +++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index 761896049..d9618960e 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -181,12 +181,13 @@ func (suite *fileCacheTestSuite) TestConfig() { maxDeletion := 10 highThreshold := 90 lowThreshold := 10 + usageFromCache := true createEmptyFile := true allowNonEmptyTemp := true cleanupOnStart := true syncToFlush := false - config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t\n sync-to-flush: %t", - suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, createEmptyFile, allowNonEmptyTemp, cleanupOnStart, syncToFlush) + config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n usage-from-cache: %t\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t\n sync-to-flush: %t", + suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, usageFromCache, createEmptyFile, allowNonEmptyTemp, cleanupOnStart, syncToFlush) suite.setupTestHelper(config) // setup a new file cache with a custom config (teardown will occur after the test as usual) suite.assert.Equal("file_cache", suite.fileCache.Name()) @@ -198,6 +199,7 @@ func (suite *fileCacheTestSuite) TestConfig() { suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).highThreshold, highThreshold) suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold) + suite.assert.Equal(suite.fileCache.statFsFromCache, usageFromCache) suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile) suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp) suite.assert.EqualValues(suite.fileCache.cacheTimeout, cacheTimeout) @@ -251,11 +253,12 @@ func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() { maxDeletion := 10 highThreshold := 90 lowThreshold := 10 + usageFromCache := true createEmptyFile := true allowNonEmptyTemp := true cleanupOnStart := true - config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t", - suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, createEmptyFile, allowNonEmptyTemp, cleanupOnStart) + config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n usage-from-cache: %t\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t", + suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, usageFromCache, createEmptyFile, allowNonEmptyTemp, cleanupOnStart) suite.setupTestHelper(config) // setup a new file cache with a custom config (teardown will occur after the test as usual) suite.assert.Equal("file_cache", suite.fileCache.Name()) @@ -268,6 +271,7 @@ func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() { suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold) suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).cacheTimeout, cacheTimeout) + suite.assert.Equal(suite.fileCache.statFsFromCache, usageFromCache) suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile) suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp) suite.assert.EqualValues(suite.fileCache.cacheTimeout, cacheTimeout) @@ -1581,7 +1585,7 @@ func (suite *fileCacheTestSuite) TestStatFS() { defer suite.cleanupTest() cacheTimeout := 5 maxSizeMb := 2 - config := fmt.Sprintf("file_cache:\n path: %s\n max-size-mb: %d\n offload-io: true\n timeout-sec: %d\n\nloopbackfs:\n path: %s", + config := fmt.Sprintf("file_cache:\n path: %s\n max-size-mb: %d\n usage-from-cache: true\n offload-io: true\n timeout-sec: %d\n\nloopbackfs:\n path: %s", suite.cache_path, maxSizeMb, cacheTimeout, suite.fake_storage_path) os.Mkdir(suite.cache_path, 0777) suite.setupTestHelper(config) // setup a new file cache with a custom config (teardown will occur after the test as usual) @@ -1594,7 +1598,7 @@ func (suite *fileCacheTestSuite) TestStatFS() { stat, ret, err := suite.fileCache.StatFs() suite.assert.True(ret) suite.assert.NoError(err) - suite.assert.NotEqual(&common.Statfs_t{}, stat) + suite.assert.NotNil(&common.Statfs_t{}, stat) // Added additional checks for StatFS suite.assert.Equal(int64(4096), stat.Bsize) @@ -1603,6 +1607,16 @@ func (suite *fileCacheTestSuite) TestStatFS() { suite.assert.Equal(uint64(255), stat.Namemax) } +func (suite *fileCacheTestSuite) TestStatFSCloud() { + defer suite.cleanupTest() + + stat, ret, err := suite.fileCache.StatFs() + // make sure it's not returning its own data + suite.assert.False(ret) + suite.assert.Nil(stat) + suite.assert.NoError(err) +} + func (suite *fileCacheTestSuite) TestReadFileWithRefresh() { defer suite.cleanupTest() // Configure to create empty files so we create the file in cloud storage From b7374b3492648024897286d786ca6ffc26b7e5f6 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 1 Nov 2024 22:22:53 -0600 Subject: [PATCH 13/18] Scrub and update Cobra documentation --- cmd/mount.go | 4 ++-- cmd/root.go | 4 ++-- cmd/version.go | 2 +- doc/cloudfuse.md | 17 ++++++++--------- doc/cloudfuse_completion.md | 4 ++-- doc/cloudfuse_mount.md | 14 +++++++------- doc/cloudfuse_mount_all.md | 6 +++--- doc/cloudfuse_mount_list.md | 4 ++-- doc/cloudfuse_secure.md | 4 ++-- doc/cloudfuse_unmount.md | 4 ++-- doc/cloudfuse_version.md | 8 ++++---- 11 files changed, 35 insertions(+), 36 deletions(-) diff --git a/cmd/mount.go b/cmd/mount.go index 5dfe9619f..6ec800c2d 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -264,8 +264,8 @@ func parseConfig() error { // Look at https://cobra.dev/ for more information var mountCmd = &cobra.Command{ Use: "mount ", - Short: "Mounts the container as a filesystem", - Long: "Mounts the container as a filesystem", + Short: "Mount the container as a filesystem", + Long: "Mount the container as a filesystem", SuggestFor: []string{"mnt", "mout"}, Args: cobra.ExactArgs(1), FlagErrorHandling: cobra.ExitOnError, diff --git a/cmd/root.go b/cmd/root.go index fc96c2ec7..382d943bb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -56,8 +56,8 @@ var disableVersionCheck bool var rootCmd = &cobra.Command{ Use: "cloudfuse", - Short: "Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage.", - Long: "Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. It uses the fuse protocol to communicate with the Linux FUSE kernel module, and implements the filesystem operations using the Azure Storage REST APIs.", + Short: "Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage.", + Long: "Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. It uses the FUSE protocol to communicate with the operating system, and implements filesystem operations using Azure or S3 cloud storage REST APIs.", Version: common.CloudfuseVersion, FlagErrorHandling: cobra.ExitOnError, SilenceUsage: true, diff --git a/cmd/version.go b/cmd/version.go index 80dc502f9..8adea5738 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -37,7 +37,7 @@ var check bool var versionCmd = &cobra.Command{ Use: "version", - Short: "Command to print the current version along with optional check for latest version", + Short: "Print the current version and optionally check for latest version", FlagErrorHandling: cobra.ExitOnError, RunE: func(cmd *cobra.Command, args []string) error { fmt.Println("cloudfuse version:", common.CloudfuseVersion) diff --git a/doc/cloudfuse.md b/doc/cloudfuse.md index 8dfe5d9ac..b5d53b22c 100644 --- a/doc/cloudfuse.md +++ b/doc/cloudfuse.md @@ -1,10 +1,10 @@ ## cloudfuse -Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. +Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. ### Synopsis -Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. It uses the fuse protocol to communicate with the Linux FUSE kernel module, and implements the filesystem operations using the Azure Storage REST APIs. +Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. It uses the FUSE protocol to communicate with the operating system, and implements filesystem operations using Azure or S3 cloud storage REST APIs. ``` cloudfuse [flags] @@ -19,11 +19,10 @@ cloudfuse [flags] ### SEE ALSO -* [cloudfuse completion](cloudfuse_completion.md) - Generate the autocompletion script for the specified shell -* [cloudfuse mount](cloudfuse_mount.md) - Mounts the container as a filesystem -* [cloudfuse secure](cloudfuse_secure.md) - Encrypt / Decrypt your config file -* [cloudfuse service](cloudfuse_service.md) - Manage cloudfuse startup process on Windows -* [cloudfuse unmount](cloudfuse_unmount.md) - Unmount container -* [cloudfuse version](cloudfuse_version.md) - Command to print the current version along with optional check for latest version +* [cloudfuse completion](cloudfuse_completion.md) - Generate the autocompletion script for the specified shell +* [cloudfuse mount](cloudfuse_mount.md) - Mount the container as a filesystem +* [cloudfuse secure](cloudfuse_secure.md) - Encrypt / Decrypt your config file +* [cloudfuse unmount](cloudfuse_unmount.md) - Unmount container +* [cloudfuse version](cloudfuse_version.md) - Print the current version and optionally check for latest version -###### Auto generated by spf13/cobra on 11-Jun-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_completion.md b/doc/cloudfuse_completion.md index 679533e9a..a5cc04502 100644 --- a/doc/cloudfuse_completion.md +++ b/doc/cloudfuse_completion.md @@ -22,10 +22,10 @@ See each sub-command's help for details on how to use the generated script. ### SEE ALSO -* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. +* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. * [cloudfuse completion bash](cloudfuse_completion_bash.md) - Generate the autocompletion script for bash * [cloudfuse completion fish](cloudfuse_completion_fish.md) - Generate the autocompletion script for fish * [cloudfuse completion powershell](cloudfuse_completion_powershell.md) - Generate the autocompletion script for powershell * [cloudfuse completion zsh](cloudfuse_completion_zsh.md) - Generate the autocompletion script for zsh -###### Auto generated by spf13/cobra on 15-Sep-2022 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_mount.md b/doc/cloudfuse_mount.md index fe11eb1c7..617072878 100644 --- a/doc/cloudfuse_mount.md +++ b/doc/cloudfuse_mount.md @@ -1,13 +1,13 @@ ## cloudfuse mount -Mounts the container as a filesystem +Mount the container as a filesystem ### Synopsis -Mounts the container as a filesystem +Mount the container as a filesystem ``` -cloudfuse mount [path] [flags] +cloudfuse mount [flags] ``` ### Options @@ -70,8 +70,8 @@ cloudfuse mount [path] [flags] ### SEE ALSO -* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. -* [cloudfuse mount all](cloudfuse_mount_all.md) - Mounts all containers for a given cloud account as a filesystem -* [cloudfuse mount list](cloudfuse_mount_list.md) - List all cloudfuse mountpoints +* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. +* [cloudfuse mount all](cloudfuse_mount_all.md) - Mounts all containers for a given cloud account as a filesystem +* [cloudfuse mount list](cloudfuse_mount_list.md) - List all cloudfuse mountpoints -###### Auto generated by spf13/cobra on 23-Sep-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_mount_all.md b/doc/cloudfuse_mount_all.md index bb171198c..91cc649c0 100644 --- a/doc/cloudfuse_mount_all.md +++ b/doc/cloudfuse_mount_all.md @@ -7,7 +7,7 @@ Mounts all containers for a given cloud account as a filesystem Mounts all containers for a given cloud account as a filesystem ``` -cloudfuse mount all [path] [flags] +cloudfuse mount all [flags] ``` ### Options @@ -69,6 +69,6 @@ cloudfuse mount all [path] [flags] ### SEE ALSO -* [cloudfuse mount](cloudfuse_mount.md) - Mounts the container as a filesystem +* [cloudfuse mount](cloudfuse_mount.md) - Mount the container as a filesystem -###### Auto generated by spf13/cobra on 23-Sep-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_mount_list.md b/doc/cloudfuse_mount_list.md index b00e6805d..0d45070cc 100644 --- a/doc/cloudfuse_mount_list.md +++ b/doc/cloudfuse_mount_list.md @@ -75,6 +75,6 @@ cloudfuse mount list ### SEE ALSO -* [cloudfuse mount](cloudfuse_mount.md) - Mounts the container as a filesystem +* [cloudfuse mount](cloudfuse_mount.md) - Mount the container as a filesystem -###### Auto generated by spf13/cobra on 23-Sep-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_secure.md b/doc/cloudfuse_secure.md index dc92b591d..342b399ee 100644 --- a/doc/cloudfuse_secure.md +++ b/doc/cloudfuse_secure.md @@ -34,10 +34,10 @@ cloudfuse secure encrypt --config-file=config.yaml --passphrase=PASSPHRASE ### SEE ALSO -* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. +* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. * [cloudfuse secure decrypt](cloudfuse_secure_decrypt.md) - Decrypt your config file * [cloudfuse secure encrypt](cloudfuse_secure_encrypt.md) - Encrypt your config file * [cloudfuse secure get](cloudfuse_secure_get.md) - Get value of requested config parameter from your encrypted config file * [cloudfuse secure set](cloudfuse_secure_set.md) - Update encrypted config by setting new value for the given config parameter -###### Auto generated by spf13/cobra on 11-Jun-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_unmount.md b/doc/cloudfuse_unmount.md index 002229e1a..db2914b7d 100644 --- a/doc/cloudfuse_unmount.md +++ b/doc/cloudfuse_unmount.md @@ -25,7 +25,7 @@ cloudfuse unmount [flags] ### SEE ALSO -* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. +* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. * [cloudfuse unmount all](cloudfuse_unmount_all.md) - Unmount all instances of Cloudfuse -###### Auto generated by spf13/cobra on 11-Jun-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 diff --git a/doc/cloudfuse_version.md b/doc/cloudfuse_version.md index 38688d32a..390efb532 100644 --- a/doc/cloudfuse_version.md +++ b/doc/cloudfuse_version.md @@ -1,9 +1,9 @@ ## cloudfuse version -Command to print the current version along with optional check for latest version +Print the current version and optionally check for latest version ``` -cloudfuse version [--check] [flags] +cloudfuse version [flags] ``` ### Options @@ -21,6 +21,6 @@ cloudfuse version [--check] [flags] ### SEE ALSO -* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by the Azure Storage. +* [cloudfuse](cloudfuse.md) - Cloudfuse is an open source project developed to provide a virtual filesystem backed by cloud storage. -###### Auto generated by spf13/cobra on 11-Jun-2024 +###### Auto generated by spf13/cobra on 1-Nov-2024 From 951fd02a86f5e07adae7f32f1dc2b74300ff2aa4 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 1 Nov 2024 23:15:41 -0600 Subject: [PATCH 14/18] Add more error handling and logging. Bugfix: return zero for free and avail so libfuse knows cloud storage is responding. --- component/file_cache/file_cache.go | 5 +++++ component/s3storage/client.go | 12 ++++++------ component/s3storage/connection.go | 2 +- component/s3storage/s3storage.go | 18 ++++++++++++++---- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index e0dcd0f9e..7cf26d1a8 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -386,12 +386,15 @@ func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { return fc.NextComponent().StatFs() } + log.Trace("FileCache::StatFs") + // cache_size = f_blocks * f_frsize/1024 // cache_size - used = f_frsize * f_bavail/1024 // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 // if cache size is set to 0 then we have the root mount usage maxCacheSize := fc.maxCacheSize * MB if maxCacheSize == 0 { + log.Err("FileCache::StatFs : Not responding to StatFs because max cache size is zero") return nil, false, nil } usage, _ := common.GetUsage(fc.tmpPath) @@ -400,6 +403,7 @@ func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { // how much space is available on the underlying file system? availableOnCacheFS, err := fc.getAvailableSize() if err != nil { + log.Err("FileCache::StatFs : Not responding to StatFs because getAvailableSize failed. Here's why: %v", err) return nil, false, err } @@ -416,6 +420,7 @@ func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { Namemax: 255, } + log.Debug("FileCache::StatFs : responding with free=%d avail=%d blocks=%d (bsize=%d)", stat.Bfree, stat.Bavail, stat.Blocks, stat.Bsize) return &stat, true, nil } diff --git a/component/s3storage/client.go b/component/s3storage/client.go index ae84e14c8..e28bca13f 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -1131,15 +1131,15 @@ func (cl *Client) combineSmallBlocks(name string, blockList []*common.Block) ([] return newBlockList, nil } -func (cl *Client) GetUsedSize() uint64 { +func (cl *Client) GetUsedSize() (uint64, error) { headBucketOutput, err := cl.headBucket() if err != nil { - return 0 + return 0, err } response, ok := middleware.GetRawResponse(headBucketOutput.ResultMetadata).(*smithyHttp.Response) if !ok || response == nil { - return 0 + return 0, fmt.Errorf("Failed GetRawResponse from HeadBucketOutput") } headerValue, ok := response.Header["X-Rstor-Size"] @@ -1147,13 +1147,13 @@ func (cl *Client) GetUsedSize() uint64 { headerValue, ok = response.Header["X-Lyve-Size"] } if !ok || len(headerValue) == 0 { - return 0 + return 0, fmt.Errorf("HeadBucket response has no size header (is the endpoint not Lyve Cloud?)") } bucketSizeBytes, err := strconv.ParseUint(headerValue[0], 10, 64) if err != nil { - return 0 + return 0, err } - return bucketSizeBytes + return bucketSizeBytes, nil } diff --git a/component/s3storage/connection.go b/component/s3storage/connection.go index 7e4499d0a..e579f7ec3 100644 --- a/component/s3storage/connection.go +++ b/component/s3storage/connection.go @@ -112,5 +112,5 @@ type S3Connection interface { StageAndCommit(name string, bol *common.BlockOffsetList) error NewCredentialKey(_, _ string) error - GetUsedSize() uint64 + GetUsedSize() (uint64, error) } diff --git a/component/s3storage/s3storage.go b/component/s3storage/s3storage.go index e3a3e2c9b..8eb91fdc9 100644 --- a/component/s3storage/s3storage.go +++ b/component/s3storage/s3storage.go @@ -479,16 +479,24 @@ func (s3 *S3Storage) FlushFile(options internal.FlushFileOptions) error { const blockSize = 4096 func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { + log.Trace("S3Storage::StatFs") // cache_size = f_blocks * f_frsize/1024 // cache_size - used = f_frsize * f_bavail/1024 // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 // if cache size is set to 0 then we have the root mount usage - sizeUsed := s3.storage.GetUsedSize() + sizeUsed, err := s3.storage.GetUsedSize() + if err != nil { + // TODO: will returning EIO break any applications that depend on StatFs? + return nil, false, err + } stat := common.Statfs_t{ - Blocks: sizeUsed / blockSize, - Bavail: sizeUsed / blockSize, - Bfree: sizeUsed / blockSize, + Blocks: sizeUsed / blockSize, + // there is no set capacity limit in cloud storage + // so we use zero for free and avail + // this zero value is used in the libfuse component to recognize that cloud storage responded + Bavail: 0, + Bfree: 0, Bsize: blockSize, Ffree: 1e9, Files: 1e9, @@ -496,6 +504,8 @@ func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { Namemax: 255, } + log.Debug("S3Storage::StatFs : responding with free=%d avail=%d blocks=%d (bsize=%d)", stat.Bfree, stat.Bavail, stat.Blocks, stat.Bsize) + return &stat, true, nil } From 46a361499be3a59bc41e31389140075d9d969552 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Tue, 5 Nov 2024 10:17:03 -0700 Subject: [PATCH 15/18] Use extra math to pass test and allow avail and free to be different --- component/libfuse/libfuse2_handler.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index a3cd3f27a..3b936edda 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -336,18 +336,19 @@ func (cf *CgofuseFS) Statfs(path string, stat *fuse.Statfs_t) int { // cloud storage always sets free and avail to zero statsFromCloudStorage := attr.Bfree == 0 && attr.Bavail == 0 // calculate blocks used from attr - usedBlocks := attr.Blocks - attr.Bfree + blocksUnavailable := attr.Blocks - attr.Bavail + blocksUsed := attr.Blocks - attr.Bfree // we only use displayCapacity to complement used size from cloud storage if statsFromCloudStorage { displayCapacityBlocks := fuseFS.displayCapacityMb * common.MbToBytes / uint64(attr.Bsize) // if used > displayCapacity, then report used and show that we are out of space - stat.Blocks = max(displayCapacityBlocks, usedBlocks) + stat.Blocks = max(displayCapacityBlocks, blocksUnavailable) } else { stat.Blocks = attr.Blocks } // adjust avail and free to make sure we display used space correctly - stat.Bavail = stat.Blocks - usedBlocks - stat.Bfree = stat.Blocks - usedBlocks + stat.Bavail = stat.Blocks - blocksUnavailable + stat.Bfree = stat.Blocks - blocksUsed stat.Files = attr.Files stat.Ffree = attr.Ffree stat.Namemax = attr.Namemax From db2c3611a7f106f9eb34fb860a57e20f5c08d10c Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Tue, 5 Nov 2024 13:24:10 -0700 Subject: [PATCH 16/18] Remove flag from file cache --- component/file_cache/file_cache.go | 13 +++++-------- component/file_cache/file_cache_test.go | 26 ++++++------------------- setup/baseConfig.yaml | 1 - 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 7cf26d1a8..0a7ca7c6b 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -55,7 +55,6 @@ type FileCache struct { fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) policy cachePolicy - statFsFromCache bool createEmptyFile bool allowNonEmpty bool cacheTimeout float64 @@ -92,7 +91,6 @@ type FileCacheOptions struct { HighThreshold uint32 `config:"high-threshold" yaml:"high-threshold,omitempty"` LowThreshold uint32 `config:"low-threshold" yaml:"low-threshold,omitempty"` - StatFsFromCache bool `config:"usage-from-cache" yaml:"usage-from-cache,omitempty"` CreateEmptyFile bool `config:"create-empty-file" yaml:"create-empty-file,omitempty"` AllowNonEmpty bool `config:"allow-non-empty-temp" yaml:"allow-non-empty-temp,omitempty"` CleanupOnStart bool `config:"cleanup-on-start" yaml:"cleanup-on-start,omitempty"` @@ -220,7 +218,6 @@ func (c *FileCache) Configure(_ bool) error { return fmt.Errorf("config error in %s [%s]", c.Name(), err.Error()) } - c.statFsFromCache = conf.StatFsFromCache c.createEmptyFile = conf.CreateEmptyFile if config.IsSet(compName + ".file-cache-timeout-in-seconds") { c.cacheTimeout = float64(conf.V1Timeout) @@ -326,8 +323,8 @@ func (c *FileCache) Configure(_ bool) error { c.diskHighWaterMark = (((conf.MaxSizeMB * MB) * float64(cacheConfig.highThreshold)) / 100) } - log.Info("FileCache::Configure : usage-from-cache %t, create-empty %t, cache-timeout %d, tmp-path %s, max-size-mb %d, high-mark %d, low-mark %d, refresh-sec %v, max-eviction %v, hard-limit %v, policy %s, allow-non-empty-temp %t, cleanup-on-start %t, policy-trace %t, offload-io %t, sync-to-flush %t, ignore-sync %t, defaultPermission %v, diskHighWaterMark %v, maxCacheSize %v, mountPath %v", - c.statFsFromCache, c.createEmptyFile, int(c.cacheTimeout), c.tmpPath, int(cacheConfig.maxSizeMB), int(cacheConfig.highThreshold), int(cacheConfig.lowThreshold), c.refreshSec, cacheConfig.maxEviction, c.hardLimit, conf.Policy, c.allowNonEmpty, c.cleanupOnStart, c.policyTrace, c.offloadIO, c.syncToFlush, c.syncToDelete, c.defaultPermission, c.diskHighWaterMark, c.maxCacheSize, c.mountPath) + log.Info("FileCache::Configure : create-empty %t, cache-timeout %d, tmp-path %s, max-size-mb %d, high-mark %d, low-mark %d, refresh-sec %v, max-eviction %v, hard-limit %v, policy %s, allow-non-empty-temp %t, cleanup-on-start %t, policy-trace %t, offload-io %t, sync-to-flush %t, ignore-sync %t, defaultPermission %v, diskHighWaterMark %v, maxCacheSize %v, mountPath %v", + c.createEmptyFile, int(c.cacheTimeout), c.tmpPath, int(cacheConfig.maxSizeMB), int(cacheConfig.highThreshold), int(cacheConfig.lowThreshold), c.refreshSec, cacheConfig.maxEviction, c.hardLimit, conf.Policy, c.allowNonEmpty, c.cleanupOnStart, c.policyTrace, c.offloadIO, c.syncToFlush, c.syncToDelete, c.defaultPermission, c.diskHighWaterMark, c.maxCacheSize, c.mountPath) return nil } @@ -343,7 +340,6 @@ func (c *FileCache) OnConfigChange() { log.Err("FileCache: config error [invalid config attributes]") } - c.statFsFromCache = conf.StatFsFromCache c.createEmptyFile = conf.CreateEmptyFile c.cacheTimeout = float64(conf.Timeout) c.policyTrace = conf.EnablePolicyTrace @@ -382,8 +378,9 @@ func (c *FileCache) GetPolicyConfig(conf FileCacheOptions) cachePolicyConfig { func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { - if !fc.statFsFromCache { - return fc.NextComponent().StatFs() + statfs, populated, err := fc.NextComponent().StatFs() + if populated { + return statfs, populated, err } log.Trace("FileCache::StatFs") diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index d9618960e..761896049 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -181,13 +181,12 @@ func (suite *fileCacheTestSuite) TestConfig() { maxDeletion := 10 highThreshold := 90 lowThreshold := 10 - usageFromCache := true createEmptyFile := true allowNonEmptyTemp := true cleanupOnStart := true syncToFlush := false - config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n usage-from-cache: %t\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t\n sync-to-flush: %t", - suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, usageFromCache, createEmptyFile, allowNonEmptyTemp, cleanupOnStart, syncToFlush) + config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t\n sync-to-flush: %t", + suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, createEmptyFile, allowNonEmptyTemp, cleanupOnStart, syncToFlush) suite.setupTestHelper(config) // setup a new file cache with a custom config (teardown will occur after the test as usual) suite.assert.Equal("file_cache", suite.fileCache.Name()) @@ -199,7 +198,6 @@ func (suite *fileCacheTestSuite) TestConfig() { suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).highThreshold, highThreshold) suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold) - suite.assert.Equal(suite.fileCache.statFsFromCache, usageFromCache) suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile) suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp) suite.assert.EqualValues(suite.fileCache.cacheTimeout, cacheTimeout) @@ -253,12 +251,11 @@ func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() { maxDeletion := 10 highThreshold := 90 lowThreshold := 10 - usageFromCache := true createEmptyFile := true allowNonEmptyTemp := true cleanupOnStart := true - config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n usage-from-cache: %t\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t", - suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, usageFromCache, createEmptyFile, allowNonEmptyTemp, cleanupOnStart) + config := fmt.Sprintf("file_cache:\n path: %s\n offload-io: true\n policy: %s\n max-size-mb: %d\n timeout-sec: %d\n max-eviction: %d\n high-threshold: %d\n low-threshold: %d\n create-empty-file: %t\n allow-non-empty-temp: %t\n cleanup-on-start: %t", + suite.cache_path, policy, maxSizeMb, cacheTimeout, maxDeletion, highThreshold, lowThreshold, createEmptyFile, allowNonEmptyTemp, cleanupOnStart) suite.setupTestHelper(config) // setup a new file cache with a custom config (teardown will occur after the test as usual) suite.assert.Equal("file_cache", suite.fileCache.Name()) @@ -271,7 +268,6 @@ func (suite *fileCacheTestSuite) TestConfigPolicyTimeout() { suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).lowThreshold, lowThreshold) suite.assert.EqualValues(suite.fileCache.policy.(*lruPolicy).cacheTimeout, cacheTimeout) - suite.assert.Equal(suite.fileCache.statFsFromCache, usageFromCache) suite.assert.Equal(suite.fileCache.createEmptyFile, createEmptyFile) suite.assert.Equal(suite.fileCache.allowNonEmpty, allowNonEmptyTemp) suite.assert.EqualValues(suite.fileCache.cacheTimeout, cacheTimeout) @@ -1585,7 +1581,7 @@ func (suite *fileCacheTestSuite) TestStatFS() { defer suite.cleanupTest() cacheTimeout := 5 maxSizeMb := 2 - config := fmt.Sprintf("file_cache:\n path: %s\n max-size-mb: %d\n usage-from-cache: true\n offload-io: true\n timeout-sec: %d\n\nloopbackfs:\n path: %s", + config := fmt.Sprintf("file_cache:\n path: %s\n max-size-mb: %d\n offload-io: true\n timeout-sec: %d\n\nloopbackfs:\n path: %s", suite.cache_path, maxSizeMb, cacheTimeout, suite.fake_storage_path) os.Mkdir(suite.cache_path, 0777) suite.setupTestHelper(config) // setup a new file cache with a custom config (teardown will occur after the test as usual) @@ -1598,7 +1594,7 @@ func (suite *fileCacheTestSuite) TestStatFS() { stat, ret, err := suite.fileCache.StatFs() suite.assert.True(ret) suite.assert.NoError(err) - suite.assert.NotNil(&common.Statfs_t{}, stat) + suite.assert.NotEqual(&common.Statfs_t{}, stat) // Added additional checks for StatFS suite.assert.Equal(int64(4096), stat.Bsize) @@ -1607,16 +1603,6 @@ func (suite *fileCacheTestSuite) TestStatFS() { suite.assert.Equal(uint64(255), stat.Namemax) } -func (suite *fileCacheTestSuite) TestStatFSCloud() { - defer suite.cleanupTest() - - stat, ret, err := suite.fileCache.StatFs() - // make sure it's not returning its own data - suite.assert.False(ret) - suite.assert.Nil(stat) - suite.assert.NoError(err) -} - func (suite *fileCacheTestSuite) TestReadFileWithRefresh() { defer suite.cleanupTest() // Configure to create empty files so we create the file in cloud storage diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index e664dac08..fd971dc10 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -102,7 +102,6 @@ file_cache: max-size-mb: high-threshold: <% disk space consumed which triggers eviction. This parameter overrides 'timeout-sec' parameter and cached files will be removed even if they have not expired. Default - 80> low-threshold: <% disk space consumed which triggers eviction to stop when previously triggered by the high-threshold. Default - 60> - usage-from-cache: true|false create-empty-file: true|false allow-non-empty-temp: true|false cleanup-on-start: true|false From 9faa81e03eda09ce5bbea8cc6b273f649ccc6313 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Tue, 5 Nov 2024 13:25:40 -0700 Subject: [PATCH 17/18] Add flag to s3storage --- component/s3storage/client_test.go | 1 + component/s3storage/config.go | 2 ++ component/s3storage/connection.go | 1 + component/s3storage/s3storage.go | 6 +++++- component/s3storage/s3storage_test.go | 5 +++-- setup/baseConfig.yaml | 1 + 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/component/s3storage/client_test.go b/component/s3storage/client_test.go index 8229c9ffd..93348edc1 100644 --- a/component/s3storage/client_test.go +++ b/component/s3storage/client_test.go @@ -88,6 +88,7 @@ func newTestClient(configuration string) (*Client, error) { partSize: conf.PartSizeMb * common.MbToBytes, uploadCutoff: conf.UploadCutoffMb * common.MbToBytes, usePathStyle: conf.UsePathStyle, + disableUsage: conf.DisableUsage, } // create a Client client, err := NewConnection(configForS3Client) diff --git a/component/s3storage/config.go b/component/s3storage/config.go index 9e0adb1e0..df11c7e6a 100644 --- a/component/s3storage/config.go +++ b/component/s3storage/config.go @@ -54,6 +54,7 @@ type Options struct { EnableChecksum bool `config:"enable-checksum" yaml:"enable-checksum,omitempty"` ChecksumAlgorithm types.ChecksumAlgorithm `config:"checksum-algorithm" yaml:"checksum-algorithm,omitempty"` UsePathStyle bool `config:"use-path-style" yaml:"use-path-style,omitempty"` + DisableUsage bool `config:"disable-usage" yaml:"disable-usage,omitempty"` } // ParseAndValidateConfig : Parse and validate config @@ -77,6 +78,7 @@ func ParseAndValidateConfig(s3 *S3Storage, opt Options) error { s3.stConfig.restrictedCharsWin = opt.RestrictedCharsWin s3.stConfig.disableConcurrentDownload = opt.DisableConcurrentDownload s3.stConfig.usePathStyle = opt.UsePathStyle + s3.stConfig.disableUsage = opt.DisableUsage // Part size must be at least 5 MB and smaller than 5GB. Otherwise, set to default. if opt.PartSizeMb < 5 || opt.PartSizeMb > MaxPartSizeMb { diff --git a/component/s3storage/connection.go b/component/s3storage/connection.go index e579f7ec3..5413c5e20 100644 --- a/component/s3storage/connection.go +++ b/component/s3storage/connection.go @@ -52,6 +52,7 @@ type Config struct { checksumAlgorithm types.ChecksumAlgorithm usePathStyle bool disableSymlink bool + disableUsage bool } // TODO: move s3AuthConfig to s3auth.go diff --git a/component/s3storage/s3storage.go b/component/s3storage/s3storage.go index 8eb91fdc9..6eb3d26a3 100644 --- a/component/s3storage/s3storage.go +++ b/component/s3storage/s3storage.go @@ -479,6 +479,10 @@ func (s3 *S3Storage) FlushFile(options internal.FlushFileOptions) error { const blockSize = 4096 func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { + if s3.stConfig.disableUsage { + return nil, false, nil + } + log.Trace("S3Storage::StatFs") // cache_size = f_blocks * f_frsize/1024 // cache_size - used = f_frsize * f_bavail/1024 @@ -487,7 +491,7 @@ func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { sizeUsed, err := s3.storage.GetUsedSize() if err != nil { // TODO: will returning EIO break any applications that depend on StatFs? - return nil, false, err + return nil, true, err } stat := common.Statfs_t{ diff --git a/component/s3storage/s3storage_test.go b/component/s3storage/s3storage_test.go index 5e6f9f1c2..6da5c0145 100644 --- a/component/s3storage/s3storage_test.go +++ b/component/s3storage/s3storage_test.go @@ -95,6 +95,7 @@ type storageTestConfiguration struct { UploadCutoffMb int64 `json:"upload-cutoff-mb"` DisableConcurrentDownload bool `json:"disable-concurrent-download"` UsePathStyle bool `json:"use-path-style"` + DisableUsage bool `json:"disable-usage"` } var storageTestConfigurationParameters storageTestConfiguration @@ -325,10 +326,10 @@ func (s *s3StorageTestSuite) setupTestHelper(configuration string, bucket string func generateConfigYaml(testParams storageTestConfiguration) string { return fmt.Sprintf("s3storage:\n bucket-name: %s\n key-id: %s\n secret-key: %s\n"+ " region: %s\n profile: %s\n endpoint: %s\n subdirectory: %s\n restricted-characters-windows: %t\n"+ - " part-size-mb: %d\n upload-cutoff-mb: %d\n disable-concurrent-download: %t\n use-path-style: %t\n", + " part-size-mb: %d\n upload-cutoff-mb: %d\n disable-concurrent-download: %t\n use-path-style: %t\n disable-usage: %t\n", testParams.BucketName, testParams.KeyID, testParams.SecretKey, testParams.Region, testParams.Profile, testParams.Endpoint, testParams.Prefix, testParams.RestrictedCharsWin, testParams.PartSizeMb, - testParams.UploadCutoffMb, testParams.DisableConcurrentDownload, testParams.UsePathStyle) + testParams.UploadCutoffMb, testParams.DisableConcurrentDownload, testParams.UsePathStyle, testParams.DisableUsage) } func (s *s3StorageTestSuite) tearDownTestHelper(delete bool) { diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index fd971dc10..999032220 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -187,6 +187,7 @@ s3storage: enable-checksum: true|false checksum-algorithm: CRC32|CRC32C|SHA1|SHA256 usePathStyle: true|false + disable-usage: true|false # Mount all configuration mountall: From 9e795e96d9bef2a70afc88919fc175fc70495315 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Tue, 5 Nov 2024 13:31:39 -0700 Subject: [PATCH 18/18] Add explicit note that only Lyve Cloud provides bucket size for StatFs --- setup/baseConfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index 999032220..1a92120a6 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -187,7 +187,7 @@ s3storage: enable-checksum: true|false checksum-algorithm: CRC32|CRC32C|SHA1|SHA256 usePathStyle: true|false - disable-usage: true|false + disable-usage: true|false # Mount all configuration mountall: