From 6d10205c657870795e21a0c3b3254ac6be85a940 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 30 Aug 2024 16:08:45 -0600 Subject: [PATCH 1/7] Add display capacity option --- common/types.go | 9 +++++---- component/libfuse/libfuse.go | 15 +++++++++++++-- component/libfuse/libfuse2_handler.go | 3 ++- component/libfuse/libfuse_handler_test.go | 7 +++++-- setup/baseConfig.yaml | 1 + 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/common/types.go b/common/types.go index bee22e40c..e2950a232 100644 --- a/common/types.go +++ b/common/types.go @@ -56,10 +56,11 @@ const ( DefaultDirectoryPermissionBits os.FileMode = 0775 DefaultAllowOtherPermissionBits os.FileMode = 0777 - MbToBytes = 1024 * 1024 - GbToBytes = 1024 * 1024 * 1024 - TbToBytes = 1024 * 1024 * 1024 * 1024 - CfuseStats = "cloudfuse_stats" + MbToBytes = 1024 * 1024 + GbToBytes = 1024 * 1024 * 1024 + TbToBytes = 1024 * 1024 * 1024 * 1024 + DefaultCapacity = TbToBytes + CfuseStats = "cloudfuse_stats" FuseAllowedFlags = "invalid FUSE options. Allowed FUSE configurations are: `-o attr_timeout=TIMEOUT`, `-o negative_timeout=TIMEOUT`, `-o entry_timeout=TIMEOUT` `-o allow_other`, `-o allow_root`, `-o umask=PERMISSIONS -o default_permissions`, `-o ro`" diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index 90ecc7962..594f3ef49 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -72,6 +72,7 @@ type Libfuse struct { maxFuseThreads uint32 directIO bool umask uint32 + displayCapacityMb uint64 } // To support pagination in readdir calls this structure holds a block of items for a given directory @@ -105,6 +106,7 @@ type LibfuseOptions struct { MaxFuseThreads uint32 `config:"max-fuse-threads" yaml:"max-fuse-threads,omitempty"` DirectIO bool `config:"direct-io" yaml:"direct-io,omitempty"` Umask uint32 `config:"umask" yaml:"umask,omitempty"` + DisplayCapacityMb uint64 `config:"display-capacity-mb" yaml:"display-capacity-mb,omitempty"` } const compName = "libfuse" @@ -248,6 +250,12 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.maxFuseThreads = defaultMaxFuseThreads } + if config.IsSet(compName + ".display-capacity-mb") { + lf.displayCapacityMb = opt.DisplayCapacityMb + } else { + lf.displayCapacityMb = common.DefaultCapacity / common.MbToBytes + } + log.Info("Libfuse::Validate : UID %v, GID %v", lf.ownerUID, lf.ownerGID) return nil @@ -309,9 +317,9 @@ func (lf *Libfuse) Configure(_ bool) error { } log.Info("Libfuse::Configure : read-only %t, allow-other %t, allow-root %t, default-perm %d, entry-timeout %d, attr-time %d, negative-timeout %d, "+ - "ignore-open-flags: %t, nonempty %t, network-share %t, direct_io %t, max-fuse-threads %d, fuse-trace %t, extension %s, disable-writeback-cache %t, dirPermission %v, mountPath %v, umask %v", + "ignore-open-flags: %t, nonempty %t, network-share %t, direct_io %t, max-fuse-threads %d, fuse-trace %t, extension %s, disable-writeback-cache %t, dirPermission %v, mountPath %v, umask %v, displayCapacityMb %v", lf.readOnly, lf.allowOther, lf.allowRoot, lf.filePermission, lf.entryExpiration, lf.attributeExpiration, lf.negativeTimeout, - lf.ignoreOpenFlags, lf.nonEmptyMount, lf.networkShare, lf.directIO, lf.maxFuseThreads, lf.traceEnable, lf.extensionPath, lf.disableWritebackCache, lf.dirPermission, lf.mountPath, lf.umask) + lf.ignoreOpenFlags, lf.nonEmptyMount, lf.networkShare, lf.directIO, lf.maxFuseThreads, lf.traceEnable, lf.extensionPath, lf.disableWritebackCache, lf.dirPermission, lf.mountPath, lf.umask, lf.displayCapacityMb) return nil } @@ -354,4 +362,7 @@ func init() { networkShareFlags := config.AddBoolFlag("network-share", false, "Run as a network share. Only supported on Windows.") config.BindPFlag(compName+".network-share", networkShareFlags) + + displayCapacityFlag := config.AddUint64Flag("display-capacity-mb", common.DefaultCapacity/common.MbToBytes, "Storage capacity to display.") + config.BindPFlag(compName+".display-capacity-mb", displayCapacityFlag) } diff --git a/component/libfuse/libfuse2_handler.go b/component/libfuse/libfuse2_handler.go index 040758585..403fac3cf 100644 --- a/component/libfuse/libfuse2_handler.go +++ b/component/libfuse/libfuse2_handler.go @@ -341,7 +341,8 @@ func (cf *CgofuseFS) Statfs(path string, stat *fuse.Statfs_t) int { stat.Namemax = attr.Namemax } else { var free, total, avail uint64 - total = common.TbToBytes + // TODO: if display capacity is specified, should it overwrite populated Bavail? + total = fuseFS.displayCapacityMb * common.MbToBytes avail = total free = total diff --git a/component/libfuse/libfuse_handler_test.go b/component/libfuse/libfuse_handler_test.go index ad82f2378..6f2094202 100644 --- a/component/libfuse/libfuse_handler_test.go +++ b/component/libfuse/libfuse_handler_test.go @@ -49,6 +49,7 @@ func (suite *libfuseTestSuite) TestDefault() { suite.assert.Equal(uint32(120), suite.libfuse.entryExpiration) suite.assert.Equal(uint32(120), suite.libfuse.attributeExpiration) suite.assert.Equal(uint32(120), suite.libfuse.negativeTimeout) + suite.assert.Equal(uint64(1024*1024), suite.libfuse.displayCapacityMb) suite.assert.False(suite.libfuse.disableWritebackCache) suite.assert.True(suite.libfuse.ignoreOpenFlags) suite.assert.False(suite.libfuse.directIO) @@ -57,7 +58,7 @@ func (suite *libfuseTestSuite) TestDefault() { func (suite *libfuseTestSuite) TestConfig() { defer suite.cleanupTest() suite.cleanupTest() // clean up the default libfuse generated - config := "allow-other: true\nread-only: true\nlibfuse:\n attribute-expiration-sec: 60\n entry-expiration-sec: 60\n negative-entry-expiration-sec: 60\n fuse-trace: true\n disable-writeback-cache: true\n ignore-open-flags: false\n direct-io: true\n network-share: true\n" + config := "allow-other: true\nread-only: true\nlibfuse:\n attribute-expiration-sec: 60\n entry-expiration-sec: 60\n negative-entry-expiration-sec: 60\n fuse-trace: true\n disable-writeback-cache: true\n ignore-open-flags: false\n direct-io: true\n network-share: true\n display-capacity-mb: 262144\n" suite.setupTestHelper(config) // setup a new libfuse with a custom config (clean up will occur after the test as usual) suite.assert.Equal("libfuse", suite.libfuse.Name()) @@ -74,13 +75,14 @@ func (suite *libfuseTestSuite) TestConfig() { suite.assert.Equal(uint32(60), suite.libfuse.entryExpiration) suite.assert.Equal(uint32(60), suite.libfuse.attributeExpiration) suite.assert.Equal(uint32(60), suite.libfuse.negativeTimeout) + suite.assert.Equal(uint64(262144), suite.libfuse.displayCapacityMb) suite.assert.True(suite.libfuse.directIO) } func (suite *libfuseTestSuite) TestConfigZero() { defer suite.cleanupTest() suite.cleanupTest() // clean up the default libfuse generated - config := "read-only: true\nlibfuse:\n attribute-expiration-sec: 0\n entry-expiration-sec: 0\n negative-entry-expiration-sec: 0\n fuse-trace: true\n direct-io: false\n" + config := "read-only: true\nlibfuse:\n attribute-expiration-sec: 0\n entry-expiration-sec: 0\n negative-entry-expiration-sec: 0\n fuse-trace: true\n direct-io: false\n display-capacity-mb: 0\n" suite.setupTestHelper(config) // setup a new libfuse with a custom config (clean up will occur after the test as usual) suite.assert.Equal("libfuse", suite.libfuse.Name()) @@ -95,6 +97,7 @@ func (suite *libfuseTestSuite) TestConfigZero() { suite.assert.Equal(uint32(0), suite.libfuse.entryExpiration) suite.assert.Equal(uint32(0), suite.libfuse.attributeExpiration) suite.assert.Equal(uint32(0), suite.libfuse.negativeTimeout) + suite.assert.Equal(uint64(0), suite.libfuse.displayCapacityMb) suite.assert.False(suite.libfuse.directIO) } diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index 6338881e6..c69a4b980 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -70,6 +70,7 @@ libfuse: max-fuse-threads: direct-io: true|false network-share: true|false + display-capacity-mb: # Streaming configuration stream: From e9ed6cf54b871e389b0f09e114656bfa84584f2a Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 30 Aug 2024 17:21:06 -0600 Subject: [PATCH 2/7] Change default capacity units to MB --- common/types.go | 10 +++++----- component/libfuse/libfuse.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/types.go b/common/types.go index e2950a232..2e47a8b94 100644 --- a/common/types.go +++ b/common/types.go @@ -56,11 +56,11 @@ const ( DefaultDirectoryPermissionBits os.FileMode = 0775 DefaultAllowOtherPermissionBits os.FileMode = 0777 - MbToBytes = 1024 * 1024 - GbToBytes = 1024 * 1024 * 1024 - TbToBytes = 1024 * 1024 * 1024 * 1024 - DefaultCapacity = TbToBytes - CfuseStats = "cloudfuse_stats" + MbToBytes = 1024 * 1024 + GbToBytes = 1024 * 1024 * 1024 + TbToBytes = 1024 * 1024 * 1024 * 1024 + DefaultCapacityMb = TbToBytes / MbToBytes + CfuseStats = "cloudfuse_stats" FuseAllowedFlags = "invalid FUSE options. Allowed FUSE configurations are: `-o attr_timeout=TIMEOUT`, `-o negative_timeout=TIMEOUT`, `-o entry_timeout=TIMEOUT` `-o allow_other`, `-o allow_root`, `-o umask=PERMISSIONS -o default_permissions`, `-o ro`" diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index 594f3ef49..5e85205a5 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -253,7 +253,7 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { if config.IsSet(compName + ".display-capacity-mb") { lf.displayCapacityMb = opt.DisplayCapacityMb } else { - lf.displayCapacityMb = common.DefaultCapacity / common.MbToBytes + lf.displayCapacityMb = common.DefaultCapacityMb } log.Info("Libfuse::Validate : UID %v, GID %v", lf.ownerUID, lf.ownerGID) @@ -363,6 +363,6 @@ func init() { networkShareFlags := config.AddBoolFlag("network-share", false, "Run as a network share. Only supported on Windows.") config.BindPFlag(compName+".network-share", networkShareFlags) - displayCapacityFlag := config.AddUint64Flag("display-capacity-mb", common.DefaultCapacity/common.MbToBytes, "Storage capacity to display.") + displayCapacityFlag := config.AddUint64Flag("display-capacity-mb", common.DefaultCapacityMb, "Storage capacity to display.") config.BindPFlag(compName+".display-capacity-mb", displayCapacityFlag) } From 390978ff76c7eb9dbdb7a1d0b0990fc6773523ea Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Fri, 30 Aug 2024 17:37:00 -0600 Subject: [PATCH 3/7] Do not allow zero capacity --- component/libfuse/libfuse.go | 2 +- component/libfuse/libfuse_handler_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index 5e85205a5..f8cabeb94 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -250,7 +250,7 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.maxFuseThreads = defaultMaxFuseThreads } - if config.IsSet(compName + ".display-capacity-mb") { + if config.IsSet(compName+".display-capacity-mb") && opt.DisplayCapacityMb > 0 { lf.displayCapacityMb = opt.DisplayCapacityMb } else { lf.displayCapacityMb = common.DefaultCapacityMb diff --git a/component/libfuse/libfuse_handler_test.go b/component/libfuse/libfuse_handler_test.go index 6f2094202..946b3421f 100644 --- a/component/libfuse/libfuse_handler_test.go +++ b/component/libfuse/libfuse_handler_test.go @@ -97,7 +97,7 @@ func (suite *libfuseTestSuite) TestConfigZero() { suite.assert.Equal(uint32(0), suite.libfuse.entryExpiration) suite.assert.Equal(uint32(0), suite.libfuse.attributeExpiration) suite.assert.Equal(uint32(0), suite.libfuse.negativeTimeout) - suite.assert.Equal(uint64(0), suite.libfuse.displayCapacityMb) + suite.assert.Equal(uint64(1024*1024), suite.libfuse.displayCapacityMb) suite.assert.False(suite.libfuse.directIO) } From 750bf2f8cc4399d8cfee0b03fe4164b231dcd906 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 5 Sep 2024 14:44:02 -0600 Subject: [PATCH 4/7] Add breadcrumbs for GH Actions issue --- component/libfuse/libfuse.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index f8cabeb94..5b2045900 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -250,6 +250,8 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.maxFuseThreads = defaultMaxFuseThreads } + fmt.Printf("Libfuse::Validate - IsSet(%s): %v\n", compName+".display-capacity-mb", config.IsSet(compName+".display-capacity-mb")) + fmt.Printf("Libfuse::Validate - opt.DisplayCapacityMb = %v\n", opt.DisplayCapacityMb) if config.IsSet(compName+".display-capacity-mb") && opt.DisplayCapacityMb > 0 { lf.displayCapacityMb = opt.DisplayCapacityMb } else { @@ -316,6 +318,7 @@ func (lf *Libfuse) Configure(_ bool) error { return fmt.Errorf("%s config error %s", lf.Name(), err.Error()) } + fmt.Printf("Libfuse::Configure - displayCapacityMb: %v\n", lf.displayCapacityMb) log.Info("Libfuse::Configure : read-only %t, allow-other %t, allow-root %t, default-perm %d, entry-timeout %d, attr-time %d, negative-timeout %d, "+ "ignore-open-flags: %t, nonempty %t, network-share %t, direct_io %t, max-fuse-threads %d, fuse-trace %t, extension %s, disable-writeback-cache %t, dirPermission %v, mountPath %v, umask %v, displayCapacityMb %v", lf.readOnly, lf.allowOther, lf.allowRoot, lf.filePermission, lf.entryExpiration, lf.attributeExpiration, lf.negativeTimeout, From ac753423344300f027b2bb2f753dd88c85d421ff Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 5 Sep 2024 15:25:21 -0600 Subject: [PATCH 5/7] Move UID/GID validation to the end of Validate, since it returns on failure. --- component/libfuse/libfuse.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index 5b2045900..eba9fcd4f 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -234,16 +234,6 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.negativeTimeout = defaultNegativeEntryExpiration } - if !(config.IsSet(compName+".uid") || config.IsSet(compName+".gid") || - config.IsSet("lfuse.uid") || config.IsSet("lfuse.gid")) { - var err error - lf.ownerUID, lf.ownerGID, err = common.GetCurrentUser() - if err != nil { - log.Err("Libfuse::Validate : config error [unable to obtain current user info]") - return nil - } - } - if config.IsSet(compName + ".max-fuse-threads") { lf.maxFuseThreads = opt.MaxFuseThreads } else { @@ -258,6 +248,16 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.displayCapacityMb = common.DefaultCapacityMb } + if !(config.IsSet(compName+".uid") || config.IsSet(compName+".gid") || + config.IsSet("lfuse.uid") || config.IsSet("lfuse.gid")) { + var err error + lf.ownerUID, lf.ownerGID, err = common.GetCurrentUser() + if err != nil { + log.Err("Libfuse::Validate : config error [unable to obtain current user info]") + return nil + } + } + log.Info("Libfuse::Validate : UID %v, GID %v", lf.ownerUID, lf.ownerGID) return nil From a58b7112b4507da14e2be3a3a65fdb418060a07e Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 5 Sep 2024 15:27:21 -0600 Subject: [PATCH 6/7] Remove breadcrumbs --- component/libfuse/libfuse.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index eba9fcd4f..b903110db 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -240,8 +240,6 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.maxFuseThreads = defaultMaxFuseThreads } - fmt.Printf("Libfuse::Validate - IsSet(%s): %v\n", compName+".display-capacity-mb", config.IsSet(compName+".display-capacity-mb")) - fmt.Printf("Libfuse::Validate - opt.DisplayCapacityMb = %v\n", opt.DisplayCapacityMb) if config.IsSet(compName+".display-capacity-mb") && opt.DisplayCapacityMb > 0 { lf.displayCapacityMb = opt.DisplayCapacityMb } else { @@ -318,7 +316,6 @@ func (lf *Libfuse) Configure(_ bool) error { return fmt.Errorf("%s config error %s", lf.Name(), err.Error()) } - fmt.Printf("Libfuse::Configure - displayCapacityMb: %v\n", lf.displayCapacityMb) log.Info("Libfuse::Configure : read-only %t, allow-other %t, allow-root %t, default-perm %d, entry-timeout %d, attr-time %d, negative-timeout %d, "+ "ignore-open-flags: %t, nonempty %t, network-share %t, direct_io %t, max-fuse-threads %d, fuse-trace %t, extension %s, disable-writeback-cache %t, dirPermission %v, mountPath %v, umask %v, displayCapacityMb %v", lf.readOnly, lf.allowOther, lf.allowRoot, lf.filePermission, lf.entryExpiration, lf.attributeExpiration, lf.negativeTimeout, From e9d3933428300b2667a5f20fd71d6515c8c641ce Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 5 Sep 2024 15:45:09 -0600 Subject: [PATCH 7/7] libfuse Validate: Don't return on failure when setting lf.ownerUID and lf.ownerGID. Add TODO for failing common.GetCurrentUser() in GitHub Actions on WIndows. --- component/libfuse/libfuse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component/libfuse/libfuse.go b/component/libfuse/libfuse.go index b903110db..4a064f2fd 100644 --- a/component/libfuse/libfuse.go +++ b/component/libfuse/libfuse.go @@ -246,13 +246,13 @@ func (lf *Libfuse) Validate(opt *LibfuseOptions) error { lf.displayCapacityMb = common.DefaultCapacityMb } + // NOTE/TODO: this always fails in GitHub Actions on Windows if !(config.IsSet(compName+".uid") || config.IsSet(compName+".gid") || config.IsSet("lfuse.uid") || config.IsSet("lfuse.gid")) { var err error lf.ownerUID, lf.ownerGID, err = common.GetCurrentUser() if err != nil { log.Err("Libfuse::Validate : config error [unable to obtain current user info]") - return nil } }