[feature] update config types to use bytesize.Size (#828)

* update config size types to use bytesize.Size * submit unchecked-out file ... 🤦 * fix bytesize config var decoding * bump bytesize version * update kim's libraries in readme * update envparse.sh to output more useful errors * improve envparse.sh * remove reliance on jq * instead, use uint64 for bytesize flag types * remove redundant type * fix viper unmarshaling * Update envparsing.sh * fix envparsing test Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
superseriousbusiness · Sep 29, 2022 · 1d99971 · 1d99971
1 parent f0bf69d
commit 1d99971
Show file tree

Hide file tree

Showing 30 changed files with 223 additions and 169 deletions.
diff --git a/README.md b/README.md
@@ -219,6 +219,8 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
 - [go-playground/validator](https://github.com/go-playground/validator); struct validation. [MIT License](https://spdx.org/licenses/MIT.html).
 - [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html).
 - [gruf/go-debug](https://codeberg.org/gruf/go-debug); profiling support in debug builds. [MIT License](https://spdx.org/licenses/MIT.html).
+- [gruf/go-bytesize](https://codeberg.org/gruf/go-bytesize); byte size parsing / formatting. [MIT License](https://spdx.org/licenses/MIT.html).
+- [gruf/go-cache](https://codeberg.org/gruf/go-cache); object caching. [MIT License](https://spdx.org/licenses/MIT.html).
 - [gruf/go-kv](https://codeberg.org/gruf/go-kv); key-value field formatting. [MIT License](https://spdx.org/licenses/MIT.html).
 - [gruf/go-mutexes](https://codeberg.org/gruf/go-mutexes); mutex map. [MIT License](https://spdx.org/licenses/MIT.html).
 - [gruf/go-runners](https://codeberg.org/gruf/go-runners); worker pool library. [MIT License](https://spdx.org/licenses/MIT.html).

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	codeberg.org/gruf/go-atomics v1.1.0
-	codeberg.org/gruf/go-bytesize v0.2.1
+	codeberg.org/gruf/go-bytesize v1.0.0
 	codeberg.org/gruf/go-byteutil v1.0.2
 	codeberg.org/gruf/go-cache/v2 v2.1.4
 	codeberg.org/gruf/go-debug v1.2.0

diff --git a/go.sum b/go.sum
@@ -64,8 +64,8 @@ codeberg.org/gruf/go-bitutil v1.0.1/go.mod h1:3ezHnADoiRJs9jgn65AEZ3HY7dsabAYLmm
 codeberg.org/gruf/go-bytes v1.0.0/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg=
 codeberg.org/gruf/go-bytes v1.0.2 h1:malqE42Ni+h1nnYWBUAJaDDtEzF4aeN4uPN8DfMNNvo=
 codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg=
-codeberg.org/gruf/go-bytesize v0.2.1 h1:nbAta3FCYe3Q18osqg8Ylk/naOopdqEKiKMpo6KTpAA=
-codeberg.org/gruf/go-bytesize v0.2.1/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacpp0OHfkvLPs=
+codeberg.org/gruf/go-bytesize v1.0.0 h1:/Mcv4prniJLkPEqZ+LZ5/D/e27rNrZZEMmty9jpIvlc=
+codeberg.org/gruf/go-bytesize v1.0.0/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacpp0OHfkvLPs=
 codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
 codeberg.org/gruf/go-byteutil v1.0.2 h1:OesVyK5VKWeWdeDR00zRJ+Oy8hjXx1pBhn7WVvcZWVE=
 codeberg.org/gruf/go-byteutil v1.0.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -21,6 +21,7 @@ package config
 import (
 	"reflect"
 
+	"codeberg.org/gruf/go-bytesize"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -76,13 +77,13 @@ type Configuration struct {
 	AccountsReasonRequired   bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`
 	AccountsAllowCustomCSS   bool `name:"accounts-allow-custom-css" usage:"Allow accounts to enable custom CSS for their profile pages and statuses."`
 
-	MediaImageMaxSize        int `name:"media-image-max-size" usage:"Max size of accepted images in bytes"`
-	MediaVideoMaxSize        int `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"`
-	MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"`
-	MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
-	MediaRemoteCacheDays     int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
-	MediaEmojiLocalMaxSize   int `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
-	MediaEmojiRemoteMaxSize  int `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
+	MediaImageMaxSize        bytesize.Size `name:"media-image-max-size" usage:"Max size of accepted images in bytes"`
+	MediaVideoMaxSize        bytesize.Size `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"`
+	MediaDescriptionMinChars int           `name:"media-description-min-chars" usage:"Min required chars for an image description"`
+	MediaDescriptionMaxChars int           `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
+	MediaRemoteCacheDays     int           `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
+	MediaEmojiLocalMaxSize   bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
+	MediaEmojiRemoteMaxSize  bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
 
 	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
 	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`

diff --git a/internal/config/flags.go b/internal/config/flags.go
@@ -72,13 +72,13 @@ func AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Bool(AccountsAllowCustomCSSFlag(), cfg.AccountsAllowCustomCSS, fieldtag("AccountsAllowCustomCSS", "usage"))
 
 		// Media
-		cmd.Flags().Int(MediaImageMaxSizeFlag(), cfg.MediaImageMaxSize, fieldtag("MediaImageMaxSize", "usage"))
-		cmd.Flags().Int(MediaVideoMaxSizeFlag(), cfg.MediaVideoMaxSize, fieldtag("MediaVideoMaxSize", "usage"))
+		cmd.Flags().Uint64(MediaImageMaxSizeFlag(), uint64(cfg.MediaImageMaxSize), fieldtag("MediaImageMaxSize", "usage"))
+		cmd.Flags().Uint64(MediaVideoMaxSizeFlag(), uint64(cfg.MediaVideoMaxSize), fieldtag("MediaVideoMaxSize", "usage"))
 		cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage"))
 		cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage"))
 		cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
-		cmd.Flags().Int(MediaEmojiLocalMaxSizeFlag(), cfg.MediaEmojiLocalMaxSize, fieldtag("MediaEmojiLocalMaxSize", "usage"))
-		cmd.Flags().Int(MediaEmojiRemoteMaxSizeFlag(), cfg.MediaEmojiRemoteMaxSize, fieldtag("MediaEmojiRemoteMaxSize", "usage"))
+		cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage"))
+		cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage"))
 
 		// Storage
 		cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage"))

diff --git a/internal/config/gen/gen.go b/internal/config/gen/gen.go
@@ -100,7 +100,7 @@ func main() {
 			fmt.Fprintf(output, "func Set%[1]s(v %[2]s) { global.Set%[1]s(v) }\n\n", field.Name, field.Type.String())
 		}
 		_ = output.Close()
-		_ = exec.Command("gofmt", "-w", out).Run()
+		_ = exec.Command("gofumports", "-w", out).Run()
 
 	// The plain here is that eventually we might be able
 	// to generate an example configuration from struct tags

diff --git a/internal/config/global.go b/internal/config/global.go
@@ -18,7 +18,9 @@
 
 package config
 
-import "github.com/spf13/cobra"
+import (
+	"github.com/spf13/cobra"
+)
 
 var global *ConfigState
 

diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
@@ -18,6 +18,8 @@
 */
 package config
 
+import "codeberg.org/gruf/go-bytesize"
+
 // GetLogLevel safely fetches the Configuration value for state's 'LogLevel' field
 func (st *ConfigState) GetLogLevel() (v string) {
 	st.mutex.Lock()
@@ -719,15 +721,15 @@ func GetAccountsAllowCustomCSS() bool { return global.GetAccountsAllowCustomCSS(
 func SetAccountsAllowCustomCSS(v bool) { global.SetAccountsAllowCustomCSS(v) }
 
 // GetMediaImageMaxSize safely fetches the Configuration value for state's 'MediaImageMaxSize' field
-func (st *ConfigState) GetMediaImageMaxSize() (v int) {
+func (st *ConfigState) GetMediaImageMaxSize() (v bytesize.Size) {
 	st.mutex.Lock()
 	v = st.config.MediaImageMaxSize
 	st.mutex.Unlock()
 	return
 }
 
 // SetMediaImageMaxSize safely sets the Configuration value for state's 'MediaImageMaxSize' field
-func (st *ConfigState) SetMediaImageMaxSize(v int) {
+func (st *ConfigState) SetMediaImageMaxSize(v bytesize.Size) {
 	st.mutex.Lock()
 	defer st.mutex.Unlock()
 	st.config.MediaImageMaxSize = v
@@ -738,21 +740,21 @@ func (st *ConfigState) SetMediaImageMaxSize(v int) {
 func MediaImageMaxSizeFlag() string { return "media-image-max-size" }
 
 // GetMediaImageMaxSize safely fetches the value for global configuration 'MediaImageMaxSize' field
-func GetMediaImageMaxSize() int { return global.GetMediaImageMaxSize() }
+func GetMediaImageMaxSize() bytesize.Size { return global.GetMediaImageMaxSize() }
 
 // SetMediaImageMaxSize safely sets the value for global configuration 'MediaImageMaxSize' field
-func SetMediaImageMaxSize(v int) { global.SetMediaImageMaxSize(v) }
+func SetMediaImageMaxSize(v bytesize.Size) { global.SetMediaImageMaxSize(v) }
 
 // GetMediaVideoMaxSize safely fetches the Configuration value for state's 'MediaVideoMaxSize' field
-func (st *ConfigState) GetMediaVideoMaxSize() (v int) {
+func (st *ConfigState) GetMediaVideoMaxSize() (v bytesize.Size) {
 	st.mutex.Lock()
 	v = st.config.MediaVideoMaxSize
 	st.mutex.Unlock()
 	return
 }
 
 // SetMediaVideoMaxSize safely sets the Configuration value for state's 'MediaVideoMaxSize' field
-func (st *ConfigState) SetMediaVideoMaxSize(v int) {
+func (st *ConfigState) SetMediaVideoMaxSize(v bytesize.Size) {
 	st.mutex.Lock()
 	defer st.mutex.Unlock()
 	st.config.MediaVideoMaxSize = v
@@ -763,10 +765,10 @@ func (st *ConfigState) SetMediaVideoMaxSize(v int) {
 func MediaVideoMaxSizeFlag() string { return "media-video-max-size" }
 
 // GetMediaVideoMaxSize safely fetches the value for global configuration 'MediaVideoMaxSize' field
-func GetMediaVideoMaxSize() int { return global.GetMediaVideoMaxSize() }
+func GetMediaVideoMaxSize() bytesize.Size { return global.GetMediaVideoMaxSize() }
 
 // SetMediaVideoMaxSize safely sets the value for global configuration 'MediaVideoMaxSize' field
-func SetMediaVideoMaxSize(v int) { global.SetMediaVideoMaxSize(v) }
+func SetMediaVideoMaxSize(v bytesize.Size) { global.SetMediaVideoMaxSize(v) }
 
 // GetMediaDescriptionMinChars safely fetches the Configuration value for state's 'MediaDescriptionMinChars' field
 func (st *ConfigState) GetMediaDescriptionMinChars() (v int) {
@@ -844,15 +846,15 @@ func GetMediaRemoteCacheDays() int { return global.GetMediaRemoteCacheDays() }
 func SetMediaRemoteCacheDays(v int) { global.SetMediaRemoteCacheDays(v) }
 
 // GetMediaEmojiLocalMaxSize safely fetches the Configuration value for state's 'MediaEmojiLocalMaxSize' field
-func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v int) {
+func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v bytesize.Size) {
 	st.mutex.Lock()
 	v = st.config.MediaEmojiLocalMaxSize
 	st.mutex.Unlock()
 	return
 }
 
 // SetMediaEmojiLocalMaxSize safely sets the Configuration value for state's 'MediaEmojiLocalMaxSize' field
-func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) {
+func (st *ConfigState) SetMediaEmojiLocalMaxSize(v bytesize.Size) {
 	st.mutex.Lock()
 	defer st.mutex.Unlock()
 	st.config.MediaEmojiLocalMaxSize = v
@@ -863,21 +865,21 @@ func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) {
 func MediaEmojiLocalMaxSizeFlag() string { return "media-emoji-local-max-size" }
 
 // GetMediaEmojiLocalMaxSize safely fetches the value for global configuration 'MediaEmojiLocalMaxSize' field
-func GetMediaEmojiLocalMaxSize() int { return global.GetMediaEmojiLocalMaxSize() }
+func GetMediaEmojiLocalMaxSize() bytesize.Size { return global.GetMediaEmojiLocalMaxSize() }
 
 // SetMediaEmojiLocalMaxSize safely sets the value for global configuration 'MediaEmojiLocalMaxSize' field
-func SetMediaEmojiLocalMaxSize(v int) { global.SetMediaEmojiLocalMaxSize(v) }
+func SetMediaEmojiLocalMaxSize(v bytesize.Size) { global.SetMediaEmojiLocalMaxSize(v) }
 
 // GetMediaEmojiRemoteMaxSize safely fetches the Configuration value for state's 'MediaEmojiRemoteMaxSize' field
-func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v int) {
+func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v bytesize.Size) {
 	st.mutex.Lock()
 	v = st.config.MediaEmojiRemoteMaxSize
 	st.mutex.Unlock()
 	return
 }
 
 // SetMediaEmojiRemoteMaxSize safely sets the Configuration value for state's 'MediaEmojiRemoteMaxSize' field
-func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) {
+func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v bytesize.Size) {
 	st.mutex.Lock()
 	defer st.mutex.Unlock()
 	st.config.MediaEmojiRemoteMaxSize = v
@@ -888,10 +890,10 @@ func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) {
 func MediaEmojiRemoteMaxSizeFlag() string { return "media-emoji-remote-max-size" }
 
 // GetMediaEmojiRemoteMaxSize safely fetches the value for global configuration 'MediaEmojiRemoteMaxSize' field
-func GetMediaEmojiRemoteMaxSize() int { return global.GetMediaEmojiRemoteMaxSize() }
+func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRemoteMaxSize() }
 
 // SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
-func SetMediaEmojiRemoteMaxSize(v int) { global.SetMediaEmojiRemoteMaxSize(v) }
+func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) }
 
 // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field
 func (st *ConfigState) GetStorageBackend() (v string) {

diff --git a/internal/config/state.go b/internal/config/state.go
@@ -133,6 +133,12 @@ func (st *ConfigState) reloadFromViper() {
 	if err := st.viper.Unmarshal(&st.config, func(c *mapstructure.DecoderConfig) {
 		c.TagName = "name"
 		c.ZeroFields = true // empty the config struct before we marshal values into it
+
+		oldhook := c.DecodeHook
+		c.DecodeHook = mapstructure.ComposeDecodeHookFunc(
+			mapstructure.TextUnmarshallerHookFunc(),
+			oldhook,
+		)
 	}); err != nil {
 		panic(err)
 	}

diff --git a/internal/config/validate.go b/internal/config/validate.go
@@ -67,14 +67,6 @@ func Validate() error {
 		errs = append(errs, fmt.Errorf("%s must be set", WebAssetBaseDirFlag()))
 	}
 
-	if m := GetMediaEmojiLocalMaxSize(); m < 0 {
-		errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiLocalMaxSizeFlag()))
-	}
-
-	if m := GetMediaEmojiRemoteMaxSize(); m < 0 {
-		errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiRemoteMaxSizeFlag()))
-	}
-
 	if len(errs) > 0 {
 		errStrings := []string{}
 		for _, err := range errs {

diff --git a/internal/config/validate_test.go b/internal/config/validate_test.go
@@ -141,16 +141,6 @@ func (suite *ConfigValidateTestSuite) TestValidateConfigBadProtocolNoHost() {
 	suite.EqualError(err, "host must be set; protocol must be set to either http or https, provided value was foo")
 }
 
-func (suite *ConfigValidateTestSuite) TestValidateConfigBadEmojiSizes() {
-	testrig.InitTestConfig()
-
-	config.SetMediaEmojiLocalMaxSize(-10)
-	config.SetMediaEmojiRemoteMaxSize(-50)
-
-	err := config.Validate()
-	suite.EqualError(err, "media-emoji-local-max-size must not be less than 0; media-emoji-remote-max-size must not be less than 0")
-}
-
 func TestConfigValidateTestSuite(t *testing.T) {
 	suite.Run(t, &ConfigValidateTestSuite{})
 }
diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
@@ -496,7 +496,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 				}
 			}
 
-			data := func(innerCtx context.Context) (io.Reader, int, error) {
+			data := func(innerCtx context.Context) (io.Reader, int64, error) {
 				return t.DereferenceMedia(innerCtx, avatarIRI)
 			}
 
@@ -562,7 +562,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 				}
 			}
 
-			data := func(innerCtx context.Context) (io.Reader, int, error) {
+			data := func(innerCtx context.Context) (io.Reader, int64, error) {
 				return t.DereferenceMedia(innerCtx, headerIRI)
 			}
 

diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go
@@ -42,7 +42,7 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r
 		return nil, fmt.Errorf("GetRemoteEmoji: error parsing url: %s", err)
 	}
 
-	dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
+	dataFunc := func(innerCtx context.Context) (io.Reader, int64, error) {
 		return t.DereferenceMedia(innerCtx, derefURI)
 	}
 

diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go
@@ -42,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a
 		return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err)
 	}
 
-	dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
+	dataFunc := func(innerCtx context.Context) (io.Reader, int64, error) {
 		return t.DereferenceMedia(innerCtx, derefURI)
 	}