Merge pull request #1012 from alfrunes/MEN-7134

feat: Add configuration for max data size when generating artifacts

api/http/api_deployments.go

@@ -52,8 +52,9 @@ const (
 	// 15 minutes
 	DefaultDownloadLinkExpire = 15 * time.Minute
 	// 10 Mb
-	DefaultMaxMetaSize  = 1024 * 1024 * 10
-	DefaultMaxImageSize = 10 * 1024 * 1024 * 1024 // 10GiB
+	DefaultMaxMetaSize         = 1024 * 1024 * 10
+	DefaultMaxImageSize        = 10 * 1024 * 1024 * 1024 // 10GiB
+	DefaultMaxGenerateDataSize = 512 * 1024 * 1024       // 512MiB
 
 	// Pagination
 	DefaultPerPage                      = 20
@@ -138,7 +139,8 @@ type Config struct {
 	// PresignScheme is the URL scheme used for generating signed URLs.
 	PresignScheme string
 	// MaxImageSize is the maximum image size
-	MaxImageSize int64
+	MaxImageSize        int64
+	MaxGenerateDataSize int64
 
 	EnableDirectUpload bool
 	// EnableDirectUploadSkipVerify allows turning off the verification of uploaded artifacts
@@ -152,9 +154,10 @@ type Config struct {
 
 func NewConfig() *Config {
 	return &Config{
-		PresignExpire: DefaultDownloadLinkExpire,
-		PresignScheme: "https",
-		MaxImageSize:  DefaultMaxImageSize,
+		PresignExpire:       DefaultDownloadLinkExpire,
+		PresignScheme:       "https",
+		MaxImageSize:        DefaultMaxImageSize,
+		MaxGenerateDataSize: DefaultMaxGenerateDataSize,
 	}
 }
 
@@ -183,6 +186,11 @@ func (conf *Config) SetMaxImageSize(size int64) *Config {
 	return conf
 }
 
+func (conf *Config) SetMaxGenerateDataSize(size int64) *Config {
+	conf.MaxGenerateDataSize = size
+	return conf
+}
+
 func (conf *Config) SetEnableDirectUpload(enable bool) *Config {
 	conf.EnableDirectUpload = enable
 	return conf
@@ -231,6 +239,9 @@ func NewDeploymentsApiHandlers(
 		if c.MaxImageSize > 0 {
 			conf.MaxImageSize = c.MaxImageSize
 		}
+		if c.MaxGenerateDataSize > 0 {
+			conf.MaxGenerateDataSize = c.MaxGenerateDataSize
+		}
 		conf.DisableNewReleasesFeature = c.DisableNewReleasesFeature
 		conf.EnableDirectUpload = c.EnableDirectUpload
 		conf.EnableDirectUploadSkipVerify = c.EnableDirectUploadSkipVerify
@@ -854,8 +865,6 @@ func (d *DeploymentsApiHandlers) ParseMultipart(
 			if err != nil {
 				return nil, err
 			}
-			// Add one since this will impose the upper limit on the
-			// artifact size.
 			if size > d.config.MaxImageSize {
 				return nil, ErrModelArtifactFileTooLarge
 			}
@@ -942,7 +951,7 @@ ParseLoop:
 			if size > 0 {
 				msg.FileReader = utils.ReadExactly(part, size)
 			} else {
-				msg.FileReader = utils.ReadAtMost(part, d.config.MaxImageSize)
+				msg.FileReader = utils.ReadAtMost(part, d.config.MaxGenerateDataSize)
 			}
 			break ParseLoop
 
@@ -974,9 +983,7 @@ ParseLoop:
 			if err != nil {
 				return nil, err
 			}
-			// Add one since this will impose the upper limit on the
-			// artifact size.
-			if size > d.config.MaxImageSize {
+			if size > d.config.MaxGenerateDataSize {
 				return nil, ErrModelArtifactFileTooLarge
 			}
 

config.yaml

@@ -108,12 +108,17 @@ storage:
     # Overwrite with environment variable: DEPLOYMENTS_STORAGE_PROXY_URI
     # proxy_uri
 
-    # Maximum image size in bytes
-    # Defaults to: 10GB
+    # storage.max_image_size: Maximum image size in bytes.
+    # Defaults to: 10GiB
     # Overwrite with environment variable: DEPLOYMENTS_STORAGE_MAX_IMAGE_SIZE
-
     # max_image_size: 10737418240
 
+    # storage.max_generate_data_size: Maximum data size for requests to 
+    # generate artifacts.
+    # Defaults to: 512MiB
+    # Overwrite with environment variable: DEPLOYMENTS_STORAGE_MAX_GENERATE_DATA_SIZE
+    # max_generate_data_size: 536870912
+
     # Download link expiry duration
     # Number of second a presigned download URL is valid
     # Defaults to: 900 (15 minutes)

config/config.go

@@ -39,12 +39,14 @@ const (
 
 	SettingStorage = "storage"
 
-	SettingDefaultStorage             = SettingStorage + ".default"
-	SettingDefaultStorageDefault      = "aws"
-	SettingStorageBucket              = SettingStorage + ".bucket"
-	SettingStorageBucketDefault       = "mender-artifact-storage"
-	SettingStorageMaxImageSize        = SettingStorage + ".max_image_size"
-	SettingStorageMaxImageSizeDefault = 10 * 1024 * 1024 * 1024 // 10 GiB
+	SettingDefaultStorage                = SettingStorage + ".default"
+	SettingDefaultStorageDefault         = "aws"
+	SettingStorageBucket                 = SettingStorage + ".bucket"
+	SettingStorageBucketDefault          = "mender-artifact-storage"
+	SettingStorageMaxImageSize           = SettingStorage + ".max_image_size"
+	SettingStorageMaxImageSizeDefault    = 10 * 1024 * 1024 * 1024 // 10 GiB
+	SettingStorageMaxGenerateSize        = SettingStorage + ".max_generate_data_size"
+	SettingStorageMaxGenerateSizeDefault = 512 * 1024 * 1024 // 512 MiB
 
 	SettingStorageProxyURI = SettingStorage + ".proxy_uri"
 
@@ -292,6 +294,7 @@ var (
 		{Key: SettingAwsS3UseAccelerate, Value: SettingAwsS3UseAccelerateDefault},
 		{Key: SettingAwsUnsignedHeaders, Value: SettingAwsUnsignedHeadersDefault},
 		{Key: SettingStorageMaxImageSize, Value: SettingStorageMaxImageSizeDefault},
+		{Key: SettingStorageMaxGenerateSize, Value: SettingStorageMaxGenerateSizeDefault},
 		{Key: SettingsStorageDownloadExpireSeconds,
 			Value: SettingsStorageDownloadExpireSecondsDefault},
 		{Key: SettingsStorageUploadExpireSeconds, Value: SettingsStorageUploadExpireSecondsDefault},

server.go

@@ -195,6 +195,7 @@ func RunServer(ctx context.Context) error {
 		SetPresignHostname(c.GetString(dconfig.SettingPresignHost)).
 		SetPresignScheme(c.GetString(dconfig.SettingPresignScheme)).
 		SetMaxImageSize(c.GetInt64(dconfig.SettingStorageMaxImageSize)).
+		SetMaxGenerateDataSize(c.GetInt64(dconfig.SettingStorageMaxGenerateSize)).
 		SetEnableDirectUpload(c.GetBool(dconfig.SettingStorageEnableDirectUpload)).
 		SetEnableDirectUploadSkipVerify(c.GetBool(dconfig.SettingStorageDirectUploadSkipVerify)).
 		SetDisableNewReleasesFeature(c.GetBool(dconfig.SettingDisableNewReleasesFeature))