mantle/kola, mantle/platform: split test buckets if merged config is too large

We have seen in the past that the non-exclusive tests' merged config can
become too large to pass to cloud providers. To fix this issue we compressed
configs (for AWS), but this is only a temporary fix.

If a non-exclusive test bucket's config becomes too large, let's split the
bucket into two buckets to lower the config size.

Author: saqibali-2k

mantle/kola/harness.go

@@ -593,11 +593,34 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 
 	if len(nonExclusiveTests) > 0 {
 		buckets := createTestBuckets(nonExclusiveTests)
-		for i, bucket := range buckets {
+		numBuckets := len(buckets)
+		for i := 0; i < numBuckets; {
 			// This test does not need to be registered since it is temporarily
 			// created to be used as a wrapper
-			nonExclusiveWrapper := makeNonExclusiveTest(i, bucket, flight)
-			tests[nonExclusiveWrapper.Name] = &nonExclusiveWrapper
+			nonExclusiveWrapper := makeNonExclusiveTest(i, buckets[i], flight)
+			if flight.ConfigTooLarge(*nonExclusiveWrapper.UserData) {
+				// Since the merged config size is too large, we will split the bucket into
+				// two buckets
+				numTests := len(buckets[i])
+				if numTests == 1 {
+					// This test bucket cannot be split further so the single test config
+					// must be too large
+					err = fmt.Errorf("test %v has a config that is too large", buckets[i][0].Name)
+					plog.Fatal(err)
+				}
+				newBucket1 := buckets[i][:numTests/2]
+				newBucket2 := buckets[i][numTests/2:]
+				buckets[i] = newBucket1
+				buckets = append(buckets, newBucket2)
+				// Since we're adding a bucket we'll bump the numBuckets and not
+				// bump `i` during this loop iteration because we want to run through
+				// the check again for the current bucket which should now have half
+				// the tests, but may still have a config that's too large.
+				numBuckets++
+			} else {
+				tests[nonExclusiveWrapper.Name] = &nonExclusiveWrapper
+				i++ // Move to the next bucket to evaluate
+			}
 		}
 	}
 

mantle/platform/machine/aws/cluster.go

@@ -30,7 +30,7 @@ type cluster struct {
 	flight *flight
 }
 
-const maxUserDataSize = 16384
+const MaxUserDataSize = 16384
 
 func (ac *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) {
 	return ac.NewMachineWithOptions(userdata, platform.MachineOptions{})
@@ -68,15 +68,15 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo
 	// Compress via gzip if needed
 	ud := conf.String()
 	encoding := base64.StdEncoding.EncodeToString([]byte(ud))
-	if len([]byte(encoding)) > maxUserDataSize {
+	if len([]byte(encoding)) > MaxUserDataSize {
 		ud, err = conf.MaybeCompress()
 		if err != nil {
 			return nil, err
 		}
 		// Check if config is still too large
 		encoding = base64.StdEncoding.EncodeToString([]byte(ud))
-		if len([]byte(encoding)) > maxUserDataSize {
-			fmt.Printf("WARNING: compressed userdata exceeds expected limit of %d\n", maxUserDataSize)
+		if len([]byte(encoding)) > MaxUserDataSize {
+			fmt.Printf("WARNING: compressed userdata exceeds expected limit of %d\n", MaxUserDataSize)
 		}
 	}
 	instances, err := ac.flight.api.CreateInstances(ac.Name(), keyname, ud, 1, int64(options.MinDiskSize))

mantle/platform/machine/aws/flight.go

@@ -16,10 +16,13 @@
 package aws
 
 import (
+	"encoding/base64"
+
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/aws"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -97,6 +100,30 @@ func (af *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 	return ac, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+	config, err := ud.Render()
+	if err != nil {
+		return true
+	}
+	configData := config.String()
+	encoding := base64.StdEncoding.EncodeToString([]byte(configData))
+	if len([]byte(encoding)) > MaxUserDataSize {
+		configData, err = config.MaybeCompress()
+		if err != nil {
+			return true
+		}
+		// Check if config is still too large
+		encoding = base64.StdEncoding.EncodeToString([]byte(configData))
+		if len([]byte(encoding)) > MaxUserDataSize {
+			// Config is too large
+			return true
+		}
+	}
+
+	// Config is not too large
+	return false
+}
+
 func (af *flight) Destroy() {
 	if af.keyAdded {
 		if err := af.api.DeleteKey(af.Name()); err != nil {

mantle/platform/machine/azure/flight.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/azure"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -118,6 +119,12 @@ func (af *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 	return ac, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 func (af *flight) Destroy() {
 	af.BaseFlight.Destroy()
 }

mantle/platform/machine/do/flight.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/do"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -106,6 +107,12 @@ func (df *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 	return dc, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 func (df *flight) Destroy() {
 	for _, keyID := range []int{df.sshKeyID, df.fakeSSHKeyID} {
 		if keyID == 0 {

mantle/platform/machine/esx/flight.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/esx"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -56,6 +57,12 @@ func NewFlight(opts *esx.Options) (platform.Flight, error) {
 	return ef, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 // NewCluster creates an instance of a Cluster suitable for spawning
 // instances on VMware ESXi vSphere platform.
 func (ef *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, error) {

mantle/platform/machine/gcloud/flight.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/gcloud"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 type flight struct {
@@ -55,6 +56,12 @@ func NewFlight(opts *gcloud.Options) (platform.Flight, error) {
 	return gf, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 func (gf *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, error) {
 	bc, err := platform.NewBaseCluster(gf.BaseFlight, rconf)
 	if err != nil {

mantle/platform/machine/openstack/flight.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/openstack"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -86,6 +87,12 @@ func (of *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 	return oc, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 func (of *flight) Destroy() {
 	if of.keyAdded {
 		if err := of.api.DeleteKey(of.Name()); err != nil {

mantle/platform/machine/packet/flight.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/api/packet"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -85,6 +86,12 @@ func (pf *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 	return pc, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 func (pf *flight) Destroy() {
 	if pf.sshKeyID != "" {
 		if err := pf.api.DeleteKey(pf.sshKeyID); err != nil {

mantle/platform/machine/qemuiso/flight.go

@@ -18,6 +18,7 @@ import (
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/mantle/platform"
+	"github.com/coreos/mantle/platform/conf"
 )
 
 const (
@@ -58,6 +59,12 @@ func NewFlight(opts *Options) (platform.Flight, error) {
 	return qf, nil
 }
 
+func (af *flight) ConfigTooLarge(ud conf.UserData) bool {
+
+	// not implemented
+	return false
+}
+
 // NewCluster creates a Cluster instance, suitable for running virtual
 // machines in QEMU.
 func (qf *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, error) {