EVG-13235: add admin field for host S3 base URL for evergreen binary …

…downloads (#4090)
evergreen-ci · Oct 20, 2020 · c1b63d3 · c1b63d3
1 parent 4d8fd79
commit c1b63d3
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 26 deletions.
diff --git a/config_db.go b/config_db.go
@@ -116,6 +116,12 @@ var (
 	hostJasperURLKey              = bsonutil.MustHaveTag(HostJasperConfig{}, "URL")
 	hostJasperVersionKey          = bsonutil.MustHaveTag(HostJasperConfig{}, "Version")
 
+	hostInitHostThrottleKey         = bsonutil.MustHaveTag(HostInitConfig{}, "HostThrottle")
+	hostInitProvisioningThrottleKey = bsonutil.MustHaveTag(HostInitConfig{}, "ProvisioningThrottle")
+	hostInitCloudStatusBatchSizeKey = bsonutil.MustHaveTag(HostInitConfig{}, "CloudStatusBatchSize")
+	hostInitMaxTotalDynamicHostsKey = bsonutil.MustHaveTag(HostInitConfig{}, "MaxTotalDynamicHosts")
+	hostInitS3BaseURLKey            = bsonutil.MustHaveTag(HostInitConfig{}, "S3BaseURL")
+
 	// Spawnhost keys
 	unexpirableHostsPerUserKey   = bsonutil.MustHaveTag(SpawnHostConfig{}, "UnexpirableHostsPerUser")
 	unexpirableVolumesPerUserKey = bsonutil.MustHaveTag(SpawnHostConfig{}, "UnexpirableVolumesPerUser")

diff --git a/config_hostinit.go b/config_hostinit.go
@@ -11,10 +11,11 @@ const defaultHostThrottle = 32
 
 // HostInitConfig holds logging settings for the hostinit process.
 type HostInitConfig struct {
-	HostThrottle         int `bson:"host_throttle" json:"host_throttle" yaml:"host_throttle"`
-	ProvisioningThrottle int `bson:"provisioning_throttle" json:"provisioning_throttle" yaml:"provisioning_throttle"`
-	CloudStatusBatchSize int `bson:"cloud_batch_size" json:"cloud_batch_size" yaml:"cloud_batch_size"`
-	MaxTotalDynamicHosts int `bson:"max_total_dynamic_hosts" json:"max_total_dynamic_hosts" yaml:"max_total_dynamic_hosts"`
+	HostThrottle         int    `bson:"host_throttle" json:"host_throttle" yaml:"host_throttle"`
+	ProvisioningThrottle int    `bson:"provisioning_throttle" json:"provisioning_throttle" yaml:"provisioning_throttle"`
+	CloudStatusBatchSize int    `bson:"cloud_batch_size" json:"cloud_batch_size" yaml:"cloud_batch_size"`
+	MaxTotalDynamicHosts int    `bson:"max_total_dynamic_hosts" json:"max_total_dynamic_hosts" yaml:"max_total_dynamic_hosts"`
+	S3BaseURL            string `bson:"s3_base_url" json:"s3_base_url" yaml:"s3_base_url"`
 }
 
 func (c *HostInitConfig) SectionId() string { return "hostinit" }
@@ -47,10 +48,11 @@ func (c *HostInitConfig) Set() error {
 
 	_, err := coll.UpdateOne(ctx, byId(c.SectionId()), bson.M{
 		"$set": bson.M{
-			"host_throttle":           c.HostThrottle,
-			"provisioning_throttle":   c.ProvisioningThrottle,
-			"cloud_batch_size":        c.CloudStatusBatchSize,
-			"max_total_dynamic_hosts": c.MaxTotalDynamicHosts,
+			hostInitHostThrottleKey:         c.HostThrottle,
+			hostInitProvisioningThrottleKey: c.ProvisioningThrottle,
+			hostInitCloudStatusBatchSizeKey: c.CloudStatusBatchSize,
+			hostInitMaxTotalDynamicHostsKey: c.MaxTotalDynamicHosts,
+			hostInitS3BaseURLKey:            c.S3BaseURL,
 		},
 	}, options.Update().SetUpsert(true))
 

diff --git a/rest/data/admin_test.go b/rest/data/admin_test.go
@@ -147,6 +147,10 @@ func (s *AdminDataSuite) TestSetAndGetSettings() {
 	s.Equal(testSettings.AuthConfig.Multi.ReadWrite[0], settingsFromConnector.AuthConfig.Multi.ReadWrite[0])
 	s.EqualValues(testSettings.HostJasper.URL, settingsFromConnector.HostJasper.URL)
 	s.EqualValues(testSettings.HostInit.HostThrottle, settingsFromConnector.HostInit.HostThrottle)
+	s.EqualValues(testSettings.HostInit.ProvisioningThrottle, settingsFromConnector.HostInit.ProvisioningThrottle)
+	s.EqualValues(testSettings.HostInit.CloudStatusBatchSize, settingsFromConnector.HostInit.CloudStatusBatchSize)
+	s.EqualValues(testSettings.HostInit.MaxTotalDynamicHosts, settingsFromConnector.HostInit.MaxTotalDynamicHosts)
+	s.EqualValues(testSettings.HostInit.S3BaseURL, settingsFromConnector.HostInit.S3BaseURL)
 	s.EqualValues(testSettings.Jira.BasicAuthConfig.Username, settingsFromConnector.Jira.BasicAuthConfig.Username)
 	// We have to check different cases because the mock connector does not set
 	// defaults for the settings.
@@ -224,27 +228,32 @@ func (s *AdminDataSuite) TestSetAndGetSettings() {
 	// test that updating the model with nil values does not change them
 	newBanner := "new banner"
 	newExpansions := map[string]string{"newkey": "newval"}
-	newHostinit := restModel.APIHostInitConfig{
+	newHostInit := restModel.APIHostInitConfig{
 		HostThrottle:         64,
 		CloudStatusBatchSize: 1,
 		ProvisioningThrottle: 200,
 		MaxTotalDynamicHosts: 1000,
+		S3BaseURL:            restModel.ToStringPtr("new_s3_base_url"),
 	}
 	updatedSettings := restModel.APIAdminSettings{
 		Banner:     &newBanner,
 		Expansions: newExpansions,
-		HostInit:   &newHostinit,
+		HostInit:   &newHostInit,
 	}
 	oldSettings, err = evergreen.GetConfig()
 	s.NoError(err)
 	_, err = s.ctx.SetEvergreenSettings(&updatedSettings, oldSettings, u, true)
 	s.NoError(err)
 	settingsFromConnector, err = s.ctx.GetEvergreenSettings()
-	s.NoError(err)
+	s.Require().NoError(err)
 	// new values should be set
 	s.EqualValues(newBanner, settingsFromConnector.Banner)
 	s.EqualValues(newExpansions, settingsFromConnector.Expansions)
-	s.EqualValues(newHostinit, settingsFromConnector.HostInit)
+	s.EqualValues(newHostInit.HostThrottle, settingsFromConnector.HostInit.HostThrottle)
+	s.EqualValues(newHostInit.ProvisioningThrottle, settingsFromConnector.HostInit.ProvisioningThrottle)
+	s.EqualValues(newHostInit.CloudStatusBatchSize, settingsFromConnector.HostInit.CloudStatusBatchSize)
+	s.EqualValues(newHostInit.MaxTotalDynamicHosts, settingsFromConnector.HostInit.MaxTotalDynamicHosts)
+	s.EqualValues(restModel.FromStringPtr(newHostInit.S3BaseURL), settingsFromConnector.HostInit.S3BaseURL)
 	// old values should still be there
 	s.EqualValues(testSettings.ServiceFlags, settingsFromConnector.ServiceFlags)
 	s.EqualValues(evergreen.Important, testSettings.BannerTheme)

diff --git a/rest/model/admin.go b/rest/model/admin.go
@@ -828,10 +828,11 @@ type APIBanner struct {
 }
 
 type APIHostInitConfig struct {
-	HostThrottle         int `json:"host_throttle"`
-	ProvisioningThrottle int `json:"provisioning_throttle"`
-	CloudStatusBatchSize int `json:"cloud_batch_size"`
-	MaxTotalDynamicHosts int `json:"max_total_dynamic_hosts"`
+	HostThrottle         int     `json:"host_throttle"`
+	ProvisioningThrottle int     `json:"provisioning_throttle"`
+	CloudStatusBatchSize int     `json:"cloud_batch_size"`
+	MaxTotalDynamicHosts int     `json:"max_total_dynamic_hosts"`
+	S3BaseURL            *string `json:"s3_base_url"`
 }
 
 func (a *APIHostInitConfig) BuildFromService(h interface{}) error {
@@ -841,6 +842,7 @@ func (a *APIHostInitConfig) BuildFromService(h interface{}) error {
 		a.ProvisioningThrottle = v.ProvisioningThrottle
 		a.CloudStatusBatchSize = v.CloudStatusBatchSize
 		a.MaxTotalDynamicHosts = v.MaxTotalDynamicHosts
+		a.S3BaseURL = ToStringPtr(v.S3BaseURL)
 	default:
 		return errors.Errorf("%T is not a supported type", h)
 	}
@@ -853,6 +855,7 @@ func (a *APIHostInitConfig) ToService() (interface{}, error) {
 		ProvisioningThrottle: a.ProvisioningThrottle,
 		CloudStatusBatchSize: a.CloudStatusBatchSize,
 		MaxTotalDynamicHosts: a.MaxTotalDynamicHosts,
+		S3BaseURL:            FromStringPtr(a.S3BaseURL),
 	}, nil
 }
 

diff --git a/rest/model/admin_test.go b/rest/model/admin_test.go
@@ -59,11 +59,6 @@ func TestModelConversion(t *testing.T) {
 	assert.EqualValues(testSettings.BannerTheme, *apiSettings.BannerTheme)
 	assert.Equal(testSettings.ClientBinariesDir, *apiSettings.ClientBinariesDir)
 	assert.Equal(testSettings.ConfigDir, *apiSettings.ConfigDir)
-	assert.Equal(testSettings.HostJasper.BinaryName, FromStringPtr(apiSettings.HostJasper.BinaryName))
-	assert.Equal(testSettings.HostJasper.DownloadFileName, FromStringPtr(apiSettings.HostJasper.DownloadFileName))
-	assert.Equal(testSettings.HostJasper.Port, apiSettings.HostJasper.Port)
-	assert.Equal(testSettings.HostJasper.URL, FromStringPtr(apiSettings.HostJasper.URL))
-	assert.Equal(testSettings.HostJasper.Version, FromStringPtr(apiSettings.HostJasper.Version))
 	assert.Equal(testSettings.GithubPRCreatorOrg, *apiSettings.GithubPRCreatorOrg)
 	assert.Equal(testSettings.LogPath, *apiSettings.LogPath)
 	assert.Equal(testSettings.PprofPort, *apiSettings.PprofPort)
@@ -120,8 +115,16 @@ func TestModelConversion(t *testing.T) {
 	assert.EqualValues(testSettings.ContainerPools.Pools[0].Id, FromStringPtr(apiSettings.ContainerPools.Pools[0].Id))
 	assert.EqualValues(testSettings.ContainerPools.Pools[0].MaxContainers, apiSettings.ContainerPools.Pools[0].MaxContainers)
 	assert.EqualValues(testSettings.ContainerPools.Pools[0].Port, apiSettings.ContainerPools.Pools[0].Port)
-	assert.EqualValues(testSettings.HostJasper.URL, FromStringPtr(apiSettings.HostJasper.URL))
+	assert.Equal(testSettings.HostJasper.BinaryName, FromStringPtr(apiSettings.HostJasper.BinaryName))
+	assert.Equal(testSettings.HostJasper.DownloadFileName, FromStringPtr(apiSettings.HostJasper.DownloadFileName))
+	assert.Equal(testSettings.HostJasper.Port, apiSettings.HostJasper.Port)
+	assert.Equal(testSettings.HostJasper.URL, FromStringPtr(apiSettings.HostJasper.URL))
+	assert.Equal(testSettings.HostJasper.Version, FromStringPtr(apiSettings.HostJasper.Version))
 	assert.EqualValues(testSettings.HostInit.HostThrottle, apiSettings.HostInit.HostThrottle)
+	assert.EqualValues(testSettings.HostInit.ProvisioningThrottle, apiSettings.HostInit.ProvisioningThrottle)
+	assert.EqualValues(testSettings.HostInit.CloudStatusBatchSize, apiSettings.HostInit.CloudStatusBatchSize)
+	assert.EqualValues(testSettings.HostInit.MaxTotalDynamicHosts, apiSettings.HostInit.MaxTotalDynamicHosts)
+	assert.EqualValues(testSettings.HostInit.S3BaseURL, FromStringPtr(apiSettings.HostInit.S3BaseURL))
 	assert.EqualValues(testSettings.Jira.BasicAuthConfig.Username, FromStringPtr(apiSettings.Jira.BasicAuthConfig.Username))
 	assert.EqualValues(testSettings.LoggerConfig.DefaultLevel, FromStringPtr(apiSettings.LoggerConfig.DefaultLevel))
 	assert.EqualValues(testSettings.LoggerConfig.Buffer.Count, apiSettings.LoggerConfig.Buffer.Count)

diff --git a/service/templates/admin.html b/service/templates/admin.html
@@ -569,6 +569,10 @@ <h2 class="modal-title">[[modalTitle]]</h2>
 										<label>Max Total Dynamic Hosts</label>
 										<input type="number" ng-model="Settings.hostinit.max_total_dynamic_hosts">
 									</md-input-container>
+									<md-input-container class="control" style="width:45%;">
+										<label>S3 Download Base URL</label>
+										<input type="text" ng-model="Settings.hostinit.s3_base_url">
+									</md-input-container>
 								</md-card-content>
 							</md-card>
 

diff --git a/testutil/config.go b/testutil/config.go
@@ -128,7 +128,7 @@ func MockConfig() *evergreen.Settings {
 				ExpireAfterMinutes: 60,
 			},
 			Naive: &evergreen.NaiveAuthConfig{
-				Users: []evergreen.AuthUser{evergreen.AuthUser{Username: "user", Password: "pw"}},
+				Users: []evergreen.AuthUser{{Username: "user", Password: "pw"}},
 			},
 			Github: &evergreen.GithubAuthConfig{
 				ClientId:     "ghclient",
@@ -154,7 +154,7 @@ func MockConfig() *evergreen.Settings {
 		ConfigDir: "cfg_dir",
 		ContainerPools: evergreen.ContainerPoolsConfig{
 			Pools: []evergreen.ContainerPool{
-				evergreen.ContainerPool{
+				{
 					Distro:        "valid-distro",
 					Id:            "test-pool-1",
 					MaxContainers: 100,
@@ -167,7 +167,11 @@ func MockConfig() *evergreen.Settings {
 		Expansions:         map[string]string{"k2": "v2"},
 		GithubPRCreatorOrg: "org",
 		HostInit: evergreen.HostInitConfig{
-			HostThrottle: 64,
+			HostThrottle:         64,
+			ProvisioningThrottle: 100,
+			CloudStatusBatchSize: 10,
+			MaxTotalDynamicHosts: 500,
+			S3BaseURL:            "s3_base_url",
 		},
 		HostJasper: evergreen.HostJasperConfig{
 			BinaryName:       "binary",
@@ -204,7 +208,7 @@ func MockConfig() *evergreen.Settings {
 				AdminEmail: []string{"email"},
 			},
 		},
-		Plugins:   map[string]map[string]interface{}{"k4": map[string]interface{}{"k5": "v5"}},
+		Plugins:   map[string]map[string]interface{}{"k4": {"k5": "v5"}},
 		PprofPort: "port",
 		Providers: evergreen.CloudProviders{
 			AWS: evergreen.AWSConfig{