EVG-7106: backup collection job (#2997)

command/s3_put.go

@@ -399,7 +399,7 @@ func (s3pc *s3put) createPailBucket(httpClient *http.Client) error {
 		Credentials: pail.CreateAWSCredentials(s3pc.AwsKey, s3pc.AwsSecret, ""),
 		Region:      endpoints.UsEast1RegionID,
 		Name:        s3pc.Bucket,
-		Permission:  s3pc.Permissions,
+		Permissions: pail.S3Permissions(s3pc.Permissions),
 		ContentType: s3pc.ContentType,
 	}
 	bucket, err := pail.NewS3MultiPartBucketWithHTTPClient(httpClient, opts)

config.go

@@ -54,6 +54,7 @@ type Settings struct {
 	Banner                  string                    `bson:"banner" json:"banner" yaml:"banner"`
 	BannerTheme             BannerTheme               `bson:"banner_theme" json:"banner_theme" yaml:"banner_theme"`
 	Bugsnag                 string                    `yaml:"bugsnag" bson:"bugsnag" json:"bugsnag"`
+	Backup                  BackupConfig              `bson:"backup" json:"backup" yaml:"backup"`
 	ClientBinariesDir       string                    `yaml:"client_binaries_dir" bson:"client_binaries_dir" json:"client_binaries_dir"`
 	CommitQueue             CommitQueueConfig         `yaml:"commit_queue" bson:"commit_queue" json:"commit_queue" id:"commit_queue"`
 	ConfigDir               string                    `yaml:"configdir" bson:"configdir" json:"configdir"`

config_backup.go

@@ -0,0 +1,64 @@
+package evergreen
+
+import (
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type BackupConfig struct {
+	BucketName string `bson:"bucket_name" json:"bucket_name" yaml:"bucket_name"`
+	Key        string `bson:"key" json:"key" yaml:"key"`
+	Secret     string `bson:"secret" json:"secret" yaml:"secret"`
+	Prefix     string `bson:"prefix" json:"prefix" yaml:"prefix"`
+	Compress   bool   `bson:"compress" json:"compress" yaml:"compress"`
+}
+
+func (c *BackupConfig) SectionId() string         { return "backup" }
+func (c *BackupConfig) ValidateAndDefault() error { return nil }
+
+func (c *BackupConfig) Populated() bool {
+	return c.BucketName != "" && c.Prefix != ""
+}
+
+func (c *BackupConfig) Set() error {
+	env := GetEnvironment()
+	ctx, cancel := env.Context()
+	defer cancel()
+	coll := env.DB().Collection(ConfigCollection)
+
+	_, err := coll.UpdateOne(ctx, byId(c.SectionId()), bson.M{
+		"$set": bson.M{
+			"bucket_name": c.BucketName,
+			"key":         c.Key,
+			"secret":      c.Secret,
+			"compress":    c.Compress,
+			"prefix":      c.Prefix,
+		},
+	}, options.Update().SetUpsert(true))
+
+	return errors.Wrapf(err, "error updating section %s", c.SectionId())
+}
+
+func (c *BackupConfig) Get(env Environment) error {
+	ctx, cancel := env.Context()
+	defer cancel()
+	coll := env.DB().Collection(ConfigCollection)
+
+	res := coll.FindOne(ctx, byId(c.SectionId()))
+	if err := res.Err(); err != nil {
+		return errors.Wrapf(err, "error retrieving section %s", c.SectionId())
+	}
+
+	if err := res.Decode(c); err != nil {
+		if err == mongo.ErrNoDocuments {
+			*c = BackupConfig{}
+			return nil
+		}
+
+		return errors.Wrap(err, "problem decoding result")
+	}
+
+	return nil
+}

config_db.go

@@ -56,6 +56,7 @@ var (
 	containerPoolsKey     = bsonutil.MustHaveTag(Settings{}, "ContainerPools")
 	commitQueueKey        = bsonutil.MustHaveTag(Settings{}, "CommitQueue")
 	ldapRoleMapKey        = bsonutil.MustHaveTag(Settings{}, "LDAPRoleMap")
+	backupConfig          = bsonutil.MustHaveTag(Settings{}, "Backup")
 
 	// degraded mode flags
 	taskDispatchKey                 = bsonutil.MustHaveTag(ServiceFlags{}, "TaskDispatchDisabled")
@@ -82,6 +83,7 @@ var (
 	commitQueueDisabledKey          = bsonutil.MustHaveTag(ServiceFlags{}, "CommitQueueDisabled")
 	plannerDisabledKey              = bsonutil.MustHaveTag(ServiceFlags{}, "PlannerDisabled")
 	hostAllocatorDisabledKey        = bsonutil.MustHaveTag(ServiceFlags{}, "HostAllocatorDisabled")
+	drBackupDisabledKey             = bsonutil.MustHaveTag(ServiceFlags{}, "DRBackupDisabled")
 
 	// ContainerPoolsConfig keys
 	poolsKey = bsonutil.MustHaveTag(ContainerPoolsConfig{}, "Pools")

config_serviceflags.go

@@ -27,6 +27,7 @@ type ServiceFlags struct {
 	CommitQueueDisabled        bool `bson:"commit_queue_disabled" json:"commit_queue_disabled"`
 	PlannerDisabled            bool `bson:"planner_disabled" json:"planner_disabled"`
 	HostAllocatorDisabled      bool `bson:"host_allocator_disabled" json:"host_allocator_disabled"`
+	DRBackupDisabled           bool `bson:"dr_backup_disabled" json:"dr_backup_disabled"`
 
 	// Notification Flags
 	EventProcessingDisabled      bool `bson:"event_processing_disabled" json:"event_processing_disabled"`
@@ -91,6 +92,7 @@ func (c *ServiceFlags) Set() error {
 			commitQueueDisabledKey:          c.CommitQueueDisabled,
 			plannerDisabledKey:              c.PlannerDisabled,
 			hostAllocatorDisabledKey:        c.HostAllocatorDisabled,
+			drBackupDisabledKey:             c.DRBackupDisabled,
 		},
 	}, options.Update().SetUpsert(true))
 

db/db_utils.go

@@ -309,7 +309,7 @@ func WriteGridFile(fsPrefix, name string, source io.Reader) error {
 	defer cancel()
 	bucket, err := pail.NewGridFSBucketWithClient(ctx, env.Client(), pail.GridFSOptions{
 		Database: env.DB().Name(),
-		Prefix:   fsPrefix,
+		Name:     fsPrefix,
 	})
 
 	if err != nil {
@@ -325,7 +325,7 @@ func GetGridFile(fsPrefix, name string) (io.ReadCloser, error) {
 	defer cancel()
 	bucket, err := pail.NewGridFSBucketWithClient(ctx, env.Client(), pail.GridFSOptions{
 		Database: env.DB().Name(),
-		Prefix:   fsPrefix,
+		Name:     fsPrefix,
 	})
 
 	if err != nil {

model/stats/db.go

@@ -80,10 +80,10 @@ import (
 )
 
 const (
-	hourlyTestStatsCollection  = "hourly_test_stats"
-	dailyTestStatsCollection   = "daily_test_stats"
+	HourlyTestStatsCollection  = "hourly_test_stats"
+	DailyTestStatsCollection   = "daily_test_stats"
 	DailyTaskStatsCollection   = "daily_task_stats"
-	dailyStatsStatusCollection = "daily_stats_status"
+	DailyStatsStatusCollection = "daily_stats_status"
 	bulkSize                   = 1000
 	nsInASecond                = time.Second / time.Nanosecond
 )
@@ -186,7 +186,7 @@ func hourlyTestStatsForOldTasksPipeline(projectId string, requester string, star
 	// And the merge the documents with the existing ones.
 	mergePipeline := []bson.M{
 		{"$lookup": bson.M{
-			"from":         hourlyTestStatsCollection,
+			"from":         HourlyTestStatsCollection,
 			"localField":   dbTestStatsIdKey,
 			"foreignField": dbTestStatsIdKey,
 			"as":           "existing",
@@ -1117,7 +1117,7 @@ func makeSum(condition bson.M) bson.M {
 
 func GetDailyTestDoc(id DbTestStatsId) (*dbTestStats, error) {
 	doc := dbTestStats{}
-	err := db.FindOne(dailyTestStatsCollection, bson.M{"_id": id}, db.NoProjection, db.NoSort, &doc)
+	err := db.FindOne(DailyTestStatsCollection, bson.M{"_id": id}, db.NoProjection, db.NoSort, &doc)
 	if adb.ResultsNotFound(err) {
 		return nil, nil
 	}
@@ -1126,7 +1126,7 @@ func GetDailyTestDoc(id DbTestStatsId) (*dbTestStats, error) {
 
 func GetHourlyTestDoc(id DbTestStatsId) (*dbTestStats, error) {
 	doc := dbTestStats{}
-	err := db.FindOne(hourlyTestStatsCollection, bson.M{"_id": id}, db.NoProjection, db.NoSort, &doc)
+	err := db.FindOne(HourlyTestStatsCollection, bson.M{"_id": id}, db.NoProjection, db.NoSort, &doc)
 	if adb.ResultsNotFound(err) {
 		return nil, nil
 	}

model/stats/query.go

@@ -263,7 +263,7 @@ func GetTestStats(filter StatsFilter) ([]TestStats, error) {
 	}
 	var stats []TestStats
 	pipeline := filter.testStatsQueryPipeline()
-	err = db.Aggregate(dailyTestStatsCollection, pipeline, &stats)
+	err = db.Aggregate(DailyTestStatsCollection, pipeline, &stats)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to aggregate test statistics")
 	}

model/stats/query_test.go

@@ -28,7 +28,7 @@ func TestStatsQuerySuite(t *testing.T) {
 }
 
 func (s *statsQuerySuite) SetupTest() {
-	s.clearCollection(dailyTestStatsCollection)
+	s.clearCollection(DailyTestStatsCollection)
 	s.clearCollection(DailyTaskStatsCollection)
 
 	s.baseTestFilter = StatsFilter{
@@ -948,7 +948,7 @@ func (s *statsQuerySuite) clearCollection(name string) {
 
 func (s *statsQuerySuite) insertDailyTestStats(project string, requester string, testFile string, taskName string, variant string, distro string, date time.Time, numPass int, numFail int, avgDuration float64) {
 
-	err := db.Insert(dailyTestStatsCollection, bson.M{
+	err := db.Insert(DailyTestStatsCollection, bson.M{
 		"_id": DbTestStatsId{
 			Project:      project,
 			Requester:    requester,

model/stats/stats.go

@@ -51,7 +51,7 @@ func createDefaultStatsStatus(projectId string) StatsStatus {
 func GetStatsStatus(projectId string) (StatsStatus, error) {
 	status := StatsStatus{}
 	query := statsStatusQuery(projectId)
-	err := db.FindOne(dailyStatsStatusCollection, query, db.NoProjection, db.NoSort, &status)
+	err := db.FindOne(DailyStatsStatusCollection, query, db.NoProjection, db.NoSort, &status)
 	if adb.ResultsNotFound(err) {
 		return createDefaultStatsStatus(projectId), nil
 	}
@@ -69,7 +69,7 @@ func UpdateStatsStatus(projectId string, lastJobRun time.Time, processedTasksUnt
 		ProcessedTasksUntil: processedTasksUntil,
 		Runtime:             runtime,
 	}
-	_, err := db.Upsert(dailyStatsStatusCollection, bson.M{"_id": projectId}, status)
+	_, err := db.Upsert(DailyStatsStatusCollection, bson.M{"_id": projectId}, status)
 	if err != nil {
 		return errors.Wrap(err, "Failed to update test stats status")
 	}
@@ -104,7 +104,7 @@ func GenerateHourlyTestStats(ctx context.Context, opts GenerateOptions) error {
 	end := start.Add(time.Hour)
 	// Generate the stats based on tasks.
 	pipeline := hourlyTestStatsPipeline(opts.ProjectID, opts.Requester, start, end, opts.Tasks, opts.Runtime)
-	err := aggregateIntoCollection(ctx, task.Collection, pipeline, hourlyTestStatsCollection)
+	err := aggregateIntoCollection(ctx, task.Collection, pipeline, HourlyTestStatsCollection)
 	if err != nil {
 		return errors.Wrap(err, "Failed to generate hourly stats")
 	}
@@ -119,7 +119,7 @@ func GenerateHourlyTestStats(ctx context.Context, opts GenerateOptions) error {
 		})
 		// Generate/Update the stats for old tasks.
 		pipeline = hourlyTestStatsForOldTasksPipeline(opts.ProjectID, opts.Requester, start, end, opts.Tasks, opts.Runtime)
-		err = aggregateIntoCollection(ctx, task.OldCollection, pipeline, hourlyTestStatsCollection)
+		err = aggregateIntoCollection(ctx, task.OldCollection, pipeline, HourlyTestStatsCollection)
 		if err != nil {
 			return errors.Wrap(err, "Failed to generate hourly stats for old tasks")
 		}
@@ -141,7 +141,7 @@ func GenerateDailyTestStatsFromHourly(ctx context.Context, opts GenerateOptions)
 	start := util.GetUTCDay(opts.Window)
 	end := start.Add(24 * time.Hour)
 	pipeline := dailyTestStatsFromHourlyPipeline(opts.ProjectID, opts.Requester, start, end, opts.Tasks, opts.Runtime)
-	err := aggregateIntoCollection(ctx, hourlyTestStatsCollection, pipeline, dailyTestStatsCollection)
+	err := aggregateIntoCollection(ctx, HourlyTestStatsCollection, pipeline, DailyTestStatsCollection)
 	if err != nil {
 		return errors.Wrap(err, "Failed to aggregate hourly stats into daily stats")
 	}

model/stats/stats_test.go

@@ -38,9 +38,9 @@ func TestStatsSuite(t *testing.T) {
 
 func (s *statsSuite) SetupTest() {
 	collectionsToClear := []string{
-		hourlyTestStatsCollection,
-		dailyTestStatsCollection,
-		dailyStatsStatusCollection,
+		HourlyTestStatsCollection,
+		DailyTestStatsCollection,
+		DailyStatsStatusCollection,
 		DailyTaskStatsCollection,
 		task.Collection,
 		task.OldCollection,
@@ -615,7 +615,7 @@ func (s *statsSuite) initHourly() {
 
 func (s *statsSuite) insertHourlyTestStats(project string, requester string, testFile string, taskName string, variant string, distro string, date time.Time, numPass int, numFail int, avgDuration float64, lastID mgobson.ObjectId) {
 
-	err := db.Insert(hourlyTestStatsCollection, bson.M{
+	err := db.Insert(HourlyTestStatsCollection, bson.M{
 		"_id": DbTestStatsId{
 			Project:      project,
 			Requester:    requester,
@@ -866,11 +866,11 @@ func (s *statsSuite) countDocs(collection string) int {
 }
 
 func (s *statsSuite) countDailyTestDocs() int {
-	return s.countDocs(dailyTestStatsCollection)
+	return s.countDocs(DailyTestStatsCollection)
 }
 
 func (s *statsSuite) countHourlyTestDocs() int {
-	return s.countDocs(hourlyTestStatsCollection)
+	return s.countDocs(HourlyTestStatsCollection)
 }
 
 func (s *statsSuite) countDailyTaskDocs() int {
@@ -926,7 +926,7 @@ func (s *statsSuite) getLastHourlyTestStat(testStatsID DbTestStatsId) (*dbTestSt
 			"$lt":  end,
 		},
 	}
-	err := db.FindAll(hourlyTestStatsCollection, qry, db.NoProjection, []string{"-last_id"}, db.NoSkip, 1, &testResults)
+	err := db.FindAll(HourlyTestStatsCollection, qry, db.NoProjection, []string{"-last_id"}, db.NoSkip, 1, &testResults)
 	if adb.ResultsNotFound(err) {
 		return nil, nil
 	}

operations/agent.go

@@ -107,7 +107,7 @@ func Agent() cli.Command {
 					Credentials: pail.CreateAWSCredentials(os.Getenv("S3_KEY"), os.Getenv("S3_SECRET"), ""),
 					Region:      endpoints.UsEast1RegionID,
 					Name:        os.Getenv("S3_BUCKET"),
-					Permission:  "public-read",
+					Permissions: pail.S3PermissionsPublicRead,
 					ContentType: "text/plain",
 				},
 			}