mongodb extension (#303)

* Implementation of operations to manage MongoDB backups:
- stream-push
- stream-fetch
- oplog-push
- delete
- backup-list

* Added docker configuration and integration tests for push commands

* make subfolder for oplog.bson to restore more comfortable

* integration test for delete command

* Integration tests improvements:
-test oplog deletion
-remove oplog dump dir after test

* move HandleStreamFetch and DownloadAndDecompressStream to internal helper

* descriptions of internal functions

* -race option for build in docker

Makefile

@@ -1,6 +1,7 @@
 MAIN_PG_PATH := main/pg
 MAIN_MYSQL_PATH := main/mysql
 MAIN_REDIS_PATH := main/redis
+MAIN_MONGO_PATH := main/mongo
 DOCKER_COMMON := golang ubuntu s3
 CMD_FILES = $(wildcard wal-g/*.go)
 PKG_FILES = $(wildcard internal/**/*.go internal/**/**/*.go internal/*.go)
@@ -14,7 +15,7 @@ ifdef GOTAGS
 override GOTAGS := -tags $(GOTAGS)
 endif
 
-test: install deps lint unittest pg_build mysql_build redis_build unlink_brotli pg_integration_test mysql_integration_test redis_integration_test
+test: install deps lint unittest pg_build mysql_build redis_build mongo_build unlink_brotli pg_integration_test mysql_integration_test redis_integration_test mongo_integration_test
 
 pg_test: install deps pg_build lint unittest unlink_brotli pg_integration_test
 
@@ -52,6 +53,18 @@ mysql_clean:
 mysql_install: mysql_build
 	mv $(MAIN_MYSQL_PATH)/wal-g $(GOBIN)/wal-g
 
+mongo_test: install deps mongo_build lint unittest unlink_brotli mongo_integration_test
+
+mongo_build: $(CMD_FILES) $(PKG_FILES)
+	(cd $(MAIN_MONGO_PATH) && go build -o wal-g $(GOTAGS) -ldflags "-s -w -X github.com/wal-g/wal-g/cmd.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd.GitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd.WalgVersion=`git tag -l --points-at HEAD`")
+
+mongo_install: mongo_build
+	mv $(MAIN_MONGO_PATH)/wal-g $(GOBIN)/wal-g
+
+mongo_integration_test:
+	docker-compose build $(DOCKER_COMMON) mongo mongo_tests
+	docker-compose up --exit-code-from mongo_tests mongo_tests
+
 redis_test: install deps redis_build lint unittest unlink_brotli redis_integration_test
 
 redis_build: $(CMD_FILES) $(PKG_FILES)

cmd/mongo/backup_list.go

@@ -0,0 +1,27 @@
+package mongo
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/wal-g/wal-g/internal"
+	"github.com/wal-g/wal-g/internal/tracelog"
+)
+
+const BackupListShortDescription = "Prints available backups"
+
+// backupListCmd represents the backupList command
+var backupListCmd = &cobra.Command{
+	Use:   "backup-list",
+	Short: BackupListShortDescription, // TODO : improve description
+	Args:  cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		folder, err := internal.ConfigureFolder()
+		if err != nil {
+			tracelog.ErrorLogger.FatalError(err)
+		}
+		internal.HandleBackupList(folder)
+	},
+}
+
+func init() {
+	MongoCmd.AddCommand(backupListCmd)
+}

cmd/mongo/delete.go

@@ -0,0 +1,71 @@
+package mongo
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/wal-g/wal-g/internal"
+	"github.com/wal-g/wal-g/internal/storages/storage"
+	"github.com/wal-g/wal-g/internal/tracelog"
+	"github.com/wal-g/wal-g/utility"
+)
+
+var confirmed = false
+
+// deleteCmd represents the delete command
+var deleteCmd = &cobra.Command{
+	Use:   "delete",
+	Short: "Clears old backups and oplog",
+}
+
+var deleteBeforeCmd = &cobra.Command{
+	Use:     "before backup_name|timestamp", // TODO : improve description
+	Example: internal.DeleteBeforeExamples,
+	Args:    internal.DeleteBeforeArgsValidator,
+	Run:     runDeleteBefore,
+}
+
+var deleteRetainCmd = &cobra.Command{
+	Use:       "retain backup_count", // TODO : improve description
+	Example:   internal.DeleteRetainExamples,
+	ValidArgs: internal.StringModifiers,
+	Args:      internal.DeleteRetainArgsValidator,
+	Run:       runDeleteRetain,
+}
+
+func runDeleteBefore(cmd *cobra.Command, args []string) {
+	folder, err := internal.ConfigureFolder()
+	if err != nil {
+		tracelog.ErrorLogger.FatalError(err)
+	}
+
+	internal.HandleDeleteBefore(folder, args, confirmed, isFullBackup, GetLessFunc(folder))
+}
+
+func runDeleteRetain(cmd *cobra.Command, args []string) {
+	folder, err := internal.ConfigureFolder()
+	if err != nil {
+		tracelog.ErrorLogger.FatalError(err)
+	}
+
+	internal.HandleDeleteRetain(folder, args, confirmed, isFullBackup, GetLessFunc(folder))
+}
+
+func isFullBackup(object storage.Object) bool {
+	return true
+}
+
+func init() {
+	MongoCmd.AddCommand(deleteCmd)
+	deleteCmd.AddCommand(deleteBeforeCmd, deleteRetainCmd)
+	deleteCmd.PersistentFlags().BoolVar(&confirmed, internal.ConfirmFlag, false, "Confirms backup deletion")
+}
+
+func GetLessFunc(folder storage.Folder) func(object1, object2 storage.Object) bool {
+	return func(object1, object2 storage.Object) bool {
+		time1, ok1 := utility.TryFetchTimeRFC3999(object1.GetName())
+		time2, ok2 := utility.TryFetchTimeRFC3999(object2.GetName())
+		if !ok1 || !ok2 {
+			return object2.GetLastModified().After(object1.GetLastModified())
+		}
+		return time1 < time2
+	}
+}

cmd/mongo/mongo.go

@@ -0,0 +1,37 @@
+package mongo
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/wal-g/wal-g/internal"
+)
+
+var MongoDBShortDescription = "MongoDB backup tool"
+
+// These variables are here only to show current version. They are set in makefile during build process
+var WalgVersion = "devel"
+var GitRevision = "devel"
+var BuildDate = "devel"
+
+var MongoCmd = &cobra.Command{
+	Use:     "wal-g",
+	Short:   MongoDBShortDescription, // TODO : improve description
+	Version: strings.Join([]string{WalgVersion, GitRevision, BuildDate, "MongoDB"}, "\t"),
+}
+
+func Execute() {
+	if err := MongoCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(internal.InitConfig, internal.Configure)
+
+	MongoCmd.PersistentFlags().StringVar(&internal.CfgFile, "config", "", "config file (default is $HOME/.wal-g.yaml)")
+	MongoCmd.InitDefaultVersionFlag()
+}

cmd/mongo/oplog_push.go

@@ -0,0 +1,29 @@
+package mongo
+
+import (
+	"github.com/wal-g/wal-g/internal"
+	"github.com/wal-g/wal-g/internal/databases/mongo"
+	"github.com/wal-g/wal-g/internal/tracelog"
+
+	"github.com/spf13/cobra"
+)
+
+const oplogPushShortDescription = ""
+
+// oplogPushCmd represents the cron command
+var oplogPushCmd = &cobra.Command{
+	Use:   "oplog-push",
+	Short: oplogPushShortDescription,
+	Args:  cobra.NoArgs,
+	Run: func(cmd *cobra.Command, args []string) {
+		uploader, err := internal.ConfigureUploader()
+		if err != nil {
+			tracelog.ErrorLogger.FatalError(err)
+		}
+		mongo.HandleOplogPush(&mongo.Uploader{Uploader: uploader})
+	},
+}
+
+func init() {
+	MongoCmd.AddCommand(oplogPushCmd)
+}

cmd/mongo/stream_fetch.go

@@ -0,0 +1,29 @@
+package mongo
+
+import (
+	"github.com/wal-g/wal-g/internal"
+	"github.com/wal-g/wal-g/internal/databases/mongo"
+	"github.com/wal-g/wal-g/internal/tracelog"
+
+	"github.com/spf13/cobra"
+)
+
+const StreamFetchShortDescription = ""
+
+// streamFetchCmd represents the streamFetch command
+var streamFetchCmd = &cobra.Command{
+	Use:   "stream-fetch backup-name",
+	Short: StreamFetchShortDescription,
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		folder, err := internal.ConfigureFolder()
+		if err != nil {
+			tracelog.ErrorLogger.FatalError(err)
+		}
+		internal.HandleStreamFetch(args[0], folder, mongo.FetchBackupStreamAndOplog)
+	},
+}
+
+func init() {
+	MongoCmd.AddCommand(streamFetchCmd)
+}

cmd/mongo/stream_push.go

@@ -0,0 +1,28 @@
+package mongo
+
+import (
+	"github.com/wal-g/wal-g/internal"
+	"github.com/wal-g/wal-g/internal/databases/mongo"
+	"github.com/wal-g/wal-g/internal/tracelog"
+
+	"github.com/spf13/cobra"
+)
+
+const StreamPushShortDescription = ""
+
+// streamPushCmd represents the streamPush command
+var streamPushCmd = &cobra.Command{
+	Use:   "stream-push",
+	Short: StreamPushShortDescription,
+	Run: func(cmd *cobra.Command, args []string) {
+		uploader, err := internal.ConfigureUploader()
+		if err != nil {
+			tracelog.ErrorLogger.FatalError(err)
+		}
+		mongo.HandleStreamPush(&mongo.Uploader{Uploader: uploader})
+	},
+}
+
+func init() {
+	MongoCmd.AddCommand(streamPushCmd)
+}

cmd/mysql/delete.go

@@ -1,20 +1,17 @@
 package mysql
 
 import (
+	"path"
+	"strings"
+
 	"github.com/spf13/cobra"
 	"github.com/wal-g/wal-g/internal"
 	"github.com/wal-g/wal-g/internal/databases/mysql"
 	"github.com/wal-g/wal-g/internal/storages/storage"
 	"github.com/wal-g/wal-g/internal/tracelog"
 	"github.com/wal-g/wal-g/utility"
-	"path"
-	"regexp"
-	"strings"
 )
 
-var patternTimeRFC3339 = "[0-9]{8}T[0-9]{6}Z"
-var regexpTimeRFC3339 = regexp.MustCompile(patternTimeRFC3339)
-var maxCountOfRFC3339 = 1
 var confirmed = false
 
 // deleteCmd represents the delete command
@@ -72,26 +69,18 @@ func IsFullBackup(folder storage.Folder, object storage.Object) bool {
 
 func GetLessFunc(folder storage.Folder) func(object1, object2 storage.Object) bool {
 	return func(object1, object2 storage.Object) bool {
-		time1, ok := tryFetchTimeRFC3999(object1)
+		time1, ok := utility.TryFetchTimeRFC3999(object1.GetName())
 		if !ok {
 			return binlogLess(folder, object1, object2)
 		}
-		time2, ok := tryFetchTimeRFC3999(object2)
+		time2, ok := utility.TryFetchTimeRFC3999(object2.GetName())
 		if !ok {
 			return binlogLess(folder, object1, object2)
 		}
 		return time1 < time2
 	}
 }
 
-func tryFetchTimeRFC3999(object storage.Object) (string, bool) {
-	found_lsn := regexpTimeRFC3339.FindAllString(object.GetName(), maxCountOfRFC3339)
-	if len(found_lsn) > 0 {
-		return regexpTimeRFC3339.FindAllString(object.GetName(), maxCountOfRFC3339)[0], true
-	}
-	return "", false
-}
-
 func binlogLess(folder storage.Folder, object1, object2 storage.Object) bool {
 	binlogName1, ok := tryFetchBinlogName(folder, object1)
 	if !ok {

docker-compose.yml

@@ -38,6 +38,10 @@ services:
       &&  mkdir -p /export/mysqlfullbucket
       &&  mkdir -p /export/mysqlbinlogpushbucket
       &&  mkdir -p /export/mysqldeleteendtoendbucket
+      &&  mkdir -p /export/mongostreampushbucket
+      &&  mkdir -p /export/mongooplogpushbucket
+      &&  mkdir -p /export/mongodeletebeforebucket
+      &&  mkdir -p /export/mongodeleteretainbucket
       &&  mkdir -p /export/redisbucket
       &&  /usr/bin/minio server /export'
 
@@ -96,6 +100,26 @@ services:
       context: .
     image: wal-g/redis_tests
     container_name: wal-g_redis_tests
+    env_file:
+      - docker/common/common_walg.env
+    depends_on:
+      - s3
+    links:
+      - s3
+
+  mongo:
+    build:
+      dockerfile: docker/mongo/Dockerfile
+      context: .
+    image: wal-g/mongo
+    container_name: wal-g_mongo
+
+  mongo_tests:
+    build:
+      dockerfile: docker/mongo_tests/Dockerfile
+      context: .
+    image: wal-g/mongo_tests
+    container_name: wal-g_mongo_tests
     env_file:
       - docker/common/common_walg.env
     depends_on:

docker/mongo/Dockerfile

@@ -0,0 +1,6 @@
+FROM wal-g/ubuntu:latest
+
+RUN apt-get update -y 
+RUN apt-get install mongodb -y
+
+COPY docker/mongo/mongodb.conf /etc/mongodb.conf

docker/mongo/mongodb.conf

@@ -0,0 +1,17 @@
+# mongodb.conf
+
+# Where to store the data.
+dbpath=/var/lib/mongodb
+
+#where to log
+logpath=/var/log/mongodb/mongodb.log
+
+logappend=true
+
+bind_ip = 127.0.0.1
+#port = 27017
+
+journal=true
+
+master = true
+