Catchup fetch maybe really works

Makefile

@@ -6,7 +6,7 @@ DOCKER_COMMON := golang ubuntu s3
 CMD_FILES = $(wildcard wal-g/*.go)
 PKG_FILES = $(wildcard internal/**/*.go internal/**/**/*.go internal/*.go)
 TEST_FILES = $(wildcard test/*.go testtools/*.go)
-PKG := github.com/wal-g/wal-g
+PKG := wal-g
 COVERAGE_FILE := coverage.out
 TEST := "pg_tests"
 
@@ -17,7 +17,7 @@ test: install deps lint unittest pg_build mysql_build redis_build mongo_build un
 pg_test: install deps pg_build lint unlink_brotli pg_integration_test
 
 pg_build: $(CMD_FILES) $(PKG_FILES)
-	(cd $(MAIN_PG_PATH) && go build -tags "brotli lzo" -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/pg.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/pg.GitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/pg.WalgVersion=`git tag -l --points-at HEAD`")
+	(cd $(MAIN_PG_PATH) && go build -mod vendor -tags "brotli lzo" -o wal-g -ldflags "-s -w -X wal-g/cmd/pg.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X wal-g/cmd/pg.GitRevision=`git rev-parse --short HEAD` -X wal-g/cmd/pg.WalgVersion=`git tag -l --points-at HEAD`")
 
 install_and_build_pg: install deps pg_build
 
@@ -59,7 +59,7 @@ pg_install: pg_build
 mysql_test: install deps mysql_build lint unlink_brotli mysql_integration_test
 
 mysql_build: $(CMD_FILES) $(PKG_FILES)
-	(cd $(MAIN_MYSQL_PATH) && go build -tags brotli -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/mysql.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/mysql.GitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/mysql.WalgVersion=`git tag -l --points-at HEAD`")
+	(cd $(MAIN_MYSQL_PATH) && go build -mod vendor -tags brotli -o wal-g -ldflags "-s -w -X wal-g/cmd/mysql.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X wal-g/cmd/mysql.GitRevision=`git rev-parse --short HEAD` -X wal-g/cmd/mysql.WalgVersion=`git tag -l --points-at HEAD`")
 
 load_docker_common:
 	@if [ "x" = "${CACHE_FILE_UBUNTU}x" ]; then\
@@ -84,7 +84,7 @@ mysql_install: mysql_build
 mongo_test: install deps mongo_build lint unlink_brotli
 
 mongo_build: $(CMD_FILES) $(PKG_FILES)
-	(cd $(MAIN_MONGO_PATH) && go build -tags brotli -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/mongo.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/mongo.GitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/mongo.WalgVersion=`git tag -l --points-at HEAD`")
+	(cd $(MAIN_MONGO_PATH) && go build -mod vendor -tags brotli -o wal-g -ldflags "-s -w -X wal-g/cmd/mongo.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X wal-g/cmd/mongo.GitRevision=`git rev-parse --short HEAD` -X wal-g/cmd/mongo.WalgVersion=`git tag -l --points-at HEAD`")
 
 mongo_install: mongo_build
 	mv $(MAIN_MONGO_PATH)/wal-g $(GOBIN)/wal-g
@@ -100,7 +100,7 @@ mongo_features: install deps mongo_build lint unlink_brotli
 redis_test: install deps redis_build lint unlink_brotli redis_integration_test
 
 redis_build: $(CMD_FILES) $(PKG_FILES)
-	(cd $(MAIN_REDIS_PATH) && go build -tags brotli -o wal-g -ldflags "-s -w -X github.com/wal-g/wal-g/cmd/redis.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X github.com/wal-g/wal-g/cmd/redis.GitRevision=`git rev-parse --short HEAD` -X github.com/wal-g/wal-g/cmd/redis.WalgVersion=`git tag -l --points-at HEAD`")
+	(cd $(MAIN_REDIS_PATH) && go build -mod vendor -tags brotli -o wal-g -ldflags "-s -w -X wal-g/cmd/redis.BuildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X wal-g/cmd/redis.GitRevision=`git rev-parse --short HEAD` -X wal-g/cmd/redis.WalgVersion=`git tag -l --points-at HEAD`")
 
 redis_integration_test: load_docker_common
 	docker-compose build redis redis_tests
@@ -135,12 +135,11 @@ lint: $(CMD_FILES) $(PKG_FILES) $(TEST_FILES)
 
 deps:
 	git submodule update --init
-	dep ensure
+	go mod vendor
 	sed -i 's|\(#cgo LDFLAGS:\) .*|\1 -Wl,-Bstatic -llzo2 -Wl,-Bdynamic|' vendor/github.com/cyberdelia/lzo/lzo.go
 	./link_brotli.sh
 
 install:
-	go get -u github.com/golang/dep/cmd/dep
 	go get -u golang.org/x/lint/golint
 	go get -u github.com/DATA-DOG/godog/cmd/godog
 

cmd/mongo/backup_fetch.go

@@ -1,10 +1,12 @@
 package mongo
 
 import (
-	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
 	"os"
+
+	"wal-g/internal"
+
+	"github.com/spf13/cobra"
+	"github.com/tinsane/tracelog"
 )
 
 const BackupFetchShortDescription = "Fetches desired backup from storage"
@@ -17,11 +19,11 @@ var backupFetchCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		folder, err := internal.ConfigureFolder()
 		tracelog.ErrorLogger.FatalOnError(err)
-		tracelog.ErrorLogger.FatalfOnError("Failed to parse until timestamp ", err, )
+		tracelog.ErrorLogger.FatalfOnError("Failed to parse until timestamp ", err)
 		internal.HandleBackupFetch(folder, args[0], internal.GetStreamFetcher(os.Stdout))
 	},
 }
 
 func init() {
 	Cmd.AddCommand(backupFetchCmd)
-}
+}

cmd/mongo/backup_list.go

@@ -1,9 +1,10 @@
 package mongo
 
 import (
+	"wal-g/internal"
+
 	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
+	"github.com/tinsane/tracelog"
 )
 
 const BackupListShortDescription = "Prints available backups"

cmd/mongo/backup_push.go

@@ -1,9 +1,10 @@
 package mongo
 
 import (
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
-	"github.com/wal-g/wal-g/internal/databases/mongo"
+	"wal-g/internal"
+	"wal-g/internal/databases/mongo"
+
+	"github.com/tinsane/tracelog"
 
 	"github.com/spf13/cobra"
 )

cmd/mongo/delete.go

@@ -1,11 +1,12 @@
 package mongo
 
 import (
+	"wal-g/internal"
+	"wal-g/utility"
+
 	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
+	"github.com/tinsane/tracelog"
 	"github.com/wal-g/storages/storage"
-	"github.com/wal-g/wal-g/internal"
-	"github.com/wal-g/wal-g/utility"
 )
 
 var confirmed = false

cmd/mongo/mongo.go

@@ -5,8 +5,9 @@ import (
 	"os"
 	"strings"
 
+	"wal-g/internal"
+
 	"github.com/spf13/cobra"
-	"github.com/wal-g/wal-g/internal"
 )
 
 var DBShortDescription = "MongoDB backup tool"

cmd/mongo/oplog_fetch.go

@@ -1,14 +1,15 @@
 package mongo
 
 import (
+	"wal-g/internal"
+	"wal-g/internal/databases/mongo"
+
 	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
-	"github.com/wal-g/wal-g/internal/databases/mongo"
+	"github.com/tinsane/tracelog"
 )
 
 const OplogFetchShortDescription = "Fetches OpLogs from storage"
-const sinceFlagShortDescription  = "backup name starting from which you want to take binlog"
+const sinceFlagShortDescription = "backup name starting from which you want to take binlog"
 
 var backupName string
 
@@ -20,12 +21,12 @@ var oplogFetchCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		folder, err := internal.ConfigureFolder()
 		tracelog.ErrorLogger.FatalOnError(err)
-		tracelog.ErrorLogger.FatalfOnError("Failed to parse until timestamp ", err, )
-		tracelog.ErrorLogger.FatalOnError(mongo.HandleOplogFetch(folder, backupName, ))
+		tracelog.ErrorLogger.FatalfOnError("Failed to parse until timestamp ", err)
+		tracelog.ErrorLogger.FatalOnError(mongo.HandleOplogFetch(folder, backupName))
 	},
 }
 
 func init() {
 	oplogFetchCmd.PersistentFlags().StringVar(&backupName, "since", "LATEST", sinceFlagShortDescription)
 	Cmd.AddCommand(oplogFetchCmd)
-}
+}

cmd/mongo/oplog_push.go

@@ -5,11 +5,12 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"wal-g/internal"
+	"wal-g/internal/databases/mongo"
+
+	"github.com/tinsane/tracelog"
 
 	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
-	"github.com/wal-g/wal-g/internal/databases/mongo"
 )
 
 const oplogPushShortDescription = ""

cmd/mysql/backup_fetch.go

@@ -1,16 +1,18 @@
 package mysql
 
 import (
-	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
 	"os"
+
+	"wal-g/internal"
+
+	"github.com/spf13/cobra"
+	"github.com/tinsane/tracelog"
 )
 
 const BackupFetchShortDescription = "Fetches desired backup from storage"
 
 // backupFetchCmd represents the streamFetch command
-var backupFetchCmd = &cobra.Command {
+var backupFetchCmd = &cobra.Command{
 	Use:   "backup-fetch backup-name",
 	Short: BackupFetchShortDescription,
 	Args:  cobra.ExactArgs(1),
@@ -24,4 +26,4 @@ var backupFetchCmd = &cobra.Command {
 
 func init() {
 	Cmd.AddCommand(backupFetchCmd)
-}
+}

cmd/mysql/backup_list.go

@@ -1,9 +1,10 @@
 package mysql
 
 import (
+	"wal-g/internal"
+
 	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
+	"github.com/tinsane/tracelog"
 )
 
 const BackupListShortDescription = "Prints available backups"

cmd/mysql/backup_push.go

@@ -1,10 +1,11 @@
 package mysql
 
 import (
+	"wal-g/internal"
+	"wal-g/internal/databases/mysql"
+
 	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
-	"github.com/wal-g/wal-g/internal/databases/mysql"
+	"github.com/tinsane/tracelog"
 )
 
 const StreamPushShortDescription = ""

cmd/mysql/binlog_fetch.go

@@ -1,11 +1,13 @@
 package mysql
 
 import (
-	"github.com/spf13/cobra"
-	"github.com/wal-g/tracelog"
-	"github.com/wal-g/wal-g/internal"
-	"github.com/wal-g/wal-g/internal/databases/mysql"
 	"time"
+
+	"wal-g/internal"
+	"wal-g/internal/databases/mysql"
+
+	"github.com/spf13/cobra"
+	"github.com/tinsane/tracelog"
 )
 
 const binlogFetchShortDescription = "fetches binlog from storage"
@@ -35,4 +37,4 @@ func init() {
 	binlogFetchCmd.PersistentFlags().StringVar(&untilDt, "until", time.Now().Format(time.RFC3339), untilFlagShortDescription)
 	binlogFetchCmd.PersistentFlags().BoolVar(&apply, "apply", false, applyFlagShortDescription)
 	Cmd.AddCommand(binlogFetchCmd)
-}
+}