Revert "Merge branch 'master' of https://github.com/wal-g/wal-g into …

…feature-upload-time"

This reverts commit 6d99b51, reversing
changes made to 32788bc.

.github/workflows/golangci-lint.yml

@@ -22,7 +22,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
         id: go
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.5.0
+        uses: golangci/golangci-lint-action@v3.4.0
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
           version: v1.52.2

cmd/common/st/put_object.go

@@ -24,14 +24,13 @@ var putObjectCmd = &cobra.Command{
 	Short: putObjectShortDescription,
 	Args:  cobra.ExactArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
+		uploader, err := internal.ConfigureUploader()
+		tracelog.ErrorLogger.FatalOnError(err)
+
 		localPath := args[0]
 		dstPath := args[1]
 
-		err := multistorage.ExecuteOnStorage(targetStorage, func(folder storage.Folder) error {
-			uploader, err := internal.ConfigureUploaderToFolder(folder)
-			if err != nil {
-				return err
-			}
+		err = multistorage.ExecuteOnStorage(targetStorage, func(folder storage.Folder) error {
 			return storagetools.HandlePutObject(localPath, dstPath, uploader, overwrite, !noEncrypt, !noCompress)
 		})
 		tracelog.ErrorLogger.FatalOnError(err)

docker-compose.yml

@@ -99,8 +99,6 @@ services:
       &&  mkdir -p /export/gpdeltabackuptestbucket
       &&  mkdir -p /export/createrestorepointbucket
       &&  mkdir -p /export/storagetoolsbucket
-      &&  mkdir -p /export/sttransferbucket
-      &&  mkdir -p /export/sttransferfailoverbucket
       &&  mkdir -p /export/walrestorebucket
       && mkdir -p /export/daemonbucket
       &&  /usr/bin/minio server /export'

docker/st_tests/Dockerfile

@@ -31,4 +31,4 @@ COPY --from=build /go/src/github.com/wal-g/wal-g/main/pg/wal-g /usr/bin
 
 COPY docker/st_tests/scripts/ /tmp
 
-CMD /tmp/run_integration_tests.sh
+CMD /tmp/tests/storage_tool_tests.sh

docs/MongoDB.md

@@ -20,10 +20,6 @@ to STDIN and push it to MongoDB instance. Required for restore procedure.
 
 URI used to connect to a MongoDB instance. Required for backup and oplog archiving procedure.
 
-* `MONGODB_RESTORE_DISABLE_HOST_RESETUP`
-
-Do not perform any MongoDB reconfiguration steps during `binary-backup-fetch`. Usefull when one might want just to restore original host state.
-
 * `OPLOG_ARCHIVE_AFTER_SIZE`
 
 Oplog archive batch in bytes which triggers upload to storage.

docs/PostgreSQL.md

@@ -85,8 +85,7 @@ If the parameter value is NOMETADATA or not specified, it will fallback to defau
 To control how frequently WAL-G will check if Postgres is alive during the backup-push. If the check fails, backup-push terminates.
 
 Examples:
-- `0` - disable the alive checks
-- `1m` - check every 1 minute (default value)
+- `0` - disable the alive checks (default value)
 - `10s` - check every 10 seconds
 - `10m` - check every 10 minutes
 

docs/StorageTools.md

@@ -56,51 +56,3 @@ Flags:
 Example:
 
 ``wal-g st put path/to/local_file path/to/remote_file`` upload the local file to the storage.
-
-### `transfer`
-Transfer all files from one configured storage to another. Is usually used to move files from a failover storage to the primary one when it becomes alive.
-
-Args:
-
-1. Path to the directory in both storages, where files should be moved to/from. Files from all subdirectories are also moved.
-
-Flags:
-
-1. Add `-s (--source)` to specify the source storage name to take files from. To specify the primary storage, use `default`. This flag is required.
-
-2. Add `-t (--target)` to specify the target storage name to save files to. The primary storage is used by default.
-
-3. Add `-o (--overwrite)` to move files and overwrite them, even if they already existed in the target storage.
-
-   Files existing in both storages will remain as they are if this flag isn't specified.
-
-   Please note that the files are checked for their existence in the target storage only once at the very beginning. So if a new file appear in the target storage while the command is working, it may be overwritten even when `--overwrite` isn't specified.
-
-4. Add `--fail-fast` so that the command stops after the first error occurs with transferring any file. 
-
-   Without this flag the command will try to move every file.
-
-   Regardless of the flag, the command will end with zero error code only if all the files have moved successfully.
-
-   Keep in mind that files aren't transferred atomically. This means that when this flag is set, an error occured with one file may interrupt transferring other files in the middle, so they may already be copied to the target storage, but not yet deleted from the source. 
-
-5. Add `-c (--concurrency)` to set the max number of concurrent workers that will move files.
-
-6. Add `-m (--max)` to set the max number of files to move in a single command run.
-
-7. Add `--appearance-checks` to set the max number of checks for files to appear in the target storage, which will be performed after moving the file and before deleting it.
-
-   This option is recommended for use with storages that don't guarantee the read-after-write consistency. 
-   Otherwise, transferring files between them may cause a moment of time, when a file doesn't exist in both storages, which may lead to problems with restoring backups at that moment.
-
-8. Add `--appearance-checks-interval` to specify the min time interval between checks of the same file to appear in the target storage.
-
-   The duration must be specified in the golang `time.Duration` [format](https://pkg.go.dev/time#ParseDuration).
-
-Examples:
-
-``wal-g st transfer / --source='my_failover_ssh'``
-
-``wal-g st transfer folder/single_file.json --source='default' --target='my_failover_ssh' --overwrite``
-
-``wal-g st transfer basebackups_005/ --source='my_failover_s3' --target='default' --fail-fast -c=50 -m=10000 --appearance-checks=5 --appearance-checks-interval=1s``

internal/config.go

@@ -98,21 +98,20 @@ const (
 	ProfileMode          = "PROFILE_MODE"
 	ProfilePath          = "PROFILE_PATH"
 
-	MongoDBUriSetting                = "MONGODB_URI"
-	MongoDBLastWriteUpdateInterval   = "MONGODB_LAST_WRITE_UPDATE_INTERVAL"
-	MongoDBRestoreDisableHostResetup = "MONGODB_RESTORE_DISABLE_HOST_RESETUP"
-	OplogArchiveAfterSize            = "OPLOG_ARCHIVE_AFTER_SIZE"
-	OplogArchiveTimeoutInterval      = "OPLOG_ARCHIVE_TIMEOUT_INTERVAL"
-	OplogPITRDiscoveryInterval       = "OPLOG_PITR_DISCOVERY_INTERVAL"
-	OplogPushStatsEnabled            = "OPLOG_PUSH_STATS_ENABLED"
-	OplogPushStatsLoggingInterval    = "OPLOG_PUSH_STATS_LOGGING_INTERVAL"
-	OplogPushStatsUpdateInterval     = "OPLOG_PUSH_STATS_UPDATE_INTERVAL"
-	OplogPushStatsExposeHTTP         = "OPLOG_PUSH_STATS_EXPOSE_HTTP"
-	OplogPushWaitForBecomePrimary    = "OPLOG_PUSH_WAIT_FOR_BECOME_PRIMARY"
-	OplogPushPrimaryCheckInterval    = "OPLOG_PUSH_PRIMARY_CHECK_INTERVAL"
-	OplogReplayOplogAlwaysUpsert     = "OPLOG_REPLAY_OPLOG_ALWAYS_UPSERT"
-	OplogReplayOplogApplicationMode  = "OPLOG_REPLAY_OPLOG_APPLICATION_MODE"
-	OplogReplayIgnoreErrorCodes      = "OPLOG_REPLAY_IGNORE_ERROR_CODES"
+	MongoDBUriSetting               = "MONGODB_URI"
+	MongoDBLastWriteUpdateInterval  = "MONGODB_LAST_WRITE_UPDATE_INTERVAL"
+	OplogArchiveAfterSize           = "OPLOG_ARCHIVE_AFTER_SIZE"
+	OplogArchiveTimeoutInterval     = "OPLOG_ARCHIVE_TIMEOUT_INTERVAL"
+	OplogPITRDiscoveryInterval      = "OPLOG_PITR_DISCOVERY_INTERVAL"
+	OplogPushStatsEnabled           = "OPLOG_PUSH_STATS_ENABLED"
+	OplogPushStatsLoggingInterval   = "OPLOG_PUSH_STATS_LOGGING_INTERVAL"
+	OplogPushStatsUpdateInterval    = "OPLOG_PUSH_STATS_UPDATE_INTERVAL"
+	OplogPushStatsExposeHTTP        = "OPLOG_PUSH_STATS_EXPOSE_HTTP"
+	OplogPushWaitForBecomePrimary   = "OPLOG_PUSH_WAIT_FOR_BECOME_PRIMARY"
+	OplogPushPrimaryCheckInterval   = "OPLOG_PUSH_PRIMARY_CHECK_INTERVAL"
+	OplogReplayOplogAlwaysUpsert    = "OPLOG_REPLAY_OPLOG_ALWAYS_UPSERT"
+	OplogReplayOplogApplicationMode = "OPLOG_REPLAY_OPLOG_APPLICATION_MODE"
+	OplogReplayIgnoreErrorCodes     = "OPLOG_REPLAY_IGNORE_ERROR_CODES"
 
 	MysqlDatasourceNameSetting     = "WALG_MYSQL_DATASOURCE_NAME"
 	MysqlSslCaSetting              = "WALG_MYSQL_SSL_CA"
@@ -241,9 +240,8 @@ var (
 	}
 
 	PGDefaultSettings = map[string]string{
-		PgWalSize:            "16",
-		PgBackRestStanza:     "main",
-		PgAliveCheckInterval: "1m",
+		PgWalSize:        "16",
+		PgBackRestStanza: "main",
 	}
 
 	GPDefaultSettings = map[string]string{