Make backup-restore an open source feature (#8067)

Open-source the backup and restore feature.

Author: ahsanbarkati

ee/backup/run.go renamed to backup/run.go

@@ -1,13 +1,17 @@
-// +build !oss
-
 /*
- * Copyright 2018 Dgraph Labs, Inc. and Contributors
+ * Copyright 2021 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Licensed under the Dgraph Community License (the "License"); you
- * may not use this file except in compliance with the License. You
- * may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- *     https://github.com/dgraph-io/dgraph/blob/master/licenses/DCL.txt
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package backup

dgraph/cmd/root.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/dgraph-io/dgraph/backup"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/alpha"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/bulk"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/cert"
@@ -41,6 +42,7 @@ import (
 	"github.com/dgraph-io/dgraph/dgraph/cmd/migrate"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/version"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/zero"
+	"github.com/dgraph-io/dgraph/updatemanifest"
 	"github.com/dgraph-io/dgraph/upgrade"
 	"github.com/dgraph-io/dgraph/x"
 
@@ -83,9 +85,10 @@ var rootConf = viper.New()
 
 // subcommands initially contains all default sub-commands.
 var subcommands = []*x.SubCommand{
-	&bulk.Bulk, &cert.Cert, &conv.Conv, &live.Live, &alpha.Alpha, &zero.Zero, &version.Version,
-	&debug.Debug, &migrate.Migrate, &debuginfo.DebugInfo, &upgrade.Upgrade, &decrypt.Decrypt,
-	&increment.Increment,
+	&bulk.Bulk, &backup.LsBackup, &backup.ExportBackup, &cert.Cert, &conv.Conv, &live.Live,
+	&alpha.Alpha, &zero.Zero, &version.Version, &debug.Debug, &migrate.Migrate,
+	&debuginfo.DebugInfo, &upgrade.Upgrade, &decrypt.Decrypt, &increment.Increment,
+	&updatemanifest.UpdateManifest,
 }
 
 func initCmds() {

dgraph/cmd/root_ee.go

@@ -15,17 +15,12 @@ package cmd
 import (
 	acl "github.com/dgraph-io/dgraph/ee/acl"
 	"github.com/dgraph-io/dgraph/ee/audit"
-	"github.com/dgraph-io/dgraph/ee/backup"
-	"github.com/dgraph-io/dgraph/ee/updatemanifest"
 )
 
 func init() {
 	// subcommands already has the default subcommands, we append to EE ones to that.
 	subcommands = append(subcommands,
-		&backup.LsBackup,
-		&backup.ExportBackup,
 		&acl.CmdAcl,
 		&audit.CmdAudit,
-		&updatemanifest.UpdateManifest,
 	)
 }

graphql/admin/admin.go

@@ -412,6 +412,224 @@ const (
 		response: AssignedIds
 	}
 
+	input BackupInput {
+
+		"""
+		Destination for the backup: e.g. Minio or S3 bucket.
+		"""
+		destination: String!
+
+		"""
+		Access key credential for the destination.
+		"""
+		accessKey: String
+
+		"""
+		Secret key credential for the destination.
+		"""
+		secretKey: String
+
+		"""
+		AWS session token, if required.
+		"""
+		sessionToken: String
+
+		"""
+		Set to true to allow backing up to S3 or Minio bucket that requires no credentials.
+		"""
+		anonymous: Boolean
+
+		"""
+		Force a full backup instead of an incremental backup.
+		"""
+		forceFull: Boolean
+	}
+
+	type BackupPayload {
+		response: Response
+		taskId: String
+	}
+
+	input RestoreInput {
+
+		"""
+		Destination for the backup: e.g. Minio or S3 bucket.
+		"""
+		location: String!
+
+		"""
+		Backup ID of the backup series to restore. This ID is included in the manifest.json file.
+		If missing, it defaults to the latest series.
+		"""
+		backupId: String
+
+		"""
+		Number of the backup within the backup series to be restored. Backups with a greater value
+		will be ignored. If the value is zero or missing, the entire series will be restored.
+		"""
+		backupNum: Int
+
+		"""
+		All the backups with num >= incrementalFrom will be restored.
+		"""
+		incrementalFrom: Int
+
+		"""
+		If isPartial is set to true then the cluster will be kept in draining mode after
+		restore. This makes sure that the db is not corrupted by any mutations or tablet moves in
+		between two restores.
+		"""
+		isPartial: Boolean
+
+		"""
+		Path to the key file needed to decrypt the backup. This file should be accessible
+		by all alphas in the group. The backup will be written using the encryption key
+		with which the cluster was started, which might be different than this key.
+		"""
+		encryptionKeyFile: String
+
+		"""
+		Vault server address where the key is stored. This server must be accessible
+		by all alphas in the group. Default "http://localhost:8200".
+		"""
+		vaultAddr: String
+
+		"""
+		Path to the Vault RoleID file.
+		"""
+		vaultRoleIDFile: String
+
+		"""
+		Path to the Vault SecretID file.
+		"""
+		vaultSecretIDFile: String
+
+		"""
+		Vault kv store path where the key lives. Default "secret/data/dgraph".
+		"""
+		vaultPath: String
+
+		"""
+		Vault kv store field whose value is the key. Default "enc_key".
+		"""
+		vaultField: String
+
+		"""
+		Vault kv store field's format. Must be "base64" or "raw". Default "base64".
+		"""
+		vaultFormat: String
+
+		"""
+		Access key credential for the destination.
+		"""
+		accessKey: String
+
+		"""
+		Secret key credential for the destination.
+		"""
+		secretKey: String
+
+		"""
+		AWS session token, if required.
+		"""
+		sessionToken: String
+
+		"""
+		Set to true to allow backing up to S3 or Minio bucket that requires no credentials.
+		"""
+		anonymous: Boolean
+	}
+
+	type RestorePayload {
+		"""
+		A short string indicating whether the restore operation was successfully scheduled.
+		"""
+		code: String
+
+		"""
+		Includes the error message if the operation failed.
+		"""
+		message: String
+	}
+
+	input ListBackupsInput {
+		"""
+		Destination for the backup: e.g. Minio or S3 bucket.
+		"""
+		location: String!
+
+		"""
+		Access key credential for the destination.
+		"""
+		accessKey: String
+
+		"""
+		Secret key credential for the destination.
+		"""
+		secretKey: String
+
+		"""
+		AWS session token, if required.
+		"""
+		sessionToken: String
+
+		"""
+		Whether the destination doesn't require credentials (e.g. S3 public bucket).
+		"""
+		anonymous: Boolean
+
+	}
+
+	type BackupGroup {
+		"""
+		The ID of the cluster group.
+		"""
+		groupId: UInt64
+
+		"""
+		List of predicates assigned to the group.
+		"""
+		predicates: [String]
+	}
+
+	type Manifest {
+		"""
+		Unique ID for the backup series.
+		"""
+		backupId: String
+
+		"""
+		Number of this backup within the backup series. The full backup always has a value of one.
+		"""
+		backupNum: UInt64
+
+		"""
+		Whether this backup was encrypted.
+		"""
+		encrypted: Boolean
+
+		"""
+		List of groups and the predicates they store in this backup.
+		"""
+		groups: [BackupGroup]
+
+		"""
+		Path to the manifest file.
+		"""
+		path: String
+
+		"""
+		The timestamp at which this backup was taken. The next incremental backup will
+		start from this timestamp.
+		"""
+		since: UInt64
+
+		"""
+		The type of backup, either full or incremental.
+		"""
+		type: String
+	}
+
 	` + adminTypes + `
 
 	type Query {
@@ -421,6 +639,10 @@ const (
 		state: MembershipState
 		config: Config
 		task(input: TaskInput!): TaskPayload
+		"""
+		Get the information about the backups at a given location.
+		"""
+		listBackups(input: ListBackupsInput!) : [Manifest]
 		` + adminQueries + `
 	}
 
@@ -474,6 +696,16 @@ const (
 		"""
 		assign(input: AssignInput!): AssignPayload
 
+		"""
+		Start a binary backup.
+		"""
+		backup(input: BackupInput!) : BackupPayload
+
+		"""
+		Start restoring a binary backup.
+		"""
+		restore(input: RestoreInput!) : RestorePayload
+
 		` + adminMutations + `
 	}
  `

graphql/admin/backup.go

@@ -34,12 +34,7 @@ type backupInput struct {
 }
 
 func resolveBackup(ctx context.Context, m schema.Mutation) (*resolve.Resolved, bool) {
-	glog.Info("Got backup request")
-	if !worker.EnterpriseEnabled() {
-		err := fmt.Errorf("you must enable enterprise features first. " +
-			"Supply the appropriate license file to Dgraph Zero using the HTTP endpoint.")
-		return resolve.EmptyResult(m, err), false
-	}
+	glog.Info("Got a backup request")
 
 	input, err := getBackupInput(m)
 	if err != nil {