feat(Dgraph): Online restores allows to restore a specific backup.

dgraph/Makefile

@@ -34,6 +34,7 @@ GOPATH        ?= $(shell go env GOPATH)
 MODIFIED = $(shell git diff-index --quiet HEAD || echo "-mod")
 
 export GO111MODULE := on
+export CGO_ENABLED=0
 
 # Build-time Go variables
 dgraphVersion   = github.com/dgraph-io/dgraph/x.dgraphVersion
@@ -42,7 +43,7 @@ gitBranch       = github.com/dgraph-io/dgraph/x.gitBranch
 lastCommitSHA   = github.com/dgraph-io/dgraph/x.lastCommitSHA
 lastCommitTime  = github.com/dgraph-io/dgraph/x.lastCommitTime
 
-BUILD_FLAGS   ?= -ldflags '-X ${lastCommitSHA}=${BUILD} -X "${lastCommitTime}=${BUILD_DATE}" -X "${dgraphVersion}=${BUILD_VERSION}" -X "${dgraphCodename}=${BUILD_CODENAME}${MODIFIED}" -X ${gitBranch}=${BUILD_BRANCH}'
+BUILD_FLAGS   ?= -ldflags '-X ${lastCommitSHA}=${BUILD} -X "${lastCommitTime}=${BUILD_DATE}" -X "${dgraphVersion}=${BUILD_VERSION}" -X "${dgraphCodename}=${BUILD_CODENAME}${MODIFIED}" -X ${gitBranch}=${BUILD_BRANCH}' -tags netgo
 
 # Insert build tags if specified
 ifneq ($(strip $(BUILD_TAGS)),)

graphql/admin/endpoints_ee.go

@@ -69,6 +69,12 @@ const adminTypes = `
 		"""
 		backupId: String
 
+		"""
+		Number of the backup within the backup series to be restored. Backups with a greater value
+		will be ignored. If the value is missing, the entire series will be restored.
+		"""
+		backupNum: Int
+
 		"""
 		Path to the key file needed to decrypt the backup. This file should be accessible
 		by all alphas in the group. The backup will be written using the encryption key
@@ -114,17 +120,17 @@ const adminTypes = `
 
 		"""
 		Secret key credential for the destination.
-		"""		
+		"""
 		secretKey: String
 
 		"""
 		AWS session token, if required.
-		"""	
+		"""
 		sessionToken: String
 
 		"""
 		Set to true to allow backing up to S3 or Minio bucket that requires no credentials.
-		"""	
+		"""
 		anonymous: Boolean
 	}
 

graphql/admin/restore.go

@@ -24,11 +24,13 @@ import (
 	"github.com/dgraph-io/dgraph/graphql/schema"
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/worker"
+	"github.com/pkg/errors"
 )
 
 type restoreInput struct {
 	Location          string
 	BackupId          string
+	BackupNum         int
 	EncryptionKeyFile string
 	AccessKey         string
 	SecretKey         string
@@ -51,6 +53,7 @@ func resolveRestore(ctx context.Context, m schema.Mutation) (*resolve.Resolved,
 	req := pb.RestoreRequest{
 		Location:          input.Location,
 		BackupId:          input.BackupId,
+		BackupNum:         uint64(input.BackupNum),
 		EncryptionKeyFile: input.EncryptionKeyFile,
 		AccessKey:         input.AccessKey,
 		SecretKey:         input.SecretKey,
@@ -93,6 +96,13 @@ func getRestoreInput(m schema.Mutation) (*restoreInput, error) {
 	}
 
 	var input restoreInput
-	err = json.Unmarshal(inputByts, &input)
-	return &input, schema.GQLWrapf(err, "couldn't get input argument")
+	if err := json.Unmarshal(inputByts, &input); err != nil {
+		return nil, schema.GQLWrapf(err, "couldn't get input argument")
+	}
+
+	if input.BackupNum < 0 {
+		err := errors.Errorf("backupNum value should be equal or greater than zero")
+		return nil, schema.GQLWrapf(err, "couldn't get input argument")
+	}
+	return &input, nil
 }

protos/pb.proto

@@ -287,6 +287,8 @@ message RestoreRequest {
 	string vault_path = 13;
 	string vault_field = 14;
 	string vault_format = 15;
+
+	uint64 backup_num = 16;
 }
 
 message Proposal {

systest/online-restore/docker-compose.yml

@@ -25,6 +25,7 @@ services:
     command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=1 --encryption_key_file /data/keys/enc_key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --badger.compression_level=0
   alpha2:
     image: dgraph/dgraph:latest
     container_name: alpha2
@@ -52,6 +53,7 @@ services:
     command: /gobin/dgraph alpha -o 102 --my=alpha2:7182 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=2 --encryption_key_file /data/keys/enc_key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --badger.compression_level=0
   alpha3:
     image: dgraph/dgraph:latest
     container_name: alpha3
@@ -79,6 +81,7 @@ services:
     command: /gobin/dgraph alpha -o 103 --my=alpha3:7183 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=3 --encryption_key_file /data/keys/enc_key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --badger.compression_level=0
   alpha4:
     image: dgraph/dgraph:latest
     container_name: alpha4
@@ -106,6 +109,7 @@ services:
     command: /gobin/dgraph alpha -o 104 --my=alpha4:7184 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=4 --encryption_key_file /data/keys/enc_key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --badger.compression_level=0
   alpha5:
     image: dgraph/dgraph:latest
     container_name: alpha5
@@ -133,6 +137,7 @@ services:
     command: /gobin/dgraph alpha -o 105 --my=alpha5:7185 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=5 --encryption_key_file /data/keys/enc_key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --badger.compression_level=0
   alpha6:
     image: dgraph/dgraph:latest
     container_name: alpha6
@@ -160,6 +165,7 @@ services:
     command: /gobin/dgraph alpha -o 106 --my=alpha6:7186 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=6 --encryption_key_file /data/keys/enc_key
       --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --badger.compression_level=0
   ratel:
     image: dgraph/dgraph:latest
     container_name: ratel
@@ -182,4 +188,5 @@ services:
       read_only: true
     command: /gobin/dgraph zero -o 100 --idx=1 --my=zero1:5180 --replicas=3 --logtostderr
       -v=2 --bindall
+      --badger.compression_level=0
 volumes: {}

systest/online-restore/online_restore_test.go

@@ -37,15 +37,15 @@ import (
 	"github.com/dgraph-io/dgraph/testutil"
 )
 
-func sendRestoreRequest(t *testing.T, backupId string) int {
+func sendRestoreRequest(t *testing.T, backupId string, backupNum int) int {
 	restoreRequest := fmt.Sprintf(`mutation restore() {
-		 restore(input: {location: "/data/backup", backupId: "%s",
+		 restore(input: {location: "/data/backup", backupId: "%s", backupNum: %d,
 		 	encryptionKeyFile: "/data/keys/enc_key"}) {
 			code
 			message
 			restoreId
 		}
-	}`, backupId)
+	}`, backupId, backupNum)
 
 	adminUrl := "http://localhost:8180/admin"
 	params := testutil.GraphQLParams{
@@ -171,7 +171,12 @@ func runQueries(t *testing.T, dg *dgo.Dgraph, shouldFail bool) {
 
 		resp, err := dg.NewTxn().Query(context.Background(), bodies[0])
 		if shouldFail {
-			require.Error(t, err)
+			if err != nil {
+				continue
+			}
+			t.Logf(string(resp.GetJson()))
+			t.Logf(string(bodies[1]))
+			require.False(t, testutil.EqualJSON(t, bodies[1], string(resp.GetJson()), "", true))
 		} else {
 			require.NoError(t, err)
 			require.True(t, testutil.EqualJSON(t, bodies[1], string(resp.GetJson()), "", true))
@@ -226,12 +231,88 @@ func TestBasicRestore(t *testing.T) {
 	ctx := context.Background()
 	require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true}))
 
-	restoreId := sendRestoreRequest(t, "youthful_rhodes3")
+	restoreId := sendRestoreRequest(t, "youthful_rhodes3", 0)
 	waitForRestore(t, restoreId, dg)
 	runQueries(t, dg, false)
 	runMutations(t, dg)
 }
 
+func TestRestoreBackupNum(t *testing.T) {
+	disableDraining(t)
+
+	conn, err := grpc.Dial(testutil.SockAddr, grpc.WithInsecure())
+	require.NoError(t, err)
+	dg := dgo.NewDgraphClient(api.NewDgraphClient(conn))
+
+	ctx := context.Background()
+	require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true}))
+	runQueries(t, dg, true)
+
+	restoreId := sendRestoreRequest(t, "youthful_rhodes3", 1)
+	waitForRestore(t, restoreId, dg)
+	runQueries(t, dg, true)
+	runMutations(t, dg)
+}
+
+func TestRestoreBackupNumInvalid(t *testing.T) {
+	disableDraining(t)
+
+	conn, err := grpc.Dial(testutil.SockAddr, grpc.WithInsecure())
+	require.NoError(t, err)
+	dg := dgo.NewDgraphClient(api.NewDgraphClient(conn))
+
+	ctx := context.Background()
+	require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true}))
+	runQueries(t, dg, true)
+
+	// Send a request with a backupNum greater than the number of manifests.
+	adminUrl := "http://localhost:8180/admin"
+	restoreRequest := fmt.Sprintf(`mutation restore() {
+		 restore(input: {location: "/data/backup", backupId: "%s", backupNum: %d,
+		 	encryptionKeyFile: "/data/keys/enc_key"}) {
+			code
+			message
+			restoreId
+		}
+	}`, "youthful_rhodes3", 1000)
+
+	params := testutil.GraphQLParams{
+		Query: restoreRequest,
+	}
+	b, err := json.Marshal(params)
+	require.NoError(t, err)
+
+	resp, err := http.Post(adminUrl, "application/json", bytes.NewBuffer(b))
+	require.NoError(t, err)
+	buf, err := ioutil.ReadAll(resp.Body)
+	bufString := string(buf)
+	require.NoError(t, err)
+	require.Contains(t, bufString, "not enough backups")
+
+	// Send a request with a negative backupNum value.
+	restoreRequest = fmt.Sprintf(`mutation restore() {
+		 restore(input: {location: "/data/backup", backupId: "%s", backupNum: %d,
+		 	encryptionKeyFile: "/data/keys/enc_key"}) {
+			code
+			message
+			restoreId
+		}
+	}`, "youthful_rhodes3", -1)
+
+	params = testutil.GraphQLParams{
+		Query: restoreRequest,
+	}
+	b, err = json.Marshal(params)
+	require.NoError(t, err)
+
+	resp, err = http.Post(adminUrl, "application/json", bytes.NewBuffer(b))
+	require.NoError(t, err)
+	buf, err = ioutil.ReadAll(resp.Body)
+	bufString = string(buf)
+	require.NoError(t, err)
+	require.Contains(t, bufString, "backupNum value should be equal or greater than zero")
+}
+
 func TestMoveTablets(t *testing.T) {
 	disableDraining(t)
 
@@ -242,13 +323,13 @@ func TestMoveTablets(t *testing.T) {
 	ctx := context.Background()
 	require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true}))
 
-	restoreId := sendRestoreRequest(t, "youthful_rhodes3")
+	restoreId := sendRestoreRequest(t, "youthful_rhodes3", 0)
 	waitForRestore(t, restoreId, dg)
 	runQueries(t, dg, false)
 
 	// Send another restore request with a different backup. This backup has some of the
 	// same predicates as the previous one but they are stored in different groups.
-	restoreId = sendRestoreRequest(t, "blissful_hermann1")
+	restoreId = sendRestoreRequest(t, "blissful_hermann1", 0)
 	waitForRestore(t, restoreId, dg)
 
 	resp, err := dg.NewTxn().Query(context.Background(), `{
@@ -304,7 +385,7 @@ func TestListBackups(t *testing.T) {
 			encrypted
 			groups {
 				groupId
-				predicates    
+				predicates
 			}
 			path
 			since

testutil/json.go

@@ -76,8 +76,10 @@ func DiffJSONMaps(t *testing.T, wantMap, gotMap map[string]interface{},
 		if err != nil {
 			t.Error("Could not marshal JSON:", err)
 		}
-		t.Errorf("Expected JSON and actual JSON differ:\n%s",
-			sdiffJSON(wantBuf, gotBuf, savepath, quiet))
+		if !quiet {
+			t.Errorf("Expected JSON and actual JSON differ:\n%s",
+				sdiffJSON(wantBuf, gotBuf, savepath, quiet))
+		}
 		return false
 	}