Merge pull request #143 from jllucas/backup_restore

QED on-demand backup

api/apihttp/apihttp.go

@@ -441,9 +441,10 @@ func LogHandler(handle http.Handler) http.HandlerFunc {
 // }
 func InfoShardsHandler(balloon raftwal.RaftBalloonApi) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != "GET" {
-			w.Header().Set("Allow", "GET")
-			w.WriteHeader(http.StatusMethodNotAllowed)
+		var err error
+		// Make sure we can only be called with an HTTP GET request.
+		w, _, err = GetReqSanitizer(w, r)
+		if err != nil {
 			return
 		}
 
@@ -497,3 +498,14 @@ func PostReqSanitizer(w http.ResponseWriter, r *http.Request) (http.ResponseWrit
 
 	return w, r, nil
 }
+
+// GetReqSanitizer function checks that certain request info exists and it is correct.
+func GetReqSanitizer(w http.ResponseWriter, r *http.Request) (http.ResponseWriter, *http.Request, error) {
+	if r.Method != "GET" {
+		w.Header().Set("Allow", "GET")
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return w, r, errors.New("Method not allowed.")
+	}
+
+	return w, r, nil
+}

api/apihttp/apihttp_test.go

@@ -26,15 +26,17 @@ import (
 	"testing"
 	"time"
 
+	assert "github.com/stretchr/testify/require"
+
 	"github.com/bbva/qed/balloon"
 	"github.com/bbva/qed/balloon/history"
 	"github.com/bbva/qed/balloon/hyper"
 	"github.com/bbva/qed/crypto/hashing"
 	"github.com/bbva/qed/protocol"
 	"github.com/bbva/qed/raftwal"
+	"github.com/bbva/qed/storage"
 	"github.com/bbva/qed/testutils/rand"
 	storage_utils "github.com/bbva/qed/testutils/storage"
-	assert "github.com/stretchr/testify/require"
 )
 
 type fakeRaftBalloon struct {
@@ -141,6 +143,14 @@ func (b fakeRaftBalloon) Info() map[string]interface{} {
 	return make(map[string]interface{})
 }
 
+func (b fakeRaftBalloon) Backup() error {
+	return nil
+}
+
+func (b fakeRaftBalloon) ListBackups() []*storage.BackupInfo {
+	return nil
+}
+
 func TestHealthCheckHandler(t *testing.T) {
 	// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
 	// pass 'nil' as the third parameter.

api/mgmthttp/mgmthttp.go

@@ -27,13 +27,15 @@ import (
 
 // NewMgmtHttp will return a mux server with the endpoint required to
 // join the raft cluster.
-func NewMgmtHttp(raftBalloon raftwal.RaftBalloonApi) *http.ServeMux {
+func NewMgmtHttp(balloon raftwal.RaftBalloonApi) *http.ServeMux {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/join", joinHandle(raftBalloon))
+	mux.HandleFunc("/join", joinHandle(balloon))
+	mux.HandleFunc("/backup", backupHandle(balloon))
+	mux.HandleFunc("/backups", listBackupsHandle(balloon))
 	return mux
 }
 
-func joinHandle(raftBalloon raftwal.RaftBalloonApi) http.HandlerFunc {
+func joinHandle(api raftwal.RaftBalloonApi) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var err error
 		// Make sure we can only be called with an HTTP POST request.
@@ -77,11 +79,56 @@ func joinHandle(raftBalloon raftwal.RaftBalloonApi) http.HandlerFunc {
 			metadata[k] = v.(string)
 		}
 
-		if err := raftBalloon.Join(nodeID, remoteAddr, metadata); err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
+		if err := api.Join(nodeID, remoteAddr, metadata); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
 		w.WriteHeader(http.StatusOK)
 	}
 }
+
+func backupHandle(api raftwal.RaftBalloonApi) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		// Make sure we can only be called with an HTTP POST request.
+		w, _, err = apihttp.PostReqSanitizer(w, r)
+		if err != nil {
+			return
+		}
+
+		if err := api.Backup(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func listBackupsHandle(api raftwal.RaftBalloonApi) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		// Make sure we can only be called with an HTTP GET request.
+		w, _, err = apihttp.GetReqSanitizer(w, r)
+		if err != nil {
+			return
+		}
+
+		backups := api.ListBackups()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		out, err := json.Marshal(backups)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(out)
+	}
+}

cmd/backup.go

@@ -0,0 +1,81 @@
+/*
+   Copyright 2018-2019 Banco Bilbao Vizcaya Argentaria, S.A.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/octago/sflags/gen/gpflag"
+	"github.com/spf13/cobra"
+
+	"github.com/bbva/qed/log"
+)
+
+type BackupConfig struct {
+	// Endpoint [host:port] to ask for QED management APIs.
+	Endpoint string `desc:"QED Log service management endpoint http://ip:port"`
+
+	// Log level
+	Log string `desc:"Set log level to info, error or debug"`
+
+	// ApiKey to query the server endpoint.
+	APIKey string `desc:"Set API Key to talk to QED Log service"`
+}
+
+func defaultConfig() *BackupConfig {
+	return &BackupConfig{
+		Endpoint: "http://127.0.0.1:8700",
+		APIKey:   "my-key",
+	}
+}
+
+var backupCmd *cobra.Command = &cobra.Command{
+	Use:               "backup",
+	Short:             "Manages QED log backups",
+	TraverseChildren:  true,
+	PersistentPreRunE: runBackup,
+}
+
+var backupCtx context.Context
+
+func init() {
+	backupCtx = configBackup()
+	Root.AddCommand(backupCmd)
+}
+
+func configBackup() context.Context {
+
+	conf := defaultConfig()
+
+	err := gpflag.ParseTo(conf, backupCmd.PersistentFlags())
+	if err != nil {
+		log.Fatalf("err: %v", err)
+	}
+	return context.WithValue(Ctx, k("backup.config"), conf)
+}
+
+func runBackup(cmd *cobra.Command, args []string) error {
+	var err error
+
+	endpoint, _ := cmd.Flags().GetString("endpoint")
+	err = urlParse(endpoint)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/backup_create.go

@@ -0,0 +1,89 @@
+/*
+   Copyright 2018-2019 Banco Bilbao Vizcaya Argentaria, S.A.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/spf13/cobra"
+
+	"github.com/bbva/qed/log"
+)
+
+var backupCreateCmd *cobra.Command = &cobra.Command{
+	Use:   "create",
+	Short: "Create a backup of the QED Log",
+	RunE:  runBackupCreate,
+}
+
+var backupCreateCtx context.Context
+
+func init() {
+	backupCmd.AddCommand(backupCreateCmd)
+}
+
+func runBackupCreate(cmd *cobra.Command, args []string) error {
+
+	config := backupCtx.Value(k("backup.config")).(*BackupConfig)
+	log.SetLogger("backup", config.Log)
+
+	_, err := createBackup(config)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("Backup created!")
+
+	return nil
+}
+
+func createBackup(config *BackupConfig) ([]byte, error) {
+
+	// Build request
+	req, err := http.NewRequest("POST", config.Endpoint+"/backup", nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Api-Key", config.APIKey)
+
+	// Get response
+	client := http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Infof("Request error: %v\n", err)
+		return nil, err
+	}
+
+	var bodyBytes []byte
+	if resp.Body != nil {
+		defer resp.Body.Close()
+		bodyBytes, err = ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if resp.StatusCode >= 400 && resp.StatusCode < 500 {
+		return nil, fmt.Errorf("Invalid request %v", string(bodyBytes))
+	}
+
+	return bodyBytes, nil
+}