feat: Add simple data import and export (#1630)

## Relevant issue(s)

Resolves #1544

## Description

This PR adds import and export functionality to the http api and cli. It
can export to json to reduce the potential file size. At this stage csv
output was not implemented as it would require extensive type casting
(everything in csv is a string) on both writing and reading from the
csv.

README.md

@@ -31,6 +31,8 @@ Read the documentation on [docs.source.network](https://docs.source.network/).
   - [Collection subscription example](#collection-subscription-example)
   - [Replicator example](#replicator-example)
 - [Securing the HTTP API with TLS](#securing-the-http-api-with-tls)
+- [Supporting CORS](#supporting-cors)
+- [Backing up and restoring](#backing-up-and-restoring)
 - [Licensing](#licensing)
 - [Contributors](#contributors)
 
@@ -387,6 +389,25 @@ defradb start --allowed-origins=http://localhost:3000
 
 The catch-all `*` is also a valid origin. 
 
+## Backing up and restoring
+
+It is currently not possible to do a full backup of DefraDB that includes the history of changes through the Merkle DAG. However, DefraDB currently supports a simple backup of the current data state in JSON format that can be used to seed a database or help with transitioning from one DefraDB version to another.
+
+To backup the data, run the following command:
+```shell
+defradb client backup export path/to/backup.json
+```
+
+To pretty print the JSON content when exporting, run the following command:
+```shell
+defradb client backup export --pretty path/to/backup.json
+```
+
+To restore the data, run the following command:
+```shell
+defradb client backup import path/to/backup.json
+```
+
 ## Community
 
 Discuss on [Discord](https://discord.source.network/) or [Github Discussions](https://github.com/sourcenetwork/defradb/discussions). The Source project is on [Twitter](https://twitter.com/sourcenetwrk).

api/http/errors.go

@@ -36,6 +36,8 @@ var (
 	ErrPeerIdUnavailable    = errors.New("no PeerID available. P2P might be disabled")
 	ErrStreamingUnsupported = errors.New("streaming unsupported")
 	ErrNoEmail              = errors.New("email address must be specified for tls with autocert")
+	ErrPayloadFormat        = errors.New("invalid payload format")
+	ErrMissingNewKey        = errors.New("missing _newKey for imported doc")
 )
 
 // ErrorResponse is the GQL top level object holding error items for the response payload.

api/http/handlerfuncs.go

@@ -53,7 +53,7 @@ func pingHandler(rw http.ResponseWriter, req *http.Request) {
 	sendJSON(
 		req.Context(),
 		rw,
-		simpleDataResponse("response", "pong", "test"),
+		simpleDataResponse("response", "pong"),
 		http.StatusOK,
 	)
 }

api/http/handlerfuncs_backup.go

@@ -0,0 +1,123 @@
+// Copyright 2023 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package http
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/errors"
+)
+
+func exportHandler(rw http.ResponseWriter, req *http.Request) {
+	db, err := dbFromContext(req.Context())
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusInternalServerError)
+		return
+	}
+
+	cfg := &client.BackupConfig{}
+	err = getJSON(req, cfg)
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusBadRequest)
+		return
+	}
+
+	err = validateBackupConfig(req.Context(), cfg, db)
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusBadRequest)
+		return
+	}
+
+	err = db.BasicExport(req.Context(), cfg)
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusInternalServerError)
+		return
+	}
+
+	sendJSON(
+		req.Context(),
+		rw,
+		simpleDataResponse("result", "success"),
+		http.StatusOK,
+	)
+}
+
+func importHandler(rw http.ResponseWriter, req *http.Request) {
+	db, err := dbFromContext(req.Context())
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusInternalServerError)
+		return
+	}
+
+	cfg := &client.BackupConfig{}
+	err = getJSON(req, cfg)
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusBadRequest)
+		return
+	}
+
+	err = validateBackupConfig(req.Context(), cfg, db)
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusBadRequest)
+		return
+	}
+
+	err = db.BasicImport(req.Context(), cfg.Filepath)
+	if err != nil {
+		handleErr(req.Context(), rw, err, http.StatusInternalServerError)
+		return
+	}
+
+	sendJSON(
+		req.Context(),
+		rw,
+		simpleDataResponse("result", "success"),
+		http.StatusOK,
+	)
+}
+
+func validateBackupConfig(ctx context.Context, cfg *client.BackupConfig, db client.DB) error {
+	if !isValidPath(cfg.Filepath) {
+		return errors.New("invalid file path")
+	}
+
+	if cfg.Format != "" && strings.ToLower(cfg.Format) != "json" {
+		return errors.New("only JSON format is supported at the moment")
+	}
+	for _, colName := range cfg.Collections {
+		_, err := db.GetCollectionByName(ctx, colName)
+		if err != nil {
+			return errors.Wrap("collection does not exist", err)
+		}
+	}
+	return nil
+}
+
+func isValidPath(filepath string) bool {
+	// if a file exists, return true
+	if _, err := os.Stat(filepath); err == nil {
+		return true
+	}
+
+	// if not, attempt to write to the path and if successful,
+	// remove the file and return true
+	var d []byte
+	if err := os.WriteFile(filepath, d, 0o644); err == nil {
+		_ = os.Remove(filepath)
+		return true
+	}
+
+	return false
+}