Adding support for GCS backend

.github/workflows/checks.yaml

@@ -34,4 +34,6 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Tests
-        run: go test -v ./...
+        run: |
+          docker run --rm  -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server -public-host localhost:4443
+          go test -v ./...

README.md

@@ -1,10 +1,23 @@
 # multikv
 
+<!-- toc -->
+
+- [Instalation](#instalation)
+- [Example usage](#example-usage)
+- [Testing](#testing)
+- [Backends](#backends)
+  * [Local](#local)
+  * [GCS](#gcs)
+- [Storage format](#storage-format)
+- [Roadmap](#roadmap)
+
+<!-- tocstop -->
+
 MultiKV is a simple and extensible library to manage file and path-based key/value stores for multiple storage backends, such as local storage, AWS S3 and Google Cloud Storage.
 
 This library is not intended to be high performance, support high volume workloads or store large amounts of data. Instead, the goal is to provide a quick and easy way to create simple key/value stores. Additional features and optimizations such as encryption at rest, replication, authentication, authorization, and others are left to be managed by the storage backends (e.g. KMS encryption in a S3 or GCS bucket).
 
-## Instalation
+## Installation
 
 ```shell
 go get github.com/marcelocarlos/multikv
@@ -53,10 +66,22 @@ Tests are executed in CI, but if you want to run them locally first, run:
 ```shell
 # Run lint
 docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.41.0 golangci-lint run
-# Test
+
+# Test - we use fake-gcs-server to spin up a local GCS server so we can safely run the tests
+docker run --rm  -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server -public-host localhost:4443
 go test -v ./...
 ```
 
+## Backends
+
+### Local
+
+The `local` backend allows `multikv` to use a local filesystem as the key/value storage layer.
+
+### GCS
+
+The `gcs` backend allows `multikv` to use Google Cloud Storage (GCS) as the key/value storage layer.
+
 ## Storage format
 
 Regardless the backend, each key/value pair generates 2 types files: `data` and `info`.
@@ -75,10 +100,8 @@ The `data` file contains the base64-encoded value of the corresponding `key`. Th
 
 ## Roadmap
 
-- v0.2
-  - gcs backend
 - v0.3
-  - s3 backend
+  - S3 backend
 - v0.4
   - versioning support
 - v0.5

backends/backends.go

@@ -2,11 +2,9 @@ package backends
 
 type KvBackend interface {
 	Exist(path string) (bool, error)
-	IsFile(path string) (bool, error)
-	IsDir(path string) (bool, error)
 	ListDir(path string) ([]string, error)
 	DeleteDir(path string) error
 	DeleteFile(path string) error
 	ReadFile(path string) ([]byte, error)
-	WriteFile(path string, data []byte) error // will be used to write both version contents and metadata files
+	WriteFile(path string, data []byte) error
 }

backends/gcs/gcs.go

@@ -0,0 +1,100 @@
+package gcs
+
+import (
+	"context"
+	"io/ioutil"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"google.golang.org/api/iterator"
+)
+
+type GCSBackend struct {
+	client     *storage.Client
+	bucketName string
+	context    context.Context
+}
+
+func NewGCSBackend(client *storage.Client, bucketName string, ctx context.Context) GCSBackend {
+	return GCSBackend{
+		client:     client,
+		bucketName: bucketName,
+		context:    ctx,
+	}
+}
+
+func (c GCSBackend) WriteFile(path string, value []byte) error {
+	bucket := c.client.Bucket(c.bucketName)
+	obj := bucket.Object(path)
+	w := obj.NewWriter(c.context)
+	_, err := w.Write(value)
+	if err != nil {
+		return err
+	}
+	return w.Close()
+}
+
+func (c GCSBackend) ReadFile(path string) ([]byte, error) {
+	bucket := c.client.Bucket(c.bucketName)
+	obj := bucket.Object(path)
+	rc, err := obj.NewReader(c.context)
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+	return ioutil.ReadAll(rc)
+}
+
+func (c GCSBackend) DeleteFile(path string) error {
+	bucket := c.client.Bucket(c.bucketName)
+	return bucket.Object(path).Delete(c.context)
+}
+
+func (c GCSBackend) DeleteDir(path string) error {
+	if !strings.HasSuffix(path, "/") {
+		path = path + "/"
+	}
+	it := c.client.Bucket(c.bucketName).Objects(c.context, &storage.Query{Prefix: path})
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		}
+		// Need to check further this is the best to way to skip the current "directory"
+		if attrs.Name != "" {
+			err = c.client.Bucket(c.bucketName).Object(attrs.Name).Delete(c.context)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c GCSBackend) ListDir(path string) ([]string, error) {
+	it := c.client.Bucket(c.bucketName).Objects(c.context, &storage.Query{Prefix: path + "/", Delimiter: "/"})
+	var fileNames []string
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		}
+		if attrs.Name != "" {
+			fileNames = append(fileNames, attrs.Name)
+		}
+	}
+	return fileNames, nil
+}
+
+func (c GCSBackend) Exist(path string) (bool, error) {
+	bucket := c.client.Bucket(c.bucketName)
+	obj := bucket.Object(path)
+	_, err := obj.Attrs(c.context)
+	if err != nil {
+		if err == storage.ErrObjectNotExist {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}