-
Notifications
You must be signed in to change notification settings - Fork 57
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3 from KengoTODA/gcs-storage
Add a simple GCS storage impl
- Loading branch information
Showing
7 changed files
with
996 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
package gcs | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"io" | ||
|
||
gcStorage "cloud.google.com/go/storage" | ||
) | ||
|
||
// A minimal interface to mock behavior of GCS client. | ||
type Client interface { | ||
// Read an object from a GCS bucket. | ||
Read(ctx context.Context) ([]byte, error) | ||
|
||
// Write an object onto a GCS bucket. | ||
Write(ctx context.Context, p []byte) error | ||
} | ||
|
||
// An implementation of Client that delegates actual operation to gcsStorage.Client. | ||
type Adapter struct { | ||
// A config to specify which bucket and object we handle. | ||
config Config | ||
// A GCS client which is delegated actual operation. | ||
client *gcStorage.Client | ||
} | ||
|
||
func (a Adapter) Read(ctx context.Context) ([]byte, error) { | ||
r, err := a.client.Bucket(a.config.Bucket).Object(a.config.Name).NewReader(ctx) | ||
if err != nil { | ||
return nil, err | ||
} | ||
defer r.Close() | ||
|
||
body, err := io.ReadAll(r) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed reading from gcs://%s/%s: %w", a.config.Bucket, a.config.Name, err) | ||
} | ||
return body, nil | ||
} | ||
|
||
func (a Adapter) Write(ctx context.Context, p []byte) error { | ||
w := a.client.Bucket(a.config.Bucket).Object(a.config.Name).NewWriter(ctx) | ||
_, err := w.Write(p) | ||
|
||
if err != nil { | ||
return fmt.Errorf("failed writing to gcs://%s/%s: %w", a.config.Bucket, a.config.Name, err) | ||
} | ||
return w.Close() | ||
} | ||
|
||
// NewClient returns a new Client with given Context and Config. | ||
func NewClient(ctx context.Context, config Config) (Client, error) { | ||
c, err := gcStorage.NewClient(ctx) | ||
a := &Adapter{ | ||
config: config, | ||
client: c, | ||
} | ||
return a, err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
package gcs | ||
|
||
import storage "github.com/minamijoyo/tfmigrate-storage" | ||
|
||
// Config is a config for Google Cloud Storage. | ||
// This is expected to have almost the same options as Terraform gcs backend. | ||
// https://www.terraform.io/language/settings/backends/gcs | ||
// However, it has many minor options and it's a pain to test all options from | ||
// first, so we added only options we need for now. | ||
type Config struct { | ||
// The name of the GCS bucket. | ||
Bucket string `hcl:"bucket"` | ||
// Path to the migration history file. | ||
Name string `hcl:"name"` | ||
} | ||
|
||
// Config implements a storage.Config. | ||
var _ storage.Config = (*Config)(nil) | ||
|
||
// NewStorage returns a new instance of storage.Storage. | ||
func (c *Config) NewStorage() (storage.Storage, error) { | ||
return NewStorage(c, nil) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
package gcs | ||
|
||
import "testing" | ||
|
||
func TestConfigNewStorage(t *testing.T) { | ||
cases := []struct { | ||
desc string | ||
config *Config | ||
ok bool | ||
}{ | ||
{ | ||
desc: "valid", | ||
config: &Config{ | ||
Bucket: "tfmigrate-test", | ||
Name: "tfmigrate/history.json", | ||
}, | ||
ok: true, | ||
}, | ||
} | ||
|
||
for _, tc := range cases { | ||
t.Run(tc.desc, func(t *testing.T) { | ||
got, err := tc.config.NewStorage() | ||
if tc.ok && err != nil { | ||
t.Fatalf("unexpected err: %s", err) | ||
} | ||
if !tc.ok && err == nil { | ||
t.Fatalf("expected to return an error, but no error, got: %#v", got) | ||
} | ||
if tc.ok { | ||
_ = got.(*Storage) | ||
} | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
package gcs | ||
|
||
import ( | ||
"context" | ||
|
||
gcStorage "cloud.google.com/go/storage" | ||
storage "github.com/minamijoyo/tfmigrate-storage" | ||
) | ||
|
||
// An implementation of [storage.Storage] interface. | ||
type Storage struct { | ||
// config is a storage config for GCS. | ||
config *Config | ||
// client is an instance of Client interface to call API. | ||
// It is intended to be replaced with a mock for testing. | ||
// https://pkg.go.dev/cloud.google.com/go/storage#Client | ||
client Client | ||
} | ||
|
||
var _ storage.Storage = (*Storage)(nil) | ||
|
||
// NewStorage returns a new instance of Storage. | ||
func NewStorage(config *Config, client Client) (*Storage, error) { | ||
s := &Storage{ | ||
config: config, | ||
client: client, | ||
} | ||
return s, nil | ||
} | ||
|
||
func (s *Storage) Write(ctx context.Context, b []byte) error { | ||
err := s.init(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
return s.client.Write(ctx, b) | ||
} | ||
|
||
func (s *Storage) Read(ctx context.Context) ([]byte, error) { | ||
err := s.init(ctx) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
r, err := s.client.Read(ctx) | ||
if err == gcStorage.ErrObjectNotExist { | ||
return []byte{}, nil | ||
} else if err != nil { | ||
return nil, err | ||
} | ||
return r, nil | ||
} | ||
|
||
func (s *Storage) init(ctx context.Context) error { | ||
if s.client == nil { | ||
client, err := gcStorage.NewClient(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
s.client = Adapter{ | ||
config: *s.config, | ||
client: client, | ||
} | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,144 @@ | ||
package gcs | ||
|
||
import ( | ||
"context" | ||
"testing" | ||
|
||
gcStorage "cloud.google.com/go/storage" | ||
) | ||
|
||
// mockClient is a mock implementation for testing. | ||
type mockClient struct { | ||
dataToRead []byte | ||
err error | ||
} | ||
|
||
func (c *mockClient) Read(ctx context.Context) ([]byte, error) { | ||
return c.dataToRead, c.err | ||
} | ||
|
||
func (c *mockClient) Write(ctx context.Context, b []byte) error { | ||
return c.err | ||
} | ||
|
||
func TestStorageWrite(t *testing.T) { | ||
cases := []struct { | ||
desc string | ||
config *Config | ||
client Client | ||
contents []byte | ||
ok bool | ||
}{ | ||
{ | ||
desc: "simple", | ||
config: &Config{ | ||
Bucket: "tfmigrate-test", | ||
Name: "tfmigrate/history.json", | ||
}, | ||
client: &mockClient{ | ||
err: nil, | ||
}, | ||
contents: []byte("foo"), | ||
ok: true, | ||
}, | ||
{ | ||
desc: "bucket does not exist", | ||
config: &Config{ | ||
Bucket: "not-exist-bucket", | ||
Name: "tfmigrate/history.json", | ||
}, | ||
client: &mockClient{ | ||
err: gcStorage.ErrBucketNotExist, | ||
}, | ||
contents: []byte("foo"), | ||
ok: false, | ||
}, | ||
} | ||
|
||
for _, tc := range cases { | ||
t.Run(tc.desc, func(t *testing.T) { | ||
s, err := NewStorage(tc.config, tc.client) | ||
if err != nil { | ||
t.Fatalf("failed to NewStorage: %s", err) | ||
} | ||
err = s.Write(context.Background(), tc.contents) | ||
if tc.ok && err != nil { | ||
t.Fatalf("unexpected err: %s", err) | ||
} | ||
if !tc.ok && err == nil { | ||
t.Fatal("expected to return an error, but no error") | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func TestStorageRead(t *testing.T) { | ||
cases := []struct { | ||
desc string | ||
config *Config | ||
client Client | ||
contents []byte | ||
ok bool | ||
}{ | ||
{ | ||
desc: "simple", | ||
config: &Config{ | ||
Bucket: "tfmigrate-test", | ||
Name: "tfmigrate/history.json", | ||
}, | ||
client: &mockClient{ | ||
dataToRead: []byte("foo"), | ||
err: nil, | ||
}, | ||
contents: []byte("foo"), | ||
ok: true, | ||
}, | ||
{ | ||
desc: "bucket does not exist", | ||
config: &Config{ | ||
Bucket: "not-exist-bucket", | ||
Name: "tfmigrate/history.json", | ||
}, | ||
client: &mockClient{ | ||
dataToRead: nil, | ||
err: gcStorage.ErrBucketNotExist, | ||
}, | ||
contents: nil, | ||
ok: false, | ||
}, | ||
{ | ||
desc: "object does not exist", | ||
config: &Config{ | ||
Bucket: "tfmigrate-test", | ||
Name: "not_exist.json", | ||
}, | ||
client: &mockClient{ | ||
err: gcStorage.ErrObjectNotExist, | ||
}, | ||
contents: []byte{}, | ||
ok: true, | ||
}, | ||
} | ||
|
||
for _, tc := range cases { | ||
t.Run(tc.desc, func(t *testing.T) { | ||
s, err := NewStorage(tc.config, tc.client) | ||
if err != nil { | ||
t.Fatalf("failed to NewStorage: %s", err) | ||
} | ||
got, err := s.Read(context.Background()) | ||
if tc.ok && err != nil { | ||
t.Fatalf("unexpected err: %s", err) | ||
} | ||
if !tc.ok && err == nil { | ||
t.Fatal("expected to return an error, but no error") | ||
} | ||
|
||
if tc.ok { | ||
if string(got) != string(tc.contents) { | ||
t.Errorf("got: %s, want: %s", string(got), string(tc.contents)) | ||
} | ||
} | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.