-
Notifications
You must be signed in to change notification settings - Fork 1
SRE-000: Feature bt map #2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
6ddb0ca
mapping: add gcs reader / uploader
RO-29 22cd331
mapping: writer + reader
RO-29 4c7173a
upload: add force_upload
RO-29 a3cf251
Fix redundant declaration
alexdebril 178e1a7
Fix ineffectual assignment to err
alexdebril 9f24258
Implement error throwing
alexdebril 47075f4
Fix linter configuration
alexdebril File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| package mapping | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io" | ||
|
|
||
| "cloud.google.com/go/storage" | ||
| "github.com/pkg/errors" | ||
| "google.golang.org/api/option" | ||
| ) | ||
|
|
||
| type GcloudCreds struct { | ||
| Type string `json:"type"` | ||
| ProjectID string `json:"project_id"` | ||
| PrivateKeyID string `json:"private_key_id"` | ||
| PrivateKey string `json:"private_key"` | ||
| ClientEmail string `json:"client_email"` | ||
| ClientID string `json:"client_id"` | ||
| AuthURI string `json:"auth_uri"` | ||
| TokenURI string `json:"token_uri"` | ||
| AuthProvider string `json:"auth_provider_x509_cert_url"` | ||
| CertURL string `json:"client_x509_cert_url"` | ||
| } | ||
|
|
||
| type gcsBucketGetter struct { | ||
| objectGetter interface { | ||
| Object(name string) *storage.ObjectHandle | ||
| } | ||
| } | ||
|
|
||
| func NewGCSBucketGetter(gcreds *GcloudCreds, bucketName string) (*gcsBucketGetter, *storage.Client, error) { | ||
| credsB, err := json.Marshal(gcreds) | ||
| if err != nil { | ||
| return nil, nil, errors.Wrap(err, "json marshal") | ||
| } | ||
|
|
||
| client, err := storage.NewClient(context.Background(), option.WithCredentialsJSON(credsB)) | ||
| if err != nil { | ||
| return nil, nil, errors.Wrap(err, "gcs storage client") | ||
| } | ||
| return &gcsBucketGetter{objectGetter: client.Bucket(bucketName)}, client, nil | ||
| } | ||
|
|
||
| func NewGCSBucketGetterFromEnvironment(bucketName string) (*gcsBucketGetter, *storage.Client, error) { | ||
| client, err := storage.NewClient(context.Background()) | ||
| if err != nil { | ||
| return nil, nil, errors.Wrap(err, "gcs storage client") | ||
| } | ||
| return &gcsBucketGetter{objectGetter: client.Bucket(bucketName)}, client, nil | ||
| } | ||
|
|
||
| func NewGCSBucketGetterWithClient(client *storage.Client, bucketName string) (*gcsBucketGetter, error) { | ||
| return &gcsBucketGetter{objectGetter: client.Bucket(bucketName)}, nil | ||
| } | ||
|
|
||
| // GetStorageWriter returns the storage writer for google cloud storage. | ||
| func (r *gcsBucketGetter) GetStorageWriter(ctx context.Context, fileName string) io.WriteCloser { | ||
| return r.objectGetter.Object(fileName).NewWriter(ctx) | ||
| } | ||
|
|
||
| // GetStorageReader returns the storage reader for google cloud storage. | ||
| func (r *gcsBucketGetter) GetStorageReader(ctx context.Context, fileName string) (io.ReadCloser, error) { | ||
| return r.objectGetter.Object(fileName).NewReader(ctx) | ||
| } | ||
|
|
||
| func getMappingFilename(eventFamily string, version string, environment string) string { | ||
| // event_family/v1.0.0.json | ||
| return fmt.Sprintf("%s/%s/%s.json", eventFamily, environment, version) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package mapping | ||
|
|
||
| import ( | ||
| "context" | ||
| "io" | ||
| "time" | ||
|
|
||
| "cloud.google.com/go/storage" | ||
| ) | ||
|
|
||
| type Reader struct { | ||
| readerBucket func(ctx context.Context, fileName string) (io.ReadCloser, error) | ||
| } | ||
|
|
||
| func NewReader(gcreds *GcloudCreds, bucketName string) (*Reader, *storage.Client, error) { | ||
| gb, gbClient, err := NewGCSBucketGetter(gcreds, bucketName) | ||
| if err != nil { | ||
| return nil, nil, err | ||
| } | ||
| return &Reader{ | ||
| readerBucket: gb.GetStorageReader, | ||
| }, gbClient, nil | ||
| } | ||
|
|
||
| func newReaderFromGCSClient(gbSL func(ctx context.Context, fileName string) (io.ReadCloser, error)) *Reader { | ||
| return &Reader{ | ||
| readerBucket: gbSL, | ||
| } | ||
| } | ||
|
|
||
| func (r *Reader) Load(ctx context.Context, eventFamily string, version string, environment string) (*Mapping, error) { | ||
| ctx, cancel := context.WithTimeout(ctx, time.Second*50) | ||
| defer cancel() | ||
| filename := getMappingFilename(eventFamily, version, environment) | ||
| reader, err := r.readerBucket(ctx, filename) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| m, err := LoadMappingIO(reader) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| err = reader.Close() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return m, nil | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package mapping | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "io" | ||
| "time" | ||
|
|
||
| std_errors "errors" | ||
|
|
||
| "cloud.google.com/go/storage" | ||
| "github.com/davecgh/go-spew/spew" | ||
| "github.com/pierrre/compare" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| type Writer struct { | ||
| writerBucket func(ctx context.Context, fileName string) io.WriteCloser | ||
| readerLoad func(ctx context.Context, eventFamily string, version string, environment string) (*Mapping, error) | ||
| } | ||
|
|
||
| func NewWriter(gcreds *GcloudCreds, bucketName string) (*Writer, *storage.Client, error) { | ||
| gb, gbClient, err := NewGCSBucketGetter(gcreds, bucketName) | ||
| if err != nil { | ||
| return nil, nil, errors.Wrap(err, "new gcs bucket") | ||
| } | ||
|
|
||
| return &Writer{ | ||
| writerBucket: gb.GetStorageWriter, | ||
| readerLoad: newReaderFromGCSClient(gb.GetStorageReader).Load, | ||
| }, gbClient, nil | ||
| } | ||
|
|
||
| func NewWriterFromGCSClient(gbSW func(ctx context.Context, fileName string) io.WriteCloser, gbSL func(ctx context.Context, fileName string) (io.ReadCloser, error)) (*Writer, error) { | ||
| return &Writer{ | ||
| writerBucket: gbSW, | ||
| readerLoad: newReaderFromGCSClient(gbSL).Load, | ||
| }, nil | ||
| } | ||
|
|
||
| func (w *Writer) Upload(ctx context.Context, eventFamily, version string, environment string, writeMapping *Mapping, forceUpload bool) error { | ||
| ctx, cancel := context.WithTimeout(ctx, time.Second*50) | ||
| defer cancel() | ||
| filename := getMappingFilename(eventFamily, version, environment) | ||
| //if force upload is false, we check for already existing mapping and return without overwriting | ||
| if !forceUpload { | ||
| readMapping, err := w.readerLoad(ctx, eventFamily, version, environment) | ||
| if err != nil && UnwrapAll(err) != storage.ErrObjectNotExist { | ||
| return errors.Wrap(err, "get storage reader") | ||
| } | ||
| diff := compare.Compare(readMapping, writeMapping) | ||
| if readMapping != nil { | ||
| return errors.Errorf("mapping already exists:\nread:\n%s\nwrite:\n%s\ndiff:\n%+v", spew.Sdump(readMapping), spew.Sdump(writeMapping), diff) | ||
| } | ||
| } | ||
|
|
||
| // only if force upload is true or object does not exists | ||
| writer := w.writerBucket(ctx, filename) | ||
| encoder := json.NewEncoder(writer) | ||
| err := encoder.Encode(writeMapping) | ||
| if err != nil { | ||
| return errors.Wrap(err, "encode mapping") | ||
| } | ||
| err = writer.Close() | ||
| if err != nil { | ||
| return errors.Wrap(err, "close uploaded gcp file") | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // Unwrap calls std_errors.Unwrap. | ||
| func Unwrap(err error) error { | ||
| return std_errors.Unwrap(err) | ||
| } | ||
|
|
||
| // UnwrapAll unwraps all nested errors, and returns the last one. | ||
| func UnwrapAll(err error) error { | ||
| for { | ||
| werr := Unwrap(err) | ||
| if werr == nil { | ||
| return err | ||
| } | ||
| err = werr | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.