From 902e9a27c4684bff532c18e93e48bfd5415b3f7a Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 13 Sep 2023 16:34:30 +0100 Subject: [PATCH 01/10] feat: add support for az table storage for non-sensitive config items support for AZ TableStorage has been added semver: feature closes #22 --- README.md | 5 + go.mod | 16 +- go.sum | 29 ++-- pkg/generator/aztablestorage.go | 127 +++++++++++++++ pkg/generator/aztablestorage_test.go | 221 +++++++++++++++++++++++++++ pkg/generator/generator.go | 2 + pkg/generator/keyvault.go | 25 +-- pkg/generator/keyvault_test.go | 32 ++-- pkg/generator/strategy.go | 6 + 9 files changed, 404 insertions(+), 59 deletions(-) create mode 100644 pkg/generator/aztablestorage.go create mode 100644 pkg/generator/aztablestorage_test.go diff --git a/README.md b/README.md index e65c913..c730cf5 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,11 @@ Currently supported variable and secrets implementations: - [AzureKeyvault Secrets](https://azure.microsoft.com/en-gb/products/key-vault/) - Implementation Indicator: `AZKVSECRET` - see [Special consideration for AZKVSECRET](#special-consideration-for-azkvsecret) around how to structure the token in this case. +- [Azure TableStorage]() + - TODO: + AZTABLESTORE://account/app1Config/db/config => `{host: foo.bar, port: 8891}` + AZTABLESTORE://account/app1Config/db/config|host => `foo.bar` + - [GCP Secrets](https://cloud.google.com/secret-manager) - Implementation Indicator: `GCPSECRETS` - [Hashicorp Vault](https://developer.hashicorp.com/vault/docs/secrets/kv) diff --git a/go.mod b/go.mod index e77d827..21ac096 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.13.0 github.com/aws/aws-sdk-go-v2 v1.18.0 github.com/aws/aws-sdk-go-v2/config v1.18.25 @@ -19,14 +18,14 @@ require ( cloud.google.com/go/compute v1.19.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.0.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/aws/aws-sdk-go v1.44.267 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.15.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -52,13 +51,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/api v0.123.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -71,6 +72,7 @@ require ( require ( cloud.google.com/go/secretmanager v1.10.1 + github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect @@ -89,6 +91,6 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spyzhov/ajson v0.8.0 - golang.org/x/sys v0.8.0 // indirect + golang.org/x/sys v0.10.0 // indirect gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 0582133..c4263db 100644 --- a/go.sum +++ b/go.sum @@ -10,16 +10,14 @@ cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBa cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/secretmanager v1.10.1 h1:9QwQ3oMurvmPEmM80spGe2SFGDa+RRgkLIdTm3gMWO8= cloud.google.com/go/secretmanager v1.10.1/go.mod h1:pxG0NLpcK6OMy54kfZgQmsKTPxJem708X1es7xv8n60= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 h1:iXFUCl7NK2DPVKfixcYDPGj3uLV7yf5eolBsoWD8Sc4= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2/go.mod h1:E1WPwLx0wZyus7NBHjhrHE1QgWwKJPE81fnUbT+FxqI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.13.0 h1:XY0plaTx8oeipK+XogAck2Qzv39KdnJNBwrxC4A0GL4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.13.0/go.mod h1:tj2JhpZY+NjcQcZ207YHkfwYuivmTrcj5ZNpQxpT3Qk= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4= @@ -254,8 +252,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -265,8 +264,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -288,8 +287,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= @@ -324,8 +323,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -338,8 +337,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/generator/aztablestorage.go b/pkg/generator/aztablestorage.go new file mode 100644 index 0000000..26385bf --- /dev/null +++ b/pkg/generator/aztablestorage.go @@ -0,0 +1,127 @@ +/** + * Azure KeyVault implementation +**/ +package generator + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" + "github.com/dnitsch/configmanager/pkg/log" +) + +var ErrIncorrectlyStructuredToken = errors.New("incorrectly structured token") + +// tableStoreApi +// uses this package https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/data/aztables +type tableStoreApi interface { + GetEntity(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) +} + +type AzTableStore struct { + svc tableStoreApi + ctx context.Context + token string + config TokenConfigVars +} + +// NewAzTableStore returns a KvScrtStore +// requires `AZURE_SUBSCRIPTION_ID` environment variable to be present to successfully work +func NewAzTableStore(ctx context.Context, token string, conf GenVarsConfig) (*AzTableStore, error) { + + ct := conf.ParseTokenVars(token) + + tstore := &AzTableStore{ + ctx: ctx, + config: ct, + } + + srvInit := azServiceFromToken(stripPrefix(ct.Token, AzTableStorePrefix, conf.TokenSeparator(), conf.KeySeparator()), "https://%s.table.core.windows.net/%s", 2) + tstore.token = srvInit.token + + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + log.Error(err) + return nil, err + } + + c, err := aztables.NewClient(srvInit.serviceUri, cred, nil) + if err != nil { + log.Error(err) + return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) + } + + tstore.svc = c + return tstore, nil + +} + +// setToken already happens in the constructor +func (implmt *AzTableStore) setToken(token string) {} + +func (imp *AzTableStore) tokenVal(v *retrieveStrategy) (string, error) { + log.Info("Concrete implementation AzTableSTore") + log.Infof("AzTableSTore Token: %s", imp.token) + + ctx, cancel := context.WithCancel(imp.ctx) + defer cancel() + + // split the token for partition and rowKey + pKey, rKey, err := azTableStoreTokenSplitter(imp.token) + if err != nil { + return "", err + } + + s, err := imp.svc.GetEntity(ctx, pKey, rKey, &aztables.GetEntityOptions{}) + if err != nil { + log.Errorf(implementationNetworkErr, AzKeyVaultSecretsPrefix, err, imp.token) + return "", fmt.Errorf(implementationNetworkErr+" %w", AzKeyVaultSecretsPrefix, err, imp.token, ErrRetrieveFailed) + } + if s.Value != nil { + return string(s.Value), nil + } + log.Errorf("value retrieved but empty for token: %v", imp.token) + return "", nil +} + +func azTableStoreTokenSplitter(token string) ( + partitionKey, rowKey string, err error) { + splitToken := strings.Split(strings.TrimPrefix(token, "/"), "/") + if len(splitToken) < 2 { + return "", "", fmt.Errorf("token: %s - could not be correctly destructured to pluck the partition and row keys\n%w", token, ErrIncorrectlyStructuredToken) + } + partitionKey = splitToken[0] + rowKey = splitToken[1] + // naked return to save having to define another struct + return +} + +// Generic Azure Service Init Helpers +// +// azTableStoreHelper returns a service URI and the stripped token +type azServiceHelper struct { + serviceUri string + token string +} + +// azServiceFromToken for azure the first part of the token __must__ always be the +// identifier of the service e.g. the account name for tableStore or the KV name for KVSecret +// take parameter specifies the number of elements to take from the start only +// +// e.g. a value of 2 for take will take first 2 elements from the slices +func azServiceFromToken(token string, formatUri string, take int) azServiceHelper { + // ensure preceding slash is trimmed + stringToken := strings.Split(strings.TrimPrefix(token, "/"), "/") + splitToken := []any{} + // recast []string slice to an []any + for _, st := range stringToken { + splitToken = append(splitToken, st) + } + + uri := fmt.Sprintf(formatUri, splitToken[0:take]...) + return azServiceHelper{serviceUri: uri, token: strings.Join(stringToken[take:], "/")} +} diff --git a/pkg/generator/aztablestorage_test.go b/pkg/generator/aztablestorage_test.go new file mode 100644 index 0000000..e0ad219 --- /dev/null +++ b/pkg/generator/aztablestorage_test.go @@ -0,0 +1,221 @@ +package generator + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" + "github.com/dnitsch/configmanager/internal/testutils" +) + +func azTableStoreCommonChecker(t *testing.T, partitionKey, rowKey, expectedPartitionKey, expectedRowKey string) { + if partitionKey == "" { + t.Errorf("expect name to not be nil") + } + if partitionKey != expectedPartitionKey { + t.Errorf(testutils.TestPhrase, partitionKey, expectedPartitionKey) + } + + if strings.Contains(partitionKey, string(AzKeyVaultSecretsPrefix)) { + t.Errorf("incorrectly stripped prefix") + } + + if rowKey != expectedRowKey { + t.Errorf(testutils.TestPhrase, rowKey, expectedPartitionKey) + } +} + +type mockAzTableStoreApi func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) + +func (m mockAzTableStoreApi) GetEntity(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + return m(ctx, partitionKey, rowKey, options) +} + +func Test_AzTableStore_Success(t *testing.T) { + + tests := map[string]struct { + token string + expect string + mockClient func(t *testing.T) tableStoreApi + config *GenVarsConfig + }{ + "successVal": {"AZTABLESTORE#/test-account/table//token/1", tsuccessParam, func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + resp := aztables.GetEntityResponse{} + resp.Value = []byte(tsuccessParam) + return resp, nil + }) + }, NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + }, + "successVal with :// token Separator": {"AZTABLESTORE:///test-account/table//token/1", tsuccessParam, func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + resp := aztables.GetEntityResponse{} + resp.Value = []byte(tsuccessParam) + return resp, nil + }) + }, NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), + }, + "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", tsuccessParam, func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + + resp := aztables.GetEntityResponse{} + resp.Value = nil + return resp, nil + }) + }, + NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + impl, err := NewAzTableStore(context.TODO(), tt.token, *tt.config) + if err != nil { + t.Errorf("failed to init aztablestore") + } + + impl.svc = tt.mockClient(t) + rs := newRetrieveStrategy(NewDefatultStrategy(), *tt.config) + rs.setImplementation(impl) + got, err := rs.getTokenValue() + if err != nil { + if err.Error() != tt.expect { + t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) + } + return + } + + if got != tt.expect { + t.Errorf(testutils.TestPhrase, got, tt.expect) + } + }) + } +} + +func Test_AzTableStore_Error(t *testing.T) { + + tests := map[string]struct { + token string + expect error + mockClient func(t *testing.T) tableStoreApi + config *GenVarsConfig + }{ + "errored on token parsing to partiationKey": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, nil + }) + }, + NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + }, + "errored on service method call": {"AZTABLESTORE#/test-account/table/token/ok", ErrRetrieveFailed, func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, fmt.Errorf("network error") + }) + }, + NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + }, + + "empty": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, nil + }) + }, + NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + impl, err := NewAzTableStore(context.TODO(), tt.token, *tt.config) + if err != nil { + t.Fatal("failed to init aztablestore") + } + + impl.svc = tt.mockClient(t) + rs := newRetrieveStrategy(NewDefatultStrategy(), *tt.config) + rs.setImplementation(impl) + if _, err := rs.getTokenValue(); !errors.Is(err, tt.expect) { + t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) + } + }) + } +} + +func Test_fail_AzTable_Client_init(t *testing.T) { + // this is basically a wrap around test for the url.Parse method in the stdlib + // as that is what the client uses under the hood + _, err := NewAzTableStore(context.TODO(), "/%25%65%6e%301-._~/") + } + if !errors.Is(err, ErrClientInitialization) { + t.Fatalf(testutils.TestPhraseWithContext, "aztables client init", err.Error(), ErrClientInitialization.Error()) + } +} + +func Test_azSplitTokenTableStore(t *testing.T) { + tests := []struct { + name string + token string + expect azServiceHelper + }{ + { + name: "simple_with_preceding_slash", + token: "/test-account/tablename/somejsontest", + expect: azServiceHelper{ + serviceUri: "https://test-account.table.core.windows.net/tablename", + token: "somejsontest", + }, + }, + { + name: "missing_initial_slash", + token: "test-account/tablename/somejsontest", + expect: azServiceHelper{ + serviceUri: "https://test-account.table.core.windows.net/tablename", + token: "somejsontest", + }, + }, + { + name: "missing_initial_slash_multislash_secretname", + token: "test-account/tablename/some/json/test", + expect: azServiceHelper{ + serviceUri: "https://test-account.table.core.windows.net/tablename", + token: "some/json/test", + }, + }, + { + name: "with_initial_slash_multislash_secretname", + token: "test-account/tablename//some/json/test", + expect: azServiceHelper{ + serviceUri: "https://test-account.table.core.windows.net/tablename", + token: "/some/json/test", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := azServiceFromToken(tt.token, "https://%s.table.core.windows.net/%s", 2) + if got.token != tt.expect.token { + t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) + } + if got.serviceUri != tt.expect.serviceUri { + t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) + } + }) + } +} diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 044258b..dc82124 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -22,6 +22,8 @@ const ( ParamStorePrefix ImplementationPrefix = "AWSPARAMSTR" // Azure Key Vault Secrets prefix AzKeyVaultSecretsPrefix ImplementationPrefix = "AZKVSECRET" + // Azure Key Vault Secrets prefix + AzTableStorePrefix ImplementationPrefix = "AZTABLESTORE" // Hashicorp Vault prefix HashicorpVaultPrefix ImplementationPrefix = "VAULT" // GcpSecrets diff --git a/pkg/generator/keyvault.go b/pkg/generator/keyvault.go index b50c385..2c538d7 100644 --- a/pkg/generator/keyvault.go +++ b/pkg/generator/keyvault.go @@ -5,8 +5,6 @@ package generator import ( "context" - "fmt" - "strings" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" @@ -24,12 +22,6 @@ type KvScrtStore struct { config TokenConfigVars } -// azVaultHelper provides a broken up string -type azVaultHelper struct { - vaultUri string - token string -} - // NewKvScrtStore returns a KvScrtStore // requires `AZURE_SUBSCRIPTION_ID` environment variable to be present to successfully work func NewKvScrtStore(ctx context.Context, token string, conf GenVarsConfig) (*KvScrtStore, error) { @@ -41,7 +33,7 @@ func NewKvScrtStore(ctx context.Context, token string, conf GenVarsConfig) (*KvS config: ct, } - vc := azSplitToken(stripPrefix(ct.Token, AzKeyVaultSecretsPrefix, conf.TokenSeparator(), conf.KeySeparator())) + vc := azServiceFromToken(stripPrefix(ct.Token, AzKeyVaultSecretsPrefix, conf.TokenSeparator(), conf.KeySeparator()), "https://%s.vault.azure.net", 1) kv.token = vc.token cred, err := azidentity.NewDefaultAzureCredential(nil) @@ -50,7 +42,7 @@ func NewKvScrtStore(ctx context.Context, token string, conf GenVarsConfig) (*KvS return nil, err } - c, err := azsecrets.NewClient(vc.vaultUri, cred, nil) + c, err := azsecrets.NewClient(vc.serviceUri, cred, nil) if err != nil { log.Error(err) return nil, err @@ -61,10 +53,8 @@ func NewKvScrtStore(ctx context.Context, token string, conf GenVarsConfig) (*KvS } -func (implmt *KvScrtStore) setToken(token string) { - // setToken already happens in AzureKVClient in the constructor - // no need to re-set it here -} +// setToken already happens in AzureKVClient in the constructor +func (implmt *KvScrtStore) setToken(token string) {} func (imp *KvScrtStore) tokenVal(v *retrieveStrategy) (string, error) { log.Infof("%s", "Concrete implementation AzKeyVault Secret") @@ -86,10 +76,3 @@ func (imp *KvScrtStore) tokenVal(v *retrieveStrategy) (string, error) { log.Errorf("value retrieved but empty for token: %v", imp.token) return "", nil } - -func azSplitToken(token string) azVaultHelper { - // ensure preceding slash is trimmed - splitToken := strings.Split(strings.TrimPrefix(token, "/"), "/") - vaultUri := fmt.Sprintf("https://%s.vault.azure.net", splitToken[0]) - return azVaultHelper{vaultUri: vaultUri, token: strings.Join(splitToken[1:], "/")} -} diff --git a/pkg/generator/keyvault_test.go b/pkg/generator/keyvault_test.go index 23b8acb..8580d8a 100644 --- a/pkg/generator/keyvault_test.go +++ b/pkg/generator/keyvault_test.go @@ -14,49 +14,49 @@ func Test_azSplitToken(t *testing.T) { tests := []struct { name string token string - expect azVaultHelper + expect azServiceHelper }{ { name: "simple_with_preceding_slash", token: "/test-vault/somejsontest", - expect: azVaultHelper{ - vaultUri: "https://test-vault.vault.azure.net", - token: "somejsontest", + expect: azServiceHelper{ + serviceUri: "https://test-vault.vault.azure.net", + token: "somejsontest", }, }, { name: "missing_initial_slash", token: "test-vault/somejsontest", - expect: azVaultHelper{ - vaultUri: "https://test-vault.vault.azure.net", - token: "somejsontest", + expect: azServiceHelper{ + serviceUri: "https://test-vault.vault.azure.net", + token: "somejsontest", }, }, { name: "missing_initial_slash_multislash_secretname", token: "test-vault/some/json/test", - expect: azVaultHelper{ - vaultUri: "https://test-vault.vault.azure.net", - token: "some/json/test", + expect: azServiceHelper{ + serviceUri: "https://test-vault.vault.azure.net", + token: "some/json/test", }, }, { name: "with_initial_slash_multislash_secretname", token: "test-vault//some/json/test", - expect: azVaultHelper{ - vaultUri: "https://test-vault.vault.azure.net", - token: "/some/json/test", + expect: azServiceHelper{ + serviceUri: "https://test-vault.vault.azure.net", + token: "/some/json/test", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := azSplitToken(tt.token) + got := azServiceFromToken(tt.token, "https://%s.vault.azure.net", 1) if got.token != tt.expect.token { t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) } - if got.vaultUri != tt.expect.vaultUri { - t.Errorf(testutils.TestPhrase, tt.expect.vaultUri, got.vaultUri) + if got.serviceUri != tt.expect.serviceUri { + t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) } }) } diff --git a/pkg/generator/strategy.go b/pkg/generator/strategy.go index 1624f7b..cf0a149 100644 --- a/pkg/generator/strategy.go +++ b/pkg/generator/strategy.go @@ -2,11 +2,17 @@ package generator import ( "context" + "errors" "fmt" "regexp" "strings" ) +var ( + ErrRetrieveFailed = errors.New("failed to retrieve config item") + ErrClientInitialization = errors.New("failed to initialize the client") +) + type retrieveStrategy struct { implementation genVarsStrategy config GenVarsConfig From 288fb177e52123ace0d2044ad472b29948f4b51d Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 13 Sep 2023 16:48:18 +0100 Subject: [PATCH 02/10] chore(docs): update readme --- README.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c730cf5..1a46862 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,9 @@ Currently supported variable and secrets implementations: - [AzureKeyvault Secrets](https://azure.microsoft.com/en-gb/products/key-vault/) - Implementation Indicator: `AZKVSECRET` - see [Special consideration for AZKVSECRET](#special-consideration-for-azkvsecret) around how to structure the token in this case. -- [Azure TableStorage]() - - TODO: - AZTABLESTORE://account/app1Config/db/config => `{host: foo.bar, port: 8891}` - AZTABLESTORE://account/app1Config/db/config|host => `foo.bar` - +- [Azure TableStorage](https://azure.microsoft.com/en-gb/products/storage/tables/) + - Implementation Indicator: `AZTABLESTORE` + - see [Special consideration for AZTABLESTORE](#special-consideration-for-azkvsecret) around how to structure the token in this case. - [GCP Secrets](https://cloud.google.com/secret-manager) - Implementation Indicator: `GCPSECRETS` - [Hashicorp Vault](https://developer.hashicorp.com/vault/docs/secrets/kv) @@ -182,6 +180,23 @@ For Azure KeyVault the first part of the token needs to be the name of the vault > The preceeding slash to the vault name is optional - `AZKVSECRET#/test-vault/no-slash-token-1` and `AZKVSECRET#test-vault/no-slash-token-1` will both identify the vault of name `test-vault` +### Special consideration for AZTABLESTORE + +The token itself must contain all of the following properties: + +- Storage account name [`STORAGE_ACCOUNT_NAME`] +- Table Name [`TABLE_NAME`] +- Partition Key [`PARTITION_KEY`] +- Row Key [`ROW_KEY`] + +So that it would look like this `AZTABLESTORE://STORAGE_ACCOUNT_NAME/TABLE_NAME/PARTITION_KEY/ROW_KEY` + +All the usual token rules apply e.g. of `keySeparator` + +`AZTABLESTORE://account/app1Config/db/config` => `{host: foo.bar, port: 8891}` + +`AZTABLESTORE://account/app1Config/db/config|host` => `foo.bar` + ### Special consideration for HashicorpVault For HashicorpVault the first part of the token needs to be the name of the mountpath. In Dev Vaults this is `"secret"`, From d1a6a7e9acf303c378dc66b4442cbedefbbe968c Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 13 Sep 2023 17:00:31 +0100 Subject: [PATCH 03/10] fix: add missing implementation --- pkg/generator/generator.go | 2 +- pkg/generator/strategy.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index dc82124..fcf9b48 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -42,7 +42,7 @@ var ( // default varPrefix used by the replacer function // any token must beging with one of these else // it will be skipped as not a replaceable token - VarPrefix = map[ImplementationPrefix]bool{SecretMgrPrefix: true, ParamStorePrefix: true, AzKeyVaultSecretsPrefix: true, GcpSecretsPrefix: true, HashicorpVaultPrefix: true} + VarPrefix = map[ImplementationPrefix]bool{SecretMgrPrefix: true, ParamStorePrefix: true, AzKeyVaultSecretsPrefix: true, GcpSecretsPrefix: true, HashicorpVaultPrefix: true, AzTableStorePrefix: true} ) // Generatoriface describes the exported methods diff --git a/pkg/generator/strategy.go b/pkg/generator/strategy.go index cf0a149..b5e5ee0 100644 --- a/pkg/generator/strategy.go +++ b/pkg/generator/strategy.go @@ -71,6 +71,8 @@ func (rs *retrieveStrategy) SelectImplementation(ctx context.Context, prefix Imp return NewGcpSecrets(ctx) case HashicorpVaultPrefix: return NewVaultStore(ctx, in, config) + case AzTableStorePrefix: + return NewAzTableStore(ctx, in, config) default: return nil, fmt.Errorf("implementation not found for input string: %s", in) } From 311d97f7c682d147c41d4a6370f046f81ae0306d Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 13 Sep 2023 17:41:48 +0100 Subject: [PATCH 04/10] fix: docs --- pkg/generator/aztablestorage.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/generator/aztablestorage.go b/pkg/generator/aztablestorage.go index 26385bf..9338dc4 100644 --- a/pkg/generator/aztablestorage.go +++ b/pkg/generator/aztablestorage.go @@ -1,5 +1,5 @@ /** - * Azure KeyVault implementation + * Azure TableStore implementation **/ package generator @@ -29,8 +29,7 @@ type AzTableStore struct { config TokenConfigVars } -// NewAzTableStore returns a KvScrtStore -// requires `AZURE_SUBSCRIPTION_ID` environment variable to be present to successfully work +// NewAzTableStore func NewAzTableStore(ctx context.Context, token string, conf GenVarsConfig) (*AzTableStore, error) { ct := conf.ParseTokenVars(token) From 3c4019393e9131b5ad5b82e4f27b859690c419ee Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 13 Sep 2023 18:05:54 +0100 Subject: [PATCH 05/10] fix: unit test for success empty semver:feature semver:feat semver:minor --- pkg/generator/aztablestorage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/generator/aztablestorage_test.go b/pkg/generator/aztablestorage_test.go index e0ad219..ea824d4 100644 --- a/pkg/generator/aztablestorage_test.go +++ b/pkg/generator/aztablestorage_test.go @@ -62,7 +62,7 @@ func Test_AzTableStore_Success(t *testing.T) { }) }, NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), }, - "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", tsuccessParam, func(t *testing.T) tableStoreApi { + "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", "", func(t *testing.T) tableStoreApi { return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { t.Helper() azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") From 09a2ebc4e8a78ed4bbeb46a5ba4d01ae86439a88 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 20 Sep 2023 19:28:17 +0100 Subject: [PATCH 06/10] fix: add value concept as a property to aztablestore add readme around it --- README.md | 7 +++ configmanager_test.go | 74 +++++++++++++++++++++----- pkg/generator/aztablestorage.go | 20 +++++-- pkg/generator/aztablestorage_test.go | 79 ++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1a46862..0efd097 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,15 @@ The token itself must contain all of the following properties: - Storage account name [`STORAGE_ACCOUNT_NAME`] - Table Name [`TABLE_NAME`] + - > It might make sense to make this table global to the domain or project - Partition Key [`PARTITION_KEY`] + - > This could correspond to the component/service name - Row Key [`ROW_KEY`] + - > This could correspond to the property itself a group of properties + - > e.g. `AZTABLESTORE://globalconfigstorageaccount/domainXyz/serviceXyz/db` => `{"value":{"host":"foo","port":1234,"enabled":true}}` + +> NOTE: if you store a more complex object inside a top level `value` property, this will also work for simple properties. +> NOTE2: it will continue to work the same way for keyseparator. So that it would look like this `AZTABLESTORE://STORAGE_ACCOUNT_NAME/TABLE_NAME/PARTITION_KEY/ROW_KEY` diff --git a/configmanager_test.go b/configmanager_test.go index 6ecb7ff..2f23779 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -218,14 +218,12 @@ db: } type MockCfgMgr struct { - RetrieveWithInputReplacedTest func(input string, config generator.GenVarsConfig) (string, error) + retrieveInput func(input string, config generator.GenVarsConfig) (string, error) + // retrieve func(input string, config generator.GenVarsConfig) (string, error) } func (m *MockCfgMgr) RetrieveWithInputReplaced(input string, config generator.GenVarsConfig) (string, error) { - if m.RetrieveWithInputReplacedTest != nil { - return m.RetrieveWithInputReplacedTest(input, config) - } - return "", nil + return m.retrieveInput(input, config) } func (m *MockCfgMgr) Insert(force bool) error { @@ -280,7 +278,7 @@ func Test_KubeControllerSpecHelper(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz","bar":"quz"}`, nil } return mcm @@ -298,7 +296,7 @@ func Test_KubeControllerSpecHelper(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz2","bar":"quz"}`, nil } return mcm @@ -353,7 +351,7 @@ func Test_KubeControllerComplex(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz","bar":"quz", "lol":{"bla":"booo","another":{"number": 1235, "float": 123.09}}}`, nil } return mcm @@ -407,7 +405,7 @@ func Test_YamlRetrieveMarshalled(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz","bar":"quz", "lol":{"bla":"booo","another":{"number": 1235, "float": 123.09}}}`, nil } return mcm @@ -426,7 +424,7 @@ func Test_YamlRetrieveMarshalled(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz","bar":"quz", "lol":{"bla":"","another":{"number": 0, "float": 0}}}`, nil } return mcm @@ -448,6 +446,58 @@ func Test_YamlRetrieveMarshalled(t *testing.T) { } } +func Test_YamlRetrieveMarshalled_errored(t *testing.T) { + tests := []struct { + name string + testType *testNestedStruct + expect error + cfmgr func(t *testing.T) mockConfigManageriface + }{ + { + name: "complex struct - complete", + testType: &testNestedStruct{ + Foo: testTokenAWS, + Bar: "quz", + Lol: testLol{ + Bla: "booo", + Another: testAnotherNEst{ + Number: 1235, + Float: 123.09, + }, + }, + }, + // expect: testNestedStruct{ + // Foo: "baz", + // Bar: "quz", + // Lol: testLol{ + // Bla: "booo", + // Another: testAnotherNEst{ + // Number: 1235, + // Float: 123.09, + // }, + // }, + // }, + cfmgr: func(t *testing.T) mockConfigManageriface { + mcm := &MockCfgMgr{} + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { + return ``, fmt.Errorf("%s", "error decoding") + } + return mcm + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := generator.NewConfig().WithTokenSeparator("://") + + _, err := RetrieveMarshalledYaml(tt.testType, tt.cfmgr(t), *config) + if err == nil { + t.Errorf(testutils.TestPhrase, nil, err.Error()) + } + }) + } +} + func Test_RetrieveMarshalledJson(t *testing.T) { tests := []struct { name string @@ -481,7 +531,7 @@ func Test_RetrieveMarshalledJson(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz","bar":"quz", "lol":{"bla":"booo","another":{"number": 1235, "float": 123.09}}}`, nil } return mcm @@ -500,7 +550,7 @@ func Test_RetrieveMarshalledJson(t *testing.T) { }, cfmgr: func(t *testing.T) mockConfigManageriface { mcm := &MockCfgMgr{} - mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) { + mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) { return `{"foo":"baz","bar":"quz", "lol":{"bla":"","another":{"number": 0, "float": 0}}}`, nil } return mcm diff --git a/pkg/generator/aztablestorage.go b/pkg/generator/aztablestorage.go index 9338dc4..68cf7be 100644 --- a/pkg/generator/aztablestorage.go +++ b/pkg/generator/aztablestorage.go @@ -5,6 +5,7 @@ package generator import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -62,6 +63,11 @@ func NewAzTableStore(ctx context.Context, token string, conf GenVarsConfig) (*Az // setToken already happens in the constructor func (implmt *AzTableStore) setToken(token string) {} +// tokenVal in AZ table storage if an Entity contains the `value` property +// we attempt to extract it and return. +// +// From this point then normal rules of configmanager apply, +// including keySeperator and lookup. func (imp *AzTableStore) tokenVal(v *retrieveStrategy) (string, error) { log.Info("Concrete implementation AzTableSTore") log.Infof("AzTableSTore Token: %s", imp.token) @@ -77,10 +83,16 @@ func (imp *AzTableStore) tokenVal(v *retrieveStrategy) (string, error) { s, err := imp.svc.GetEntity(ctx, pKey, rKey, &aztables.GetEntityOptions{}) if err != nil { - log.Errorf(implementationNetworkErr, AzKeyVaultSecretsPrefix, err, imp.token) - return "", fmt.Errorf(implementationNetworkErr+" %w", AzKeyVaultSecretsPrefix, err, imp.token, ErrRetrieveFailed) + log.Errorf(implementationNetworkErr, AzTableStorePrefix, err, imp.token) + return "", fmt.Errorf(implementationNetworkErr+" %w", AzTableStorePrefix, err, imp.token, ErrRetrieveFailed) } - if s.Value != nil { + if len(s.Value) > 0 { + // check for `value` property in entity + checkVal := make(map[string]interface{}) + json.Unmarshal(s.Value, &checkVal) + if checkVal["value"] != nil { + return fmt.Sprintf("%v", checkVal["value"]), nil + } return string(s.Value), nil } log.Errorf("value retrieved but empty for token: %v", imp.token) @@ -101,7 +113,7 @@ func azTableStoreTokenSplitter(token string) ( // Generic Azure Service Init Helpers // -// azTableStoreHelper returns a service URI and the stripped token +// azServiceHelper returns a service URI and the stripped token type azServiceHelper struct { serviceUri string token string diff --git a/pkg/generator/aztablestorage_test.go b/pkg/generator/aztablestorage_test.go index ea824d4..cea6222 100644 --- a/pkg/generator/aztablestorage_test.go +++ b/pkg/generator/aztablestorage_test.go @@ -101,6 +101,85 @@ func Test_AzTableStore_Success(t *testing.T) { } } +func Test_azstorage_with_value_property(t *testing.T) { + conf := NewConfig().WithKeySeparator("|").WithTokenSeparator("://") + ttests := map[string]struct { + token string + expect string + mockClient func(t *testing.T) tableStoreApi + config *GenVarsConfig + }{ + "return value property with json like object": { + "AZTABLESTORE:///test-account/table/partitionkey/rowKey|host", + "map[bool:true host:foo port:1234]", + func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":{"host":"foo","port":1234,"bool":true}}`)} + return resp, nil + }) + }, + conf, + }, + "return value property with string only": { + "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + "foo.bar.com", + func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} + return resp, nil + }) + }, + conf, + }, + "return value property with numeric only": { + "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + "1234", + func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} + return resp, nil + }) + }, + conf, + }, + "return value property with boolean only": { + "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + "false", + func(t *testing.T) tableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} + return resp, nil + }) + }, + conf, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + impl, err := NewAzTableStore(context.TODO(), tt.token, *tt.config) + if err != nil { + t.Fatal("failed to init aztablestore") + } + + impl.svc = tt.mockClient(t) + rs := newRetrieveStrategy(NewDefatultStrategy(), *tt.config) + rs.setImplementation(impl) + got, err := rs.getTokenValue() + if err != nil { + t.Fatalf(testutils.TestPhrase, err.Error(), nil) + } + + if got != tt.expect { + t.Errorf(testutils.TestPhraseWithContext, "AZ Table storage with value property inside entity", fmt.Sprintf("%q", got), fmt.Sprintf("%q", tt.expect)) + } + }) + } +} + func Test_AzTableStore_Error(t *testing.T) { tests := map[string]struct { From ff814635a46c41d347ae862cb9ff0d81da8ddfd1 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Tue, 16 Jan 2024 11:17:12 +0000 Subject: [PATCH 07/10] fix: clean up comments and delete unusued --- configmanager.go | 8 ++++---- go.mod | 2 +- go.sum | 5 +++++ pkg/generator/secretsmanager.go | 4 ---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/configmanager.go b/configmanager.go index 2701d18..40455fe 100644 --- a/configmanager.go +++ b/configmanager.go @@ -98,12 +98,12 @@ type CMRetrieveWithInputReplacediface interface { RetrieveWithInputReplaced(input string, config generator.GenVarsConfig) (string, error) } -// @deprecated -// left for compatibility // KubeControllerSpecHelper is a helper method, it marshalls an input value of that type into a string and passes it into the relevant configmanger retrieve method -// and returns the unmarshalled object back +// and returns the unmarshalled object back. // -// It accepts a DI of configmanager and the config (for testability) to replace all occurences of replaceable tokens inside a Marshalled string of that type +// # It accepts a DI of configmanager and the config (for testability) to replace all occurences of replaceable tokens inside a Marshalled string of that type +// +// Deprecated: Left for compatibility reasons func KubeControllerSpecHelper[T any](inputType T, cm CMRetrieveWithInputReplacediface, config generator.GenVarsConfig) (*T, error) { outType := new(T) rawBytes, err := json.Marshal(inputType) diff --git a/go.mod b/go.mod index 21ac096..3a885a3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/dnitsch/configmanager -go 1.20 +go 1.21 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 diff --git a/go.sum b/go.sum index c4263db..909743b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= @@ -8,6 +9,7 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl cloud.google.com/go/iam v1.0.1 h1:lyeCAU6jpnVNrE9zGQkTl3WgNgK/X+uWwaw0kynZJMU= cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/secretmanager v1.10.1 h1:9QwQ3oMurvmPEmM80spGe2SFGDa+RRgkLIdTm3gMWO8= cloud.google.com/go/secretmanager v1.10.1/go.mod h1:pxG0NLpcK6OMy54kfZgQmsKTPxJem708X1es7xv8n60= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= @@ -79,6 +81,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -299,6 +302,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -399,6 +403,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/generator/secretsmanager.go b/pkg/generator/secretsmanager.go index 6903929..ce6b2f9 100644 --- a/pkg/generator/secretsmanager.go +++ b/pkg/generator/secretsmanager.go @@ -35,10 +35,6 @@ func NewSecretsMgr(ctx context.Context) (*SecretsMgr, error) { } -// func(imp *SecretsMgr) getTokenConfig() AdditionalVars { -// return -// } - func (imp *SecretsMgr) setToken(token string) { ct := (GenVarsConfig{}).ParseTokenVars(token) imp.config = ct From 8baf9938637622f3adb6c67f5c4b77b7eeef7f3c Mon Sep 17 00:00:00 2001 From: dnitsch Date: Tue, 16 Jan 2024 17:53:02 +0000 Subject: [PATCH 08/10] fix: normalize getter/setter --- configmanager.go | 3 ++- pkg/generator/aztablestorage.go | 2 +- pkg/generator/defaultstrategy.go | 2 +- pkg/generator/gcpsecrets.go | 2 +- pkg/generator/gcpsecrets_test.go | 2 +- pkg/generator/generator_test.go | 2 +- pkg/generator/hashivault.go | 2 +- pkg/generator/keyvault.go | 2 +- pkg/generator/paramstore.go | 2 +- pkg/generator/paramstore_test.go | 2 +- pkg/generator/secretsmanager.go | 2 +- pkg/generator/secretsmanager_test.go | 2 +- pkg/generator/strategy.go | 8 ++++---- pkg/generator/strategy_test.go | 2 +- 14 files changed, 18 insertions(+), 17 deletions(-) diff --git a/configmanager.go b/configmanager.go index 40455fe..a47e247 100644 --- a/configmanager.go +++ b/configmanager.go @@ -25,6 +25,7 @@ func (c *ConfigManager) Retrieve(tokens []string, config generator.GenVarsConfig return retrieve(tokens, gv) } +// GenerateAPI type GenerateAPI interface { Generate(tokens []string) (generator.ParsedMap, error) } @@ -34,7 +35,7 @@ func retrieve(tokens []string, gv GenerateAPI) (generator.ParsedMap, error) { } // RetrieveWithInputReplaced parses given input against all possible token strings -// using regex to grab a list of found tokens in the given string and return the replaced string +// using regex to grab a list of found tokens in the given string and returns the replaced string func (c *ConfigManager) RetrieveWithInputReplaced(input string, config generator.GenVarsConfig) (string, error) { gv := generator.NewGenerator().WithConfig(&config) return retrieveWithInputReplaced(input, gv) diff --git a/pkg/generator/aztablestorage.go b/pkg/generator/aztablestorage.go index 68cf7be..2e6e835 100644 --- a/pkg/generator/aztablestorage.go +++ b/pkg/generator/aztablestorage.go @@ -61,7 +61,7 @@ func NewAzTableStore(ctx context.Context, token string, conf GenVarsConfig) (*Az } // setToken already happens in the constructor -func (implmt *AzTableStore) setToken(token string) {} +func (implmt *AzTableStore) setTokenVal(token string) {} // tokenVal in AZ table storage if an Entity contains the `value` property // we attempt to extract it and return. diff --git a/pkg/generator/defaultstrategy.go b/pkg/generator/defaultstrategy.go index 9ae513f..0931940 100644 --- a/pkg/generator/defaultstrategy.go +++ b/pkg/generator/defaultstrategy.go @@ -12,7 +12,7 @@ func NewDefatultStrategy() *DefaultStrategy { } // setToken on default strategy -func (implmt *DefaultStrategy) setToken(token string) {} +func (implmt *DefaultStrategy) setTokenVal(token string) {} // setValue on default strategy func (implmt *DefaultStrategy) setValue(val string) {} diff --git a/pkg/generator/gcpsecrets.go b/pkg/generator/gcpsecrets.go index ee31b26..1c6f90a 100644 --- a/pkg/generator/gcpsecrets.go +++ b/pkg/generator/gcpsecrets.go @@ -37,7 +37,7 @@ func NewGcpSecrets(ctx context.Context) (*GcpSecrets, error) { }, nil } -func (imp *GcpSecrets) setToken(token string) { +func (imp *GcpSecrets) setTokenVal(token string) { ct := (GenVarsConfig{}).ParseTokenVars(token) imp.config = ct imp.token = ct.Token diff --git a/pkg/generator/gcpsecrets_test.go b/pkg/generator/gcpsecrets_test.go index c57108d..b188426 100644 --- a/pkg/generator/gcpsecrets_test.go +++ b/pkg/generator/gcpsecrets_test.go @@ -126,7 +126,7 @@ func Test_GetGcpSecretVarHappy(t *testing.T) { rs := newRetrieveStrategy(NewDefatultStrategy(), *tt.config) rs.setImplementation(impl) - rs.setToken(tt.token) + rs.setTokenVal(tt.token) got, err := rs.getTokenValue() if err != nil { diff --git a/pkg/generator/generator_test.go b/pkg/generator/generator_test.go index 0cb69ea..4083b06 100644 --- a/pkg/generator/generator_test.go +++ b/pkg/generator/generator_test.go @@ -314,7 +314,7 @@ type mockImpl struct { func (m *mockImpl) tokenVal(rs *retrieveStrategy) (s string, e error) { return m.value, m.err } -func (m *mockImpl) setToken(s string) { +func (m *mockImpl) setTokenVal(s string) { m.token = s } diff --git a/pkg/generator/hashivault.go b/pkg/generator/hashivault.go index 858a3ff..814d617 100644 --- a/pkg/generator/hashivault.go +++ b/pkg/generator/hashivault.go @@ -91,7 +91,7 @@ func newVaultStoreWithAWSAuthIAM(client *vault.Client, role string) (*vault.Clie // setToken already happens in Vault constructor // no need to re-set it here -func (imp *VaultStore) setToken(token string) { +func (imp *VaultStore) setTokenVal(token string) { // this happens inside the New func call // due to the way the client needs to be // initialised with a mountpath diff --git a/pkg/generator/keyvault.go b/pkg/generator/keyvault.go index 2c538d7..d436b5c 100644 --- a/pkg/generator/keyvault.go +++ b/pkg/generator/keyvault.go @@ -54,7 +54,7 @@ func NewKvScrtStore(ctx context.Context, token string, conf GenVarsConfig) (*KvS } // setToken already happens in AzureKVClient in the constructor -func (implmt *KvScrtStore) setToken(token string) {} +func (implmt *KvScrtStore) setTokenVal(token string) {} func (imp *KvScrtStore) tokenVal(v *retrieveStrategy) (string, error) { log.Infof("%s", "Concrete implementation AzKeyVault Secret") diff --git a/pkg/generator/paramstore.go b/pkg/generator/paramstore.go index d38fe4a..c6bc7db 100644 --- a/pkg/generator/paramstore.go +++ b/pkg/generator/paramstore.go @@ -34,7 +34,7 @@ func NewParamStore(ctx context.Context) (*ParamStore, error) { }, nil } -func (imp *ParamStore) setToken(token string) { +func (imp *ParamStore) setTokenVal(token string) { ct := (GenVarsConfig{}).ParseTokenVars(token) imp.config = ct imp.token = ct.Token diff --git a/pkg/generator/paramstore_test.go b/pkg/generator/paramstore_test.go index 2675f5d..3a140ce 100644 --- a/pkg/generator/paramstore_test.go +++ b/pkg/generator/paramstore_test.go @@ -103,7 +103,7 @@ func Test_GetParamStore(t *testing.T) { } impl.svc = tt.mockClient(t) rs.setImplementation(impl) - rs.setToken(tt.token) + rs.setTokenVal(tt.token) got, err := rs.getTokenValue() if err != nil { if err.Error() != tt.expect { diff --git a/pkg/generator/secretsmanager.go b/pkg/generator/secretsmanager.go index ce6b2f9..e696a0e 100644 --- a/pkg/generator/secretsmanager.go +++ b/pkg/generator/secretsmanager.go @@ -35,7 +35,7 @@ func NewSecretsMgr(ctx context.Context) (*SecretsMgr, error) { } -func (imp *SecretsMgr) setToken(token string) { +func (imp *SecretsMgr) setTokenVal(token string) { ct := (GenVarsConfig{}).ParseTokenVars(token) imp.config = ct imp.token = ct.Token diff --git a/pkg/generator/secretsmanager_test.go b/pkg/generator/secretsmanager_test.go index 87d7b56..627eb1a 100644 --- a/pkg/generator/secretsmanager_test.go +++ b/pkg/generator/secretsmanager_test.go @@ -100,7 +100,7 @@ func Test_GetSecretMgr(t *testing.T) { rs := newRetrieveStrategy(NewDefatultStrategy(), *tt.config) rs.setImplementation(impl) - rs.setToken(tt.token) + rs.setTokenVal(tt.token) got, err := rs.getTokenValue() if err != nil { if err.Error() != tt.expect { diff --git a/pkg/generator/strategy.go b/pkg/generator/strategy.go index b5e5ee0..67dba45 100644 --- a/pkg/generator/strategy.go +++ b/pkg/generator/strategy.go @@ -27,15 +27,15 @@ type genVarsStrategy interface { // getTokenConfig() AdditionalVars // setTokenConfig(AdditionalVars) tokenVal(rs *retrieveStrategy) (s string, e error) - setToken(s string) + setTokenVal(s string) } func (rs *retrieveStrategy) setImplementation(strategy genVarsStrategy) { rs.implementation = strategy } -func (rs *retrieveStrategy) setToken(s string) { - rs.implementation.setToken(s) +func (rs *retrieveStrategy) setTokenVal(s string) { + rs.implementation.setTokenVal(s) } func (rs *retrieveStrategy) getTokenValue() (string, error) { @@ -49,7 +49,7 @@ func (rs *retrieveStrategy) RetrieveByToken(ctx context.Context, impl genVarsStr cr.err = nil cr.key = in rs.setImplementation(impl) - rs.setToken(in) + rs.setTokenVal(in) s, err := rs.getTokenValue() if err != nil { cr.err = err diff --git a/pkg/generator/strategy_test.go b/pkg/generator/strategy_test.go index 04af908..8c7d278 100644 --- a/pkg/generator/strategy_test.go +++ b/pkg/generator/strategy_test.go @@ -16,7 +16,7 @@ type mockGenerate struct { err error } -func (m *mockGenerate) setToken(s string) { +func (m *mockGenerate) setTokenVal(s string) { } func (m *mockGenerate) tokenVal(rs *retrieveStrategy) (s string, e error) { return m.value, m.err From d3fa860cc93055131c432d86932ede4ed10fbd16 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 17 Jan 2024 16:00:02 +0000 Subject: [PATCH 09/10] fix: update docs --- README.md | 12 +++++------- pkg/generator/aztablestorage.go | 7 ++++--- pkg/generator/generator.go | 2 ++ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0efd097..814dab4 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Currently supported variable and secrets implementations: - see [Special consideration for AZKVSECRET](#special-consideration-for-azkvsecret) around how to structure the token in this case. - [Azure TableStorage](https://azure.microsoft.com/en-gb/products/storage/tables/) - Implementation Indicator: `AZTABLESTORE` - - see [Special consideration for AZTABLESTORE](#special-consideration-for-azkvsecret) around how to structure the token in this case. + - see [Special consideration for AZTABLESTORE](#special-consideration-for-aztablestore) around how to structure the token in this case. - [GCP Secrets](https://cloud.google.com/secret-manager) - Implementation Indicator: `GCPSECRETS` - [Hashicorp Vault](https://developer.hashicorp.com/vault/docs/secrets/kv) @@ -182,7 +182,7 @@ For Azure KeyVault the first part of the token needs to be the name of the vault ### Special consideration for AZTABLESTORE -The token itself must contain all of the following properties: +The token itself must contain all of the following properties, so that it would look like this `AZTABLESTORE://STORAGE_ACCOUNT_NAME/TABLE_NAME/PARTITION_KEY/ROW_KEY`: - Storage account name [`STORAGE_ACCOUNT_NAME`] - Table Name [`TABLE_NAME`] @@ -190,13 +190,11 @@ The token itself must contain all of the following properties: - Partition Key [`PARTITION_KEY`] - > This could correspond to the component/service name - Row Key [`ROW_KEY`] - - > This could correspond to the property itself a group of properties + - > This could correspond to the property itself or a group of properties - > e.g. `AZTABLESTORE://globalconfigstorageaccount/domainXyz/serviceXyz/db` => `{"value":{"host":"foo","port":1234,"enabled":true}}` + - > It will continue to work the same way with additional keyseparators inside values. -> NOTE: if you store a more complex object inside a top level `value` property, this will also work for simple properties. -> NOTE2: it will continue to work the same way for keyseparator. - -So that it would look like this `AZTABLESTORE://STORAGE_ACCOUNT_NAME/TABLE_NAME/PARTITION_KEY/ROW_KEY` +> NOTE: if you store a more complex object inside a top level `value` property this will reduce the number of columns and normalize the table - **THE DATA INSIDE THE VALUE MUST BE JSON PARSEABLE** All the usual token rules apply e.g. of `keySeparator` diff --git a/pkg/generator/aztablestorage.go b/pkg/generator/aztablestorage.go index 2e6e835..5dfdae9 100644 --- a/pkg/generator/aztablestorage.go +++ b/pkg/generator/aztablestorage.go @@ -99,8 +99,7 @@ func (imp *AzTableStore) tokenVal(v *retrieveStrategy) (string, error) { return "", nil } -func azTableStoreTokenSplitter(token string) ( - partitionKey, rowKey string, err error) { +func azTableStoreTokenSplitter(token string) (partitionKey, rowKey string, err error) { splitToken := strings.Split(strings.TrimPrefix(token, "/"), "/") if len(splitToken) < 2 { return "", "", fmt.Errorf("token: %s - could not be correctly destructured to pluck the partition and row keys\n%w", token, ErrIncorrectlyStructuredToken) @@ -111,8 +110,10 @@ func azTableStoreTokenSplitter(token string) ( return } +/* // Generic Azure Service Init Helpers -// +*/ + // azServiceHelper returns a service URI and the stripped token type azServiceHelper struct { serviceUri string diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index fcf9b48..558040e 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -217,6 +217,8 @@ func (c *GenVars) generate(rawMap map[string]string, rs retrieveIface) error { if cr.err != nil { log.Debugf("cr.err %v, for token: %s", cr.err, cr.key) errors = append(errors, cr.err) + // Skip adding not found key to the RawMap + continue } c.AddRawMap(cr.key, cr.value) } From 9e1c3c4df2f42bb2a5089b79f68288720a299ed4 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Wed, 17 Jan 2024 17:23:04 +0000 Subject: [PATCH 10/10] fix: tests for adjust non exchanged/errored tokens --- Makefile | 3 +- pkg/generator/generator_test.go | 54 ++++++++++++--------------------- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 3b94881..621629d 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ LDFLAGS := -ldflags="-s -w -X \"github.com/$(OWNER)/$(NAME)/cmd/configmanager.Ve .PHONY: test test_ci tidy install cross-build test: test_prereq - go test `go list ./... | grep -v */generated/` -v -buildvcs=false -mod=readonly -coverprofile=.coverage/out | go-junit-report > .coverage/report-junit.xml && \ + go test `go list ./... | grep -v */generated/` -v -buildvcs=false -mod=readonly -coverprofile=.coverage/out ; \ + cat .coverage/out | go-junit-report > .coverage/report-junit.xml && \ gocov convert .coverage/out | gocov-xml > .coverage/report-cobertura.xml test_ci: diff --git a/pkg/generator/generator_test.go b/pkg/generator/generator_test.go index 4083b06..90aed07 100644 --- a/pkg/generator/generator_test.go +++ b/pkg/generator/generator_test.go @@ -318,11 +318,11 @@ func (m *mockImpl) setTokenVal(s string) { m.token = s } -func Test_generate(t *testing.T) { +func Test_generate_rawmap_of_tokens_mapped_to_values(t *testing.T) { ttests := map[string]struct { - rawMap func(t *testing.T) map[string]string - rs func(t *testing.T) retrieveIface - expect string + rawMap func(t *testing.T) map[string]string + rs func(t *testing.T) retrieveIface + expectMap func() map[string]string }{ "success": { func(t *testing.T) map[string]string { @@ -342,7 +342,11 @@ func Test_generate(t *testing.T) { return &mockImpl{"foo", "bar", nil}, nil }} }, - "", + func() map[string]string { + rm := make(map[string]string) + rm["foo"] = "bar" + return rm + }, }, // as the method swallows errors at the moment this is not very useful "error in implementation": { @@ -362,7 +366,10 @@ func Test_generate(t *testing.T) { return &mockImpl{"foo", "bar", nil}, nil }} }, - "unable to retrieve", + func() map[string]string { + rm := make(map[string]string) + return rm + }, }, "error in imp selection": { func(t *testing.T) map[string]string { @@ -381,22 +388,19 @@ func Test_generate(t *testing.T) { return nil, fmt.Errorf("implementation not found for input string: %s", in) }} }, - "implementation not found for input string: foo", + func() map[string]string { + rm := make(map[string]string) + return rm + }, }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { generator := newGenVars() - err := generator.generate(tt.rawMap(t), tt.rs(t)) - if err != nil { - if err.Error() != tt.expect { - t.Errorf(testutils.TestPhrase, err, tt.expect) - } - return - } + generator.generate(tt.rawMap(t), tt.rs(t)) got := generator.RawMap() - if !(len(got) > 0) { - t.Errorf(testutils.TestPhrase, len(got), "1 or more keys") + if len(got) != len(tt.expectMap()) { + t.Errorf(testutils.TestPhraseWithContext, "generated raw map did not match", len(got), len(tt.expectMap())) } }) } @@ -427,21 +431,3 @@ func TestGenerate(t *testing.T) { }) } } - -// func TestFlushtToFile(t *testing.T) { -// ttests := map[string]struct { -// objType any - -// }{ -// "test1": -// { -// objType: nil, - -// }, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { - -// }) -// } -// }