From a3cded2d25b24f755234e2c27b13c4d587e1514a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 17:44:37 +0100 Subject: [PATCH 01/72] add transforms to structs --- internals/config/loader.go | 5 +- internals/config/parser.go | 21 +++++ internals/config/structure/structure.go | 18 ++-- internals/config/tokens.go | 4 +- utils/configutils/configutils.go | 113 ++++++++++++++++++++---- 5 files changed, 132 insertions(+), 29 deletions(-) create mode 100644 internals/config/parser.go diff --git a/internals/config/loader.go b/internals/config/loader.go index b4ca9c8d..2545516a 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -44,7 +44,8 @@ func Load() { config.MergeLayers(defaultsConf.Layer, userConf.Layer) - config.NormalizeKeys() + config.ApplyTransformFuncs(&ENV, transformFuncs) + config.TemplateConfig() InitTokens() @@ -72,7 +73,7 @@ func InitEnv() { var settings structure.SETTINGS - config.TransformChildren("settings.message.variables", transformVariables) + //config.TransformChildren("settings.message.variables", transformVariables) config.Layer.Unmarshal("settings", &settings) diff --git a/internals/config/parser.go b/internals/config/parser.go new file mode 100644 index 00000000..0a810ac8 --- /dev/null +++ b/internals/config/parser.go @@ -0,0 +1,21 @@ +package config + +import "strings" + +var transformFuncs = map[string]func(string, any) (string, any) { + "default": defaultTransform, + "lower": lowercaseTransform, + "upper": uppercaseTransform, +} + +func defaultTransform(key string, value any) (string, any) { + return key, value +} + +func lowercaseTransform(key string, value any) (string, any) { + return strings.ToLower(key), value +} + +func uppercaseTransform(key string, value any) (string, any) { + return strings.ToLower(key), value +} \ No newline at end of file diff --git a/internals/config/structure/structure.go b/internals/config/structure/structure.go index 0fd2e428..9d7ce056 100644 --- a/internals/config/structure/structure.go +++ b/internals/config/structure/structure.go @@ -14,27 +14,27 @@ type ENV struct { } type SETTINGS struct { - ACCESS ACCESS_SETTINGS `koanf:"access"` - MESSAGE MESSAGE_SETTINGS `koanf:"message"` + ACCESS ACCESS_SETTINGS `koanf:"access" transform:"lower"` + MESSAGE MESSAGE_SETTINGS `koanf:"message" transform:"lower"` } type MESSAGE_SETTINGS struct { - VARIABLES map[string]any `koanf:"variables"` + VARIABLES map[string]any `koanf:"variables" transform:"upper"` FIELD_MAPPINGS map[string][]FieldMapping `koanf:"fieldmappings"` - TEMPLATE string `koanf:"template"` + TEMPLATE string `koanf:"template" transform:"lower"` } type FieldMapping struct { - Field string `koanf:"field"` - Score int `koanf:"score"` + Field string `koanf:"field" transform:"lower"` + Score int `koanf:"score" transform:"lower"` } type ACCESS_SETTINGS struct { - ENDPOINTS []string `koanf:"endpoints"` + ENDPOINTS []string `koanf:"endpoints" transform:"lower"` FIELD_POLOCIES map[string]FieldPolicy `koanf:"fieldpolicies"` } type FieldPolicy struct { - Value any `koanf:"value"` - Action string `koanf:"action"` + Value any `koanf:"value" transform:"lower"` + Action string `koanf:"action" transform:"lower"` } \ No newline at end of file diff --git a/internals/config/tokens.go b/internals/config/tokens.go index d963312f..fdb6d8b3 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -22,8 +22,6 @@ func LoadTokens() { log.Error("Could not Load Configs in ", ENV.TOKENS_DIR, ": ", err.Error()) } - tokenConf.NormalizeKeys() - tokenConf.TemplateConfig() } @@ -32,7 +30,7 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - tokenConf.TransformChildrenUnderArray("tokenconfigs", "overrides.message.variables", transformVariables) + //tokenConf.TransformChildrenUnderArray("tokenconfigs", "overrides.message.variables", transformVariables) tokenConf.Layer.Unmarshal("tokenconfigs", &tokenConfigs) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index df5b0b62..5a5f4393 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" "sync" @@ -23,6 +24,12 @@ type Config struct { LoadFunc func() } +type TransformTarget struct { + Key string + Transform string + Value any +} + func New() *Config { return &Config{ Layer: koanf.New("."), @@ -52,6 +59,97 @@ func (config *Config) LoadFile(path string, parser koanf.Parser) (koanf.Provider return f, err } +func GetKeyToTransformMap(value any) map[string]TransformTarget { + data := map[string]TransformTarget{} + + if value == nil { + return data + } + + v := reflect.ValueOf(value) + t := reflect.TypeOf(value) + + if t.Kind() == reflect.Ptr { + v = v.Elem() + t = t.Elem() + } + + if t.Kind() != reflect.Struct { + return data + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + fieldValue := v.Field(i) + + key := field.Tag.Get("koanf") + if key == "" { + continue + } + + transformTag := field.Tag.Get("transform") + + data[key] = TransformTarget{ + Key: key, + Transform: transformTag, + Value: getValueSafe(fieldValue), + } + + // Recursively walk nested structs + if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { + + sub := GetKeyToTransformMap(fieldValue.Interface()) + + for subKey, subValue := range sub { + fullKey := key + "." + subKey + + data[fullKey] = subValue + } + } + } + + return data +} + +func getValueSafe(value reflect.Value) any { + if !value.IsValid() { + return nil + } + if value.Kind() == reflect.Ptr { + if value.IsNil() { + return nil + } + return getValueSafe(value.Elem()) + } + return value.Interface() +} + +func (config Config) ApplyTransformFuncs(structSchema any, funcs map[string]func(string, any) (string, any)) { + transformTargets := GetKeyToTransformMap(structSchema) + + all := config.Layer.All() + res := map[string]any{} + + for key, value := range all { + transformTarget, ok := transformTargets[key] + if !ok { + transformTarget.Transform = "default" + } + + fn, ok := funcs[transformTarget.Transform] + if !ok { + fn = funcs["default"] + } + + newKey, newValue := fn(key, value) + + res[newKey] = newValue + } + + config.Layer.Delete("") + config.Layer.Load(confmap.Provider(res, "."), nil) +} + func WatchFile(path string, f *file.File, loadFunc func()) { f.Watch(func(event any, err error) { if err != nil { @@ -133,21 +231,6 @@ func (config *Config) MergeLayers(layers ...*koanf.Koanf) { } } -func (config *Config) NormalizeKeys() { - data := map[string]any{} - - for _, key := range config.Layer.Keys() { - lower := strings.ToLower(key) - - log.Dev("Lowering key: ", key) - - data[lower] = config.Layer.Get(key) - } - - config.Layer.Delete("") - config.Layer.Load(confmap.Provider(data, "."), nil) -} - // Transforms Children of path func (config *Config) TransformChildren(path string, transform func(key string, value any) (string, any)) error { var sub map[string]any From ca81663ac640b1b61d53221e14af507fdfda0ba7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 17:52:52 +0100 Subject: [PATCH 02/72] add logging --- internals/config/structure/structure.go | 2 +- utils/configutils/configutils.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internals/config/structure/structure.go b/internals/config/structure/structure.go index 9d7ce056..802441b2 100644 --- a/internals/config/structure/structure.go +++ b/internals/config/structure/structure.go @@ -9,7 +9,7 @@ type ENV struct { PORT string API_URL string API_TOKENS []string - SETTINGS map[string]*SETTINGS + SETTINGS map[string]*SETTINGS `koanf:"settings"` INSECURE bool } diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 5a5f4393..92e7b8e8 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -95,6 +95,8 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { Value: getValueSafe(fieldValue), } + log.Info(key, ": ", v.String()) + // Recursively walk nested structs if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { From 84b57b94ea9836f24193feba7ac2f8a1f04a47d4 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 18:35:07 +0100 Subject: [PATCH 03/72] added new normalization func --- internals/config/loader.go | 49 +++++++++++++++++++++++++++++++- utils/configutils/configutils.go | 6 ++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 2545516a..1e7e0270 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -13,6 +13,8 @@ import ( log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/confmap" + "github.com/knadh/koanf/v2" ) var ENV *structure.ENV = &structure.ENV{ @@ -44,7 +46,7 @@ func Load() { config.MergeLayers(defaultsConf.Layer, userConf.Layer) - config.ApplyTransformFuncs(&ENV, transformFuncs) + Normalize() config.TemplateConfig() @@ -58,6 +60,51 @@ func Load() { log.Dev("Loaded Token Configs:\n" + jsonutils.ToJson(tokenConf.Layer.All())) } +func LowercaseKeys(config *configutils.Config) { + data := map[string]any{} + + for _, key := range config.Layer.Keys() { + lower := strings.ToLower(key) + + data[lower] = config.Layer.Get(key) + } + + config.Layer.Delete("") + config.Layer.Load(confmap.Provider(data, "."), nil) +} + +func Normalize() { + // Create temporary configs + tmpConf := configutils.New() + tmpConf.Layer.Load(confmap.Provider(config.Layer.Get("settings").(map[string]any), "."), nil) + + // Apply transforms to the new configs + tmpConf.ApplyTransformFuncs(&ENV.SETTINGS, ".", transformFuncs) + + tkConfigs := koanf.New(".") + tkConfigArray := []map[string]any{} + + for _, tkConfig := range tokenConf.Layer.Slices("tokensconfig") { + tmpTkConf := configutils.New() + tmpTkConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) + + tmpTkConf.ApplyTransformFuncs(&ENV.SETTINGS, "overrides", transformFuncs) + + tkConfigArray = append(tkConfigArray, tkConfig.All()) + } + + // Merge token configs together into new temporary config + tkConfigs.Set("tokenconfigs", tkConfigArray) + + // Lowercase actual configs + LowercaseKeys(config) + LowercaseKeys(tokenConf) + + // Load temporary configs back into paths + config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "."), nil) + tokenConf.Layer.Load(confmap.Provider(tkConfigs.All(), "."), nil) +} + func InitReload() { defaultsConf.OnLoad(Load) userConf.OnLoad(Load) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 92e7b8e8..d67b4fee 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -126,13 +126,13 @@ func getValueSafe(value reflect.Value) any { return value.Interface() } -func (config Config) ApplyTransformFuncs(structSchema any, funcs map[string]func(string, any) (string, any)) { +func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { transformTargets := GetKeyToTransformMap(structSchema) - all := config.Layer.All() + all := config.Layer.Get(path) res := map[string]any{} - for key, value := range all { + for key, value := range all.(map[string]any) { transformTarget, ok := transformTargets[key] if !ok { transformTarget.Transform = "default" From 9530fa691762064d461e55af5fac133b3d85eff8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 18:41:06 +0100 Subject: [PATCH 04/72] debugging nil values --- internals/config/loader.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internals/config/loader.go b/internals/config/loader.go index 1e7e0270..49bb6bf5 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -88,6 +88,8 @@ func Normalize() { tmpTkConf := configutils.New() tmpTkConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) + log.Info("TMP:\n", tmpTkConf.Layer.Sprint()) + tmpTkConf.ApplyTransformFuncs(&ENV.SETTINGS, "overrides", transformFuncs) tkConfigArray = append(tkConfigArray, tkConfig.All()) From ca98ea1577ad4babb3dd016bf68728ab50c0d0db Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 18:51:40 +0100 Subject: [PATCH 05/72] more debugging --- internals/config/loader.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 49bb6bf5..9fcb6ea5 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -78,6 +78,8 @@ func Normalize() { tmpConf := configutils.New() tmpConf.Layer.Load(confmap.Provider(config.Layer.Get("settings").(map[string]any), "."), nil) + log.Info("CONF:\n", tmpConf.Layer.Sprint()) + // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&ENV.SETTINGS, ".", transformFuncs) @@ -103,7 +105,7 @@ func Normalize() { LowercaseKeys(tokenConf) // Load temporary configs back into paths - config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "."), nil) + config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) tokenConf.Layer.Load(confmap.Provider(tkConfigs.All(), "."), nil) } From 5199ef439b4b5a4a43d4e3945c66cd1e33364a3a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:04:47 +0100 Subject: [PATCH 06/72] handle root --- utils/configutils/configutils.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index d67b4fee..7f8d9b5f 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -129,10 +129,17 @@ func getValueSafe(value reflect.Value) any { func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { transformTargets := GetKeyToTransformMap(structSchema) - all := config.Layer.Get(path) + var all map[string]any + + if path == "." { + all = config.Layer.All() + } else { + all = config.Layer.Get(path).(map[string]any) + } + res := map[string]any{} - for key, value := range all.(map[string]any) { + for key, value := range all { transformTarget, ok := transformTargets[key] if !ok { transformTarget.Transform = "default" From 73272982f145de2f08bb6e562f17fe5ea0290e2e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:20:22 +0100 Subject: [PATCH 07/72] testing empty tags --- internals/config/loader.go | 2 +- utils/configutils/configutils.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 9fcb6ea5..fe5caf90 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -86,7 +86,7 @@ func Normalize() { tkConfigs := koanf.New(".") tkConfigArray := []map[string]any{} - for _, tkConfig := range tokenConf.Layer.Slices("tokensconfig") { + for _, tkConfig := range tokenConf.Layer.Slices("tokenconfigs") { tmpTkConf := configutils.New() tmpTkConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 7f8d9b5f..a29e92c4 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -82,6 +82,8 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { field := t.Field(i) fieldValue := v.Field(i) + log.Info("Field: ", field.Name) + key := field.Tag.Get("koanf") if key == "" { continue @@ -150,6 +152,8 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma fn = funcs["default"] } + log.Info() + newKey, newValue := fn(key, value) res[newKey] = newValue From 643c673c685ab2f5e1b7e405c53adb9202481915 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:32:20 +0100 Subject: [PATCH 08/72] debugging early return --- utils/configutils/configutils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index a29e92c4..e37fb73e 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" stringutils "github.com/codeshelldev/secured-signal-api/utils/stringutils" @@ -62,6 +63,8 @@ func (config *Config) LoadFile(path string, parser koanf.Parser) (koanf.Provider func GetKeyToTransformMap(value any) map[string]TransformTarget { data := map[string]TransformTarget{} + log.Info("Value: ", jsonutils.ToJson(value)) + if value == nil { return data } From 0f36ece971e9e67ca8e9b7d9369fb2acddfdef2f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:49:01 +0100 Subject: [PATCH 09/72] fix strucschema --- internals/config/loader.go | 4 ++-- utils/configutils/configutils.go | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index fe5caf90..ded5b55d 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -81,7 +81,7 @@ func Normalize() { log.Info("CONF:\n", tmpConf.Layer.Sprint()) // Apply transforms to the new configs - tmpConf.ApplyTransformFuncs(&ENV.SETTINGS, ".", transformFuncs) + tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, ".", transformFuncs) tkConfigs := koanf.New(".") tkConfigArray := []map[string]any{} @@ -92,7 +92,7 @@ func Normalize() { log.Info("TMP:\n", tmpTkConf.Layer.Sprint()) - tmpTkConf.ApplyTransformFuncs(&ENV.SETTINGS, "overrides", transformFuncs) + tmpTkConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) tkConfigArray = append(tkConfigArray, tkConfig.All()) } diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index e37fb73e..a29e92c4 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" stringutils "github.com/codeshelldev/secured-signal-api/utils/stringutils" @@ -63,8 +62,6 @@ func (config *Config) LoadFile(path string, parser koanf.Parser) (koanf.Provider func GetKeyToTransformMap(value any) map[string]TransformTarget { data := map[string]TransformTarget{} - log.Info("Value: ", jsonutils.ToJson(value)) - if value == nil { return data } From 3be831e24e8b79a53a112272b9be8fa971eab19a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:54:30 +0100 Subject: [PATCH 10/72] more debugging debugging why transforms are not applied --- utils/configutils/configutils.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index a29e92c4..f139ec80 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" stringutils "github.com/codeshelldev/secured-signal-api/utils/stringutils" @@ -97,7 +98,7 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { Value: getValueSafe(fieldValue), } - log.Info(key, ": ", v.String()) + log.Info(key, ": ", jsonutils.ToJson(getValueSafe(fieldValue))) // Recursively walk nested structs if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { @@ -152,8 +153,6 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma fn = funcs["default"] } - log.Info() - newKey, newValue := fn(key, value) res[newKey] = newValue From b619429e10901e16f1d21ea8e47c0299e30afa2c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:58:54 +0100 Subject: [PATCH 11/72] testing --- utils/configutils/configutils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index f139ec80..2b81efae 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -83,8 +83,6 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { field := t.Field(i) fieldValue := v.Field(i) - log.Info("Field: ", field.Name) - key := field.Tag.Get("koanf") if key == "" { continue @@ -98,8 +96,6 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { Value: getValueSafe(fieldValue), } - log.Info(key, ": ", jsonutils.ToJson(getValueSafe(fieldValue))) - // Recursively walk nested structs if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { @@ -153,8 +149,12 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma fn = funcs["default"] } + log.Info("Key: ", key, "; Value: ", jsonutils.ToJson(value)) + newKey, newValue := fn(key, value) + log.Info("[NEW] Key: ", newKey, "; Value: ", jsonutils.ToJson(newValue)) + res[newKey] = newValue } From c9761ed00feac2d7a43c843cb7e0353206fbad17 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Wed, 29 Oct 2025 20:26:15 +0100 Subject: [PATCH 12/72] more debugging --- utils/configutils/configutils.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 2b81efae..43ae3c53 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -149,13 +149,21 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma fn = funcs["default"] } - log.Info("Key: ", key, "; Value: ", jsonutils.ToJson(value)) + log.Info("Transform: ", transformTarget.Transform) - newKey, newValue := fn(key, value) + keyParts := strings.Split(key, ".") + + lastKey := keyParts[len(keyParts)-1] + + newKey, newValue := fn(lastKey, value) log.Info("[NEW] Key: ", newKey, "; Value: ", jsonutils.ToJson(newValue)) - res[newKey] = newValue + keyParts[len(keyParts)-1] = newKey + + newFullKey := strings.Join(keyParts, ".") + + res[newFullKey] = newValue } config.Layer.Delete("") From 01a2a95dc464562fc2708ea5092e1c016fd417a5 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 18:43:20 +0100 Subject: [PATCH 13/72] recusively traverse config --- utils/configutils/configutils.go | 104 ++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 24 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 43ae3c53..9c07fe73 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -2,13 +2,14 @@ package configutils import ( "errors" + "maps" "os" "path/filepath" "reflect" + "strconv" "strings" "sync" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" stringutils "github.com/codeshelldev/secured-signal-api/utils/stringutils" @@ -28,6 +29,7 @@ type Config struct { type TransformTarget struct { Key string Transform string + ChildrenTransform string Value any } @@ -89,11 +91,13 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { } transformTag := field.Tag.Get("transform") + childTransformTag := field.Tag.Get("childrentransform") data[key] = TransformTarget{ - Key: key, - Transform: transformTag, - Value: getValueSafe(fieldValue), + Key: key, + Transform: transformTag, + ChildrenTransform: childTransformTag, + Value: getValueSafe(fieldValue), } // Recursively walk nested structs @@ -136,38 +140,90 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma all = config.Layer.Get(path).(map[string]any) } - res := map[string]any{} + _, res := applyTransform("", all, transformTargets, funcs) - for key, value := range all { - transformTarget, ok := transformTargets[key] - if !ok { - transformTarget.Transform = "default" - } + mapRes, ok := res.(map[string]any) - fn, ok := funcs[transformTarget.Transform] - if !ok { - fn = funcs["default"] - } + if !ok { + return + } + + config.Layer.Delete("") + config.Layer.Load(confmap.Provider(mapRes, "."), nil) +} + +func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { + target := transformTargets[key] + + targets := map[string]TransformTarget{} + + maps.Copy(transformTargets, targets) + + newKey, _ := applyTransformToAny(key, value, transformTargets, funcs) + + switch asserted := value.(type) { + case map[string]any: + res := map[string]any{} + + for k, v := range asserted { + childTarget := TransformTarget{ + Key: k, + Transform: target.ChildrenTransform, + ChildrenTransform: target.ChildrenTransform, + } - log.Info("Transform: ", transformTarget.Transform) + targets[k] = childTarget - keyParts := strings.Split(key, ".") + childKey, childValue := applyTransform(k, v, targets, funcs) - lastKey := keyParts[len(keyParts)-1] + res[childKey] = childValue + } - newKey, newValue := fn(lastKey, value) + return newKey, res + case []any: + res := []any{} + + for i, child := range asserted { + childTarget := TransformTarget{ + Key: strconv.Itoa(i), + Transform: target.ChildrenTransform, + ChildrenTransform: target.ChildrenTransform, + } - log.Info("[NEW] Key: ", newKey, "; Value: ", jsonutils.ToJson(newValue)) + targets[strconv.Itoa(i)] = childTarget + + _, childValue := applyTransform(strconv.Itoa(i), child, targets, funcs) - keyParts[len(keyParts)-1] = newKey + res = append(res, childValue) + } - newFullKey := strings.Join(keyParts, ".") + return newKey, res + default: + return applyTransformToAny(key, asserted, transformTargets, funcs) + } +} - res[newFullKey] = newValue +func applyTransformToAny(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { + transformTarget, ok := transformTargets[key] + if !ok { + transformTarget.Transform = "default" } - config.Layer.Delete("") - config.Layer.Load(confmap.Provider(res, "."), nil) + fn, ok := funcs[transformTarget.Transform] + if !ok { + fn = funcs["default"] + } + + keyParts := strings.Split(key, ".") + + lastKey := keyParts[len(keyParts)-1] + + newKey, newValue := fn(lastKey, value) + keyParts[len(keyParts)-1] = newKey + + newFullKey := strings.Join(keyParts, ".") + + return newFullKey, newValue } func WatchFile(path string, f *file.File, loadFunc func()) { From 714d60c03f0fca610d6e926c7dbeff61d7ed5fcb Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:02:14 +0100 Subject: [PATCH 14/72] lowercase transformkeys --- internals/config/loader.go | 6 +----- internals/config/structure/structure.go | 6 +++--- utils/configutils/configutils.go | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index ded5b55d..be4b4cf5 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -77,9 +77,7 @@ func Normalize() { // Create temporary configs tmpConf := configutils.New() tmpConf.Layer.Load(confmap.Provider(config.Layer.Get("settings").(map[string]any), "."), nil) - - log.Info("CONF:\n", tmpConf.Layer.Sprint()) - + // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, ".", transformFuncs) @@ -90,8 +88,6 @@ func Normalize() { tmpTkConf := configutils.New() tmpTkConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) - log.Info("TMP:\n", tmpTkConf.Layer.Sprint()) - tmpTkConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) tkConfigArray = append(tkConfigArray, tkConfig.All()) diff --git a/internals/config/structure/structure.go b/internals/config/structure/structure.go index 802441b2..b0e3356d 100644 --- a/internals/config/structure/structure.go +++ b/internals/config/structure/structure.go @@ -19,8 +19,8 @@ type SETTINGS struct { } type MESSAGE_SETTINGS struct { - VARIABLES map[string]any `koanf:"variables" transform:"upper"` - FIELD_MAPPINGS map[string][]FieldMapping `koanf:"fieldmappings"` + VARIABLES map[string]any `koanf:"variables" childtransform:"upper"` + FIELD_MAPPINGS map[string][]FieldMapping `koanf:"fieldmappings" childtransform:"default"` TEMPLATE string `koanf:"template" transform:"lower"` } @@ -31,7 +31,7 @@ type FieldMapping struct { type ACCESS_SETTINGS struct { ENDPOINTS []string `koanf:"endpoints" transform:"lower"` - FIELD_POLOCIES map[string]FieldPolicy `koanf:"fieldpolicies"` + FIELD_POLOCIES map[string]FieldPolicy `koanf:"fieldpolicies" childtransform:"default"` } type FieldPolicy struct { diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 9c07fe73..9bef2fef 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -90,11 +90,15 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { continue } + lower := strings.ToLower(key) + transformTag := field.Tag.Get("transform") childTransformTag := field.Tag.Get("childrentransform") - data[key] = TransformTarget{ - Key: key, + log.Dev("Registering ", lower, " with ", transformTag, " and ", childTransformTag) + + data[lower] = TransformTarget{ + Key: lower, Transform: transformTag, ChildrenTransform: childTransformTag, Value: getValueSafe(fieldValue), @@ -106,7 +110,7 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { sub := GetKeyToTransformMap(fieldValue.Interface()) for subKey, subValue := range sub { - fullKey := key + "." + subKey + fullKey := lower + "." + strings.ToLower(subKey) data[fullKey] = subValue } @@ -204,7 +208,9 @@ func applyTransform(key string, value any, transformTargets map[string]Transform } func applyTransformToAny(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { - transformTarget, ok := transformTargets[key] + lower := strings.ToLower(key) + + transformTarget, ok := transformTargets[lower] if !ok { transformTarget.Transform = "default" } @@ -223,6 +229,8 @@ func applyTransformToAny(key string, value any, transformTargets map[string]Tran newFullKey := strings.Join(keyParts, ".") + log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey) + return newFullKey, newValue } From b9913fd4d6a6d353716c41f0daab9cf08251bbbb Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:10:41 +0100 Subject: [PATCH 15/72] rename childrentransform to childtransform --- utils/configutils/configutils.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 9bef2fef..1f342d6a 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -29,7 +29,7 @@ type Config struct { type TransformTarget struct { Key string Transform string - ChildrenTransform string + ChildTransform string Value any } @@ -93,14 +93,14 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { lower := strings.ToLower(key) transformTag := field.Tag.Get("transform") - childTransformTag := field.Tag.Get("childrentransform") + childTransformTag := field.Tag.Get("childtransform") log.Dev("Registering ", lower, " with ", transformTag, " and ", childTransformTag) data[lower] = TransformTarget{ Key: lower, Transform: transformTag, - ChildrenTransform: childTransformTag, + ChildTransform: childTransformTag, Value: getValueSafe(fieldValue), } @@ -172,8 +172,8 @@ func applyTransform(key string, value any, transformTargets map[string]Transform for k, v := range asserted { childTarget := TransformTarget{ Key: k, - Transform: target.ChildrenTransform, - ChildrenTransform: target.ChildrenTransform, + Transform: target.ChildTransform, + ChildTransform: target.ChildTransform, } targets[k] = childTarget @@ -190,8 +190,8 @@ func applyTransform(key string, value any, transformTargets map[string]Transform for i, child := range asserted { childTarget := TransformTarget{ Key: strconv.Itoa(i), - Transform: target.ChildrenTransform, - ChildrenTransform: target.ChildrenTransform, + Transform: target.ChildTransform, + ChildTransform: target.ChildTransform, } targets[strconv.Itoa(i)] = childTarget From 4fd942f9319747e0449c0be801369235cac15237 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:21:37 +0100 Subject: [PATCH 16/72] give fullkey as key param instead of lastkey --- utils/configutils/configutils.go | 182 ---------------------------- utils/configutils/transform.go | 198 +++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 182 deletions(-) create mode 100644 utils/configutils/transform.go diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 1f342d6a..3b3fa803 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -2,11 +2,8 @@ package configutils import ( "errors" - "maps" "os" "path/filepath" - "reflect" - "strconv" "strings" "sync" @@ -26,13 +23,6 @@ type Config struct { LoadFunc func() } -type TransformTarget struct { - Key string - Transform string - ChildTransform string - Value any -} - func New() *Config { return &Config{ Layer: koanf.New("."), @@ -62,178 +52,6 @@ func (config *Config) LoadFile(path string, parser koanf.Parser) (koanf.Provider return f, err } -func GetKeyToTransformMap(value any) map[string]TransformTarget { - data := map[string]TransformTarget{} - - if value == nil { - return data - } - - v := reflect.ValueOf(value) - t := reflect.TypeOf(value) - - if t.Kind() == reflect.Ptr { - v = v.Elem() - t = t.Elem() - } - - if t.Kind() != reflect.Struct { - return data - } - - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - fieldValue := v.Field(i) - - key := field.Tag.Get("koanf") - if key == "" { - continue - } - - lower := strings.ToLower(key) - - transformTag := field.Tag.Get("transform") - childTransformTag := field.Tag.Get("childtransform") - - log.Dev("Registering ", lower, " with ", transformTag, " and ", childTransformTag) - - data[lower] = TransformTarget{ - Key: lower, - Transform: transformTag, - ChildTransform: childTransformTag, - Value: getValueSafe(fieldValue), - } - - // Recursively walk nested structs - if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { - - sub := GetKeyToTransformMap(fieldValue.Interface()) - - for subKey, subValue := range sub { - fullKey := lower + "." + strings.ToLower(subKey) - - data[fullKey] = subValue - } - } - } - - return data -} - -func getValueSafe(value reflect.Value) any { - if !value.IsValid() { - return nil - } - if value.Kind() == reflect.Ptr { - if value.IsNil() { - return nil - } - return getValueSafe(value.Elem()) - } - return value.Interface() -} - -func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { - transformTargets := GetKeyToTransformMap(structSchema) - - var all map[string]any - - if path == "." { - all = config.Layer.All() - } else { - all = config.Layer.Get(path).(map[string]any) - } - - _, res := applyTransform("", all, transformTargets, funcs) - - mapRes, ok := res.(map[string]any) - - if !ok { - return - } - - config.Layer.Delete("") - config.Layer.Load(confmap.Provider(mapRes, "."), nil) -} - -func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { - target := transformTargets[key] - - targets := map[string]TransformTarget{} - - maps.Copy(transformTargets, targets) - - newKey, _ := applyTransformToAny(key, value, transformTargets, funcs) - - switch asserted := value.(type) { - case map[string]any: - res := map[string]any{} - - for k, v := range asserted { - childTarget := TransformTarget{ - Key: k, - Transform: target.ChildTransform, - ChildTransform: target.ChildTransform, - } - - targets[k] = childTarget - - childKey, childValue := applyTransform(k, v, targets, funcs) - - res[childKey] = childValue - } - - return newKey, res - case []any: - res := []any{} - - for i, child := range asserted { - childTarget := TransformTarget{ - Key: strconv.Itoa(i), - Transform: target.ChildTransform, - ChildTransform: target.ChildTransform, - } - - targets[strconv.Itoa(i)] = childTarget - - _, childValue := applyTransform(strconv.Itoa(i), child, targets, funcs) - - res = append(res, childValue) - } - - return newKey, res - default: - return applyTransformToAny(key, asserted, transformTargets, funcs) - } -} - -func applyTransformToAny(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { - lower := strings.ToLower(key) - - transformTarget, ok := transformTargets[lower] - if !ok { - transformTarget.Transform = "default" - } - - fn, ok := funcs[transformTarget.Transform] - if !ok { - fn = funcs["default"] - } - - keyParts := strings.Split(key, ".") - - lastKey := keyParts[len(keyParts)-1] - - newKey, newValue := fn(lastKey, value) - keyParts[len(keyParts)-1] = newKey - - newFullKey := strings.Join(keyParts, ".") - - log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey) - - return newFullKey, newValue -} - func WatchFile(path string, f *file.File, loadFunc func()) { f.Watch(func(event any, err error) { if err != nil { diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go new file mode 100644 index 00000000..864aecd3 --- /dev/null +++ b/utils/configutils/transform.go @@ -0,0 +1,198 @@ +package configutils + +import ( + "maps" + "reflect" + "strconv" + "strings" + + log "github.com/codeshelldev/secured-signal-api/utils/logger" + "github.com/knadh/koanf/providers/confmap" +) + +type TransformTarget struct { + Key string + Transform string + ChildTransform string + Value any +} + +func GetKeyToTransformMap(value any) map[string]TransformTarget { + data := map[string]TransformTarget{} + + if value == nil { + return data + } + + v := reflect.ValueOf(value) + t := reflect.TypeOf(value) + + if t.Kind() == reflect.Ptr { + v = v.Elem() + t = t.Elem() + } + + if t.Kind() != reflect.Struct { + return data + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + fieldValue := v.Field(i) + + key := field.Tag.Get("koanf") + if key == "" { + continue + } + + lower := strings.ToLower(key) + + transformTag := field.Tag.Get("transform") + childTransformTag := field.Tag.Get("childtransform") + + log.Dev("Registering ", lower, " with ", transformTag, " and ", childTransformTag) + + data[lower] = TransformTarget{ + Key: lower, + Transform: transformTag, + ChildTransform: childTransformTag, + Value: getValueSafe(fieldValue), + } + + // Recursively walk nested structs + if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { + + sub := GetKeyToTransformMap(fieldValue.Interface()) + + for subKey, subValue := range sub { + fullKey := lower + "." + strings.ToLower(subKey) + + data[fullKey] = subValue + } + } + } + + return data +} + +func getValueSafe(value reflect.Value) any { + if !value.IsValid() { + return nil + } + if value.Kind() == reflect.Ptr { + if value.IsNil() { + return nil + } + return getValueSafe(value.Elem()) + } + return value.Interface() +} + +func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { + transformTargets := GetKeyToTransformMap(structSchema) + + var all map[string]any + + if path == "." { + all = config.Layer.All() + } else { + all = config.Layer.Get(path).(map[string]any) + } + + _, res := applyTransform("", all, transformTargets, funcs) + + mapRes, ok := res.(map[string]any) + + if !ok { + return + } + + config.Layer.Delete("") + config.Layer.Load(confmap.Provider(mapRes, "."), nil) +} + +func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { + target := transformTargets[key] + + targets := map[string]TransformTarget{} + + maps.Copy(transformTargets, targets) + + newKey, _ := applyTransformToAny(key, value, transformTargets, funcs) + + switch asserted := value.(type) { + case map[string]any: + res := map[string]any{} + + for k, v := range asserted { + fullKey := newKey + "." + k + + childTarget := TransformTarget{ + Key: fullKey, + Transform: target.ChildTransform, + ChildTransform: target.ChildTransform, + } + + targets[fullKey] = childTarget + + childKey, childValue := applyTransform(fullKey, v, targets, funcs) + + res[childKey] = childValue + } + + return newKey, res + case []any: + res := []any{} + + for i, child := range asserted { + fullKey := newKey + "." + strconv.Itoa(i) + + childTarget := TransformTarget{ + Key: strconv.Itoa(i), + Transform: target.ChildTransform, + ChildTransform: target.ChildTransform, + } + + targets[fullKey] = childTarget + + _, childValue := applyTransform(fullKey, child, targets, funcs) + + res = append(res, childValue) + } + + return newKey, res + default: + return applyTransformToAny(key, asserted, transformTargets, funcs) + } +} + +func applyTransformToAny(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { + lower := strings.ToLower(key) + + transformTarget, ok := transformTargets[lower] + if !ok { + transformTarget.Transform = "default" + } + + fn, ok := funcs[transformTarget.Transform] + if !ok { + fn = funcs["default"] + } + + keyParts := getKeyParts(key) + + newKey, newValue := fn(keyParts[len(keyParts)-1], value) + keyParts[len(keyParts)-1] = newKey + + newFullKey := strings.Join(keyParts, ".") + + log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey) + + return newFullKey, newValue +} + +func getKeyParts(fullKey string) []string { + keyParts := strings.Split(fullKey, ".") + + return keyParts +} \ No newline at end of file From f46a990e29bbdad6305db8274025c37a6a14a556 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:28:32 +0100 Subject: [PATCH 17/72] fixed incorrect `.` at the beginning of fullkey --- internals/config/parser.go | 8 +++++++- utils/configutils/transform.go | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internals/config/parser.go b/internals/config/parser.go index 0a810ac8..df697c10 100644 --- a/internals/config/parser.go +++ b/internals/config/parser.go @@ -1,6 +1,10 @@ package config -import "strings" +import ( + "strings" + + "github.com/codeshelldev/secured-signal-api/utils/logger" +) var transformFuncs = map[string]func(string, any) (string, any) { "default": defaultTransform, @@ -13,9 +17,11 @@ func defaultTransform(key string, value any) (string, any) { } func lowercaseTransform(key string, value any) (string, any) { + logger.Dev("LOWER: ", key) return strings.ToLower(key), value } func uppercaseTransform(key string, value any) (string, any) { + logger.Dev("UPPER: ", key) return strings.ToLower(key), value } \ No newline at end of file diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 864aecd3..38cf48ce 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -125,7 +125,7 @@ func applyTransform(key string, value any, transformTargets map[string]Transform res := map[string]any{} for k, v := range asserted { - fullKey := newKey + "." + k + fullKey := newKey + k + "." childTarget := TransformTarget{ Key: fullKey, @@ -145,7 +145,7 @@ func applyTransform(key string, value any, transformTargets map[string]Transform res := []any{} for i, child := range asserted { - fullKey := newKey + "." + strconv.Itoa(i) + fullKey := newKey + strconv.Itoa(i) + "." childTarget := TransformTarget{ Key: strconv.Itoa(i), From 1984d847409f2cd5adcc0bfcf4740917bc9df4a9 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:01:57 +0100 Subject: [PATCH 18/72] stop dot-appending if newkey is empty --- utils/configutils/transform.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 38cf48ce..6a6df94a 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -120,12 +120,18 @@ func applyTransform(key string, value any, transformTargets map[string]Transform newKey, _ := applyTransformToAny(key, value, transformTargets, funcs) + newKeyWithDot := newKey + + if newKey != "" { + newKeyWithDot = newKey + "." + } + switch asserted := value.(type) { case map[string]any: res := map[string]any{} for k, v := range asserted { - fullKey := newKey + k + "." + fullKey := newKeyWithDot + k childTarget := TransformTarget{ Key: fullKey, @@ -145,7 +151,7 @@ func applyTransform(key string, value any, transformTargets map[string]Transform res := []any{} for i, child := range asserted { - fullKey := newKey + strconv.Itoa(i) + "." + fullKey := newKeyWithDot + strconv.Itoa(i) childTarget := TransformTarget{ Key: strconv.Itoa(i), From 632c07e536d71e6b3a81fc5f3cad0f9a9f6dfb2d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:09:42 +0100 Subject: [PATCH 19/72] debugging transformmap structure --- utils/configutils/transform.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 6a6df94a..cebcca4d 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -62,6 +63,8 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { // Recursively walk nested structs if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { + log.Dev("Recursively walking ", lower) + sub := GetKeyToTransformMap(fieldValue.Interface()) for subKey, subValue := range sub { @@ -91,6 +94,8 @@ func getValueSafe(value reflect.Value) any { func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { transformTargets := GetKeyToTransformMap(structSchema) + log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") + var all map[string]any if path == "." { From 8a50b9a0669a0a1adddfd3b6783a5e71e87144b7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:13:49 +0100 Subject: [PATCH 20/72] fix typo in fieldPolicies --- internals/config/structure/structure.go | 2 +- internals/proxy/middlewares/policy.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internals/config/structure/structure.go b/internals/config/structure/structure.go index b0e3356d..15f0d8d3 100644 --- a/internals/config/structure/structure.go +++ b/internals/config/structure/structure.go @@ -31,7 +31,7 @@ type FieldMapping struct { type ACCESS_SETTINGS struct { ENDPOINTS []string `koanf:"endpoints" transform:"lower"` - FIELD_POLOCIES map[string]FieldPolicy `koanf:"fieldpolicies" childtransform:"default"` + FIELD_POLICIES map[string]FieldPolicy `koanf:"fieldpolicies" childtransform:"default"` } type FieldPolicy struct { diff --git a/internals/proxy/middlewares/policy.go b/internals/proxy/middlewares/policy.go index e7cfc6d9..2eb54c96 100644 --- a/internals/proxy/middlewares/policy.go +++ b/internals/proxy/middlewares/policy.go @@ -19,10 +19,10 @@ func policyHandler(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { settings := getSettingsByReq(req) - policies := settings.ACCESS.FIELD_POLOCIES + policies := settings.ACCESS.FIELD_POLICIES if policies == nil { - policies = getSettings("*").ACCESS.FIELD_POLOCIES + policies = getSettings("*").ACCESS.FIELD_POLICIES } body, err := request.GetReqBody(req) From e2e71d9d0c2d8de5d1d8e0dd4570b059986940f2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:18:09 +0100 Subject: [PATCH 21/72] debugging incoming keys and transform in `applyTransform()` --- utils/configutils/transform.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index cebcca4d..613ee9bf 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -117,13 +117,16 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma } func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { - target := transformTargets[key] + lower := strings.ToLower(key) + target := transformTargets[lower] + + log.Dev("Got ", target.Transform, " for ", lower) targets := map[string]TransformTarget{} maps.Copy(transformTargets, targets) - newKey, _ := applyTransformToAny(key, value, transformTargets, funcs) + newKey, _ := applyTransformToAny(lower, value, transformTargets, funcs) newKeyWithDot := newKey From 10aec596cf04f56e522b872275587f3fd68bd875 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:23:34 +0100 Subject: [PATCH 22/72] investigate foreach skipping nested maps --- utils/configutils/transform.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 613ee9bf..e107f238 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -138,6 +138,8 @@ func applyTransform(key string, value any, transformTargets map[string]Transform case map[string]any: res := map[string]any{} + log.Dev("MapData:\n-------------------------------------------\n", jsonutils.ToJson(asserted), "\n-------------------------------------------") + for k, v := range asserted { fullKey := newKeyWithDot + k From dd0ff10b14b50cb31a2265984e61eac47cf448d1 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:32:43 +0100 Subject: [PATCH 23/72] added `Unflatten()` to use for transforming --- utils/configutils/configutils.go | 30 ++++++++++++++++++++++++++++++ utils/configutils/transform.go | 10 +--------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 3b3fa803..71a8e320 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -67,6 +67,36 @@ func WatchFile(path string, f *file.File, loadFunc func()) { }) } +func (config *Config) Unflatten(path string) map[string]any { + var all map[string]any + + if path == "." { + all = config.Layer.All() + } else { + all = config.Layer.Get(path).(map[string]any) + } + + res := map[string]any{} + + for key, value := range all { + parts := strings.Split(key, ".") + + for i, p := range parts { + if i == len(parts)-1 { + res[p] = value + } else { + if _, ok := res[p]; !ok { + res[p] = map[string]any{} + } + res = res[p].(map[string]any) + } + } + } + + return res +} + + func (config *Config) LoadDir(path string, dir string, ext string, parser koanf.Parser) error { files, err := filepath.Glob(filepath.Join(dir, "*" + ext)) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index e107f238..e427ddb0 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -96,15 +96,7 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") - var all map[string]any - - if path == "." { - all = config.Layer.All() - } else { - all = config.Layer.Get(path).(map[string]any) - } - - _, res := applyTransform("", all, transformTargets, funcs) + _, res := applyTransform("", config.Unflatten(path), transformTargets, funcs) mapRes, ok := res.(map[string]any) From 1b1e4b7783208decf0beb903feb49cbadab46588 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:41:05 +0100 Subject: [PATCH 24/72] more debugging --- utils/configutils/transform.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index e427ddb0..82e0f5a7 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -94,9 +94,12 @@ func getValueSafe(value reflect.Value) any { func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { transformTargets := GetKeyToTransformMap(structSchema) + data := config.Unflatten(path) + log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") + log.Dev("BeginningMapData:\n-------------------------------------------\n", jsonutils.ToJson(data), "\n-------------------------------------------") - _, res := applyTransform("", config.Unflatten(path), transformTargets, funcs) + _, res := applyTransform("", data, transformTargets, funcs) mapRes, ok := res.(map[string]any) From 60a8c4f334cf83a7238d029480d238c14f2371d0 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:46:22 +0100 Subject: [PATCH 25/72] fix defaults quoted endpoints because yaml parsing `!` as a type indicator / tag --- data/defaults.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/data/defaults.yml b/data/defaults.yml index 08681bc5..b03b8096 100644 --- a/data/defaults.yml +++ b/data/defaults.yml @@ -32,11 +32,11 @@ settings: access: endpoints: - - !/v1/about - - !/v1/configuration - - !/v1/devices - - !/v1/register - - !/v1/unregister - - !/v1/qrcodelink - - !/v1/accounts - - !/v1/contacts + - "!/v1/about" + - "!/v1/configuration" + - "!/v1/devices" + - "!/v1/register" + - "!/v1/unregister" + - "!/v1/qrcodelink" + - "!/v1/accounts" + - "!/v1/contacts" From 022d4e580ba4d54512d9aa967b7621bf7a42b482 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:48:37 +0100 Subject: [PATCH 26/72] debugging config map for missing keys --- utils/configutils/transform.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 82e0f5a7..c297e842 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -97,7 +97,8 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma data := config.Unflatten(path) log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") - log.Dev("BeginningMapData:\n-------------------------------------------\n", jsonutils.ToJson(data), "\n-------------------------------------------") + log.Dev("InitMapData:\n-------------------------------------------\n", jsonutils.ToJson(data), "\n-------------------------------------------") + log.Dev("RawMapData:\n-------------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n-------------------------------------------") _, res := applyTransform("", data, transformTargets, funcs) From 3cd42cb3e589bc4b2b78d9b9fcc26a9df154e903 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:55:42 +0100 Subject: [PATCH 27/72] fixed map overriding --- utils/configutils/configutils.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 71a8e320..bd120681 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -81,14 +81,16 @@ func (config *Config) Unflatten(path string) map[string]any { for key, value := range all { parts := strings.Split(key, ".") + sub := res + for i, p := range parts { if i == len(parts)-1 { - res[p] = value + sub[p] = value } else { - if _, ok := res[p]; !ok { - res[p] = map[string]any{} + if _, ok := sub[p]; !ok { + sub[p] = map[string]any{} } - res = res[p].(map[string]any) + sub = sub[p].(map[string]any) } } } From d4f4d44b7c73fdcef5e7932731f93b63ac329091 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 20:56:06 +0100 Subject: [PATCH 28/72] rename for clarity --- utils/configutils/configutils.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index bd120681..c8fec316 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -83,14 +83,16 @@ func (config *Config) Unflatten(path string) map[string]any { sub := res - for i, p := range parts { + for i, part := range parts { if i == len(parts)-1 { - sub[p] = value + sub[part] = value } else { - if _, ok := sub[p]; !ok { - sub[p] = map[string]any{} + _, ok := sub[part] + + if !ok { + sub[part] = map[string]any{} } - sub = sub[p].(map[string]any) + sub = sub[part].(map[string]any) } } } From 349f9856450082e830615e6f2c9eab90af658cf6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:13:55 +0100 Subject: [PATCH 29/72] only overwrite if no transformtarget present --- utils/configutils/transform.go | 35 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index c297e842..c9e248cb 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -98,8 +98,7 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") log.Dev("InitMapData:\n-------------------------------------------\n", jsonutils.ToJson(data), "\n-------------------------------------------") - log.Dev("RawMapData:\n-------------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n-------------------------------------------") - + _, res := applyTransform("", data, transformTargets, funcs) mapRes, ok := res.(map[string]any) @@ -139,13 +138,17 @@ func applyTransform(key string, value any, transformTargets map[string]Transform for k, v := range asserted { fullKey := newKeyWithDot + k - childTarget := TransformTarget{ - Key: fullKey, - Transform: target.ChildTransform, - ChildTransform: target.ChildTransform, - } + _, ok := targets[fullKey] - targets[fullKey] = childTarget + if !ok { + childTarget := TransformTarget{ + Key: fullKey, + Transform: target.ChildTransform, + ChildTransform: target.ChildTransform, + } + + targets[fullKey] = childTarget + } childKey, childValue := applyTransform(fullKey, v, targets, funcs) @@ -159,13 +162,17 @@ func applyTransform(key string, value any, transformTargets map[string]Transform for i, child := range asserted { fullKey := newKeyWithDot + strconv.Itoa(i) - childTarget := TransformTarget{ - Key: strconv.Itoa(i), - Transform: target.ChildTransform, - ChildTransform: target.ChildTransform, - } + _, ok := targets[fullKey] + + if !ok { + childTarget := TransformTarget{ + Key: fullKey, + Transform: target.ChildTransform, + ChildTransform: target.ChildTransform, + } - targets[fullKey] = childTarget + targets[fullKey] = childTarget + } _, childValue := applyTransform(fullKey, child, targets, funcs) From 450029399e9ece21f103e76f77d90bca0e96c5bd Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:17:24 +0100 Subject: [PATCH 30/72] debugging transformmap changes afer walking recursively --- utils/configutils/transform.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index c9e248cb..d1181c43 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -96,7 +96,7 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma data := config.Unflatten(path) - log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") + log.Dev("InitTransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") log.Dev("InitMapData:\n-------------------------------------------\n", jsonutils.ToJson(data), "\n-------------------------------------------") _, res := applyTransform("", data, transformTargets, funcs) @@ -115,6 +115,8 @@ func applyTransform(key string, value any, transformTargets map[string]Transform lower := strings.ToLower(key) target := transformTargets[lower] + log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") + log.Dev("Got ", target.Transform, " for ", lower) targets := map[string]TransformTarget{} From 22bbdd4f73b25e42f4a5f3131bb4121eda15a1f0 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:21:31 +0100 Subject: [PATCH 31/72] fixed maps.Copy writing to src instead of dst --- utils/configutils/transform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index d1181c43..68e5c8a9 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -121,7 +121,7 @@ func applyTransform(key string, value any, transformTargets map[string]Transform targets := map[string]TransformTarget{} - maps.Copy(transformTargets, targets) + maps.Copy(targets, transformTargets) newKey, _ := applyTransformToAny(lower, value, transformTargets, funcs) From 78ab3ab350e541fe484f736ba479cf294fd34300 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:00:10 +0100 Subject: [PATCH 32/72] debugging keyparts array --- utils/configutils/transform.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 68e5c8a9..c8533864 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -95,9 +95,6 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma transformTargets := GetKeyToTransformMap(structSchema) data := config.Unflatten(path) - - log.Dev("InitTransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") - log.Dev("InitMapData:\n-------------------------------------------\n", jsonutils.ToJson(data), "\n-------------------------------------------") _, res := applyTransform("", data, transformTargets, funcs) @@ -115,10 +112,6 @@ func applyTransform(key string, value any, transformTargets map[string]Transform lower := strings.ToLower(key) target := transformTargets[lower] - log.Dev("TransformMap:\n-------------------------------------------\n", jsonutils.ToJson(transformTargets), "\n-------------------------------------------") - - log.Dev("Got ", target.Transform, " for ", lower) - targets := map[string]TransformTarget{} maps.Copy(targets, transformTargets) @@ -135,8 +128,6 @@ func applyTransform(key string, value any, transformTargets map[string]Transform case map[string]any: res := map[string]any{} - log.Dev("MapData:\n-------------------------------------------\n", jsonutils.ToJson(asserted), "\n-------------------------------------------") - for k, v := range asserted { fullKey := newKeyWithDot + k @@ -203,11 +194,12 @@ func applyTransformToAny(key string, value any, transformTargets map[string]Tran keyParts := getKeyParts(key) newKey, newValue := fn(keyParts[len(keyParts)-1], value) + keyParts[len(keyParts)-1] = newKey newFullKey := strings.Join(keyParts, ".") - log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey) + log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey, "\n--------------------------------------------\n", jsonutils.ToJson(keyParts)) return newFullKey, newValue } From c54c0b74e17efc7f8a902810a8887a1fed9e8c7b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:05:51 +0100 Subject: [PATCH 33/72] debugging transformfunc's output --- utils/configutils/transform.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index c8533864..acd0dbfb 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -195,6 +195,8 @@ func applyTransformToAny(key string, value any, transformTargets map[string]Tran newKey, newValue := fn(keyParts[len(keyParts)-1], value) + log.Dev("NewKey: ", newKey) + keyParts[len(keyParts)-1] = newKey newFullKey := strings.Join(keyParts, ".") From ab1c920c06d8d3ae18d975c08cc2f0c262a1fb2e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:10:01 +0100 Subject: [PATCH 34/72] fixed `uppercaseTransform()` lowercasing --- internals/config/parser.go | 6 +----- utils/configutils/transform.go | 5 ----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/internals/config/parser.go b/internals/config/parser.go index df697c10..96acd486 100644 --- a/internals/config/parser.go +++ b/internals/config/parser.go @@ -2,8 +2,6 @@ package config import ( "strings" - - "github.com/codeshelldev/secured-signal-api/utils/logger" ) var transformFuncs = map[string]func(string, any) (string, any) { @@ -17,11 +15,9 @@ func defaultTransform(key string, value any) (string, any) { } func lowercaseTransform(key string, value any) (string, any) { - logger.Dev("LOWER: ", key) return strings.ToLower(key), value } func uppercaseTransform(key string, value any) (string, any) { - logger.Dev("UPPER: ", key) - return strings.ToLower(key), value + return strings.ToUpper(key), value } \ No newline at end of file diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index acd0dbfb..ad304cd3 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,7 +6,6 @@ import ( "strconv" "strings" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -195,14 +194,10 @@ func applyTransformToAny(key string, value any, transformTargets map[string]Tran newKey, newValue := fn(keyParts[len(keyParts)-1], value) - log.Dev("NewKey: ", newKey) - keyParts[len(keyParts)-1] = newKey newFullKey := strings.Join(keyParts, ".") - log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey, "\n--------------------------------------------\n", jsonutils.ToJson(keyParts)) - return newFullKey, newValue } From 483bfccb25433dae76615d9656746641ce5d9e34 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:20:06 +0100 Subject: [PATCH 35/72] debugging output of applyTransform --- utils/configutils/configutils.go | 17 ++++++++++++----- utils/configutils/transform.go | 11 +++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index c8fec316..5d50bf86 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -67,13 +67,20 @@ func WatchFile(path string, f *file.File, loadFunc func()) { }) } +func getPath(str string) string { + if str == "." { + str = "" + } + + return str +} + func (config *Config) Unflatten(path string) map[string]any { - var all map[string]any + data := config.Layer.Get(path) + all, ok := data.(map[string]any) - if path == "." { - all = config.Layer.All() - } else { - all = config.Layer.Get(path).(map[string]any) + if !ok { + return nil } res := map[string]any{} diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index ad304cd3..56bcda6b 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -91,20 +92,24 @@ func getValueSafe(value reflect.Value) any { } func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs map[string]func(string, any) (string, any)) { + path = getPath(path) + transformTargets := GetKeyToTransformMap(structSchema) data := config.Unflatten(path) _, res := applyTransform("", data, transformTargets, funcs) + log.Dev("Result:\n-----------------------------\n", jsonutils.ToJson(res)) + mapRes, ok := res.(map[string]any) if !ok { return } - config.Layer.Delete("") - config.Layer.Load(confmap.Provider(mapRes, "."), nil) + config.Layer.Delete(path) + config.Layer.Load(confmap.Provider(mapRes, path), nil) } func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { @@ -198,6 +203,8 @@ func applyTransformToAny(key string, value any, transformTargets map[string]Tran newFullKey := strings.Join(keyParts, ".") + log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey) + return newFullKey, newValue } From 16896eb166a6c1ae22dbbceb4f00c6bca08b6349 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:30:07 +0100 Subject: [PATCH 36/72] only output lastkey instead of fullkey in `applyTransformToAny()` --- utils/configutils/transform.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 56bcda6b..4edbb6ae 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -199,13 +199,7 @@ func applyTransformToAny(key string, value any, transformTargets map[string]Tran newKey, newValue := fn(keyParts[len(keyParts)-1], value) - keyParts[len(keyParts)-1] = newKey - - newFullKey := strings.Join(keyParts, ".") - - log.Dev("Applying ", lower, " with ", transformTarget.Transform, " to ", newFullKey) - - return newFullKey, newValue + return newKey, newValue } func getKeyParts(fullKey string) []string { From b8a5743812b24dfd6d0a883a969b044f43caea96 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:48:54 +0100 Subject: [PATCH 37/72] debug config loading --- internals/config/loader.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internals/config/loader.go b/internals/config/loader.go index be4b4cf5..ea092a3f 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -81,6 +81,8 @@ func Normalize() { // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, ".", transformFuncs) + log.Dev("After Transforms:\n-----------------------------------\n", tmpConf.Layer.Sprint(), "\n-----------------------------------") + tkConfigs := koanf.New(".") tkConfigArray := []map[string]any{} From a389a1a79e62ca5e6dfaf09f47f5e3b2829d0d94 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:52:59 +0100 Subject: [PATCH 38/72] delete config before loading --- internals/config/loader.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internals/config/loader.go b/internals/config/loader.go index ea092a3f..23581bc9 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -103,7 +103,10 @@ func Normalize() { LowercaseKeys(tokenConf) // Load temporary configs back into paths + config.Layer.Delete("settings") config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) + + tokenConf.Layer.Delete("") tokenConf.Layer.Load(confmap.Provider(tkConfigs.All(), "."), nil) } From f942065d6b38b7f514395a7ada3250541ea00bb8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:57:01 +0100 Subject: [PATCH 39/72] misintepreted 2nd arg of `confmap.Provider()` being path --- internals/config/loader.go | 4 +--- utils/configutils/transform.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 23581bc9..55443c56 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -79,9 +79,7 @@ func Normalize() { tmpConf.Layer.Load(confmap.Provider(config.Layer.Get("settings").(map[string]any), "."), nil) // Apply transforms to the new configs - tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, ".", transformFuncs) - - log.Dev("After Transforms:\n-----------------------------------\n", tmpConf.Layer.Sprint(), "\n-----------------------------------") + tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) tkConfigs := koanf.New(".") tkConfigArray := []map[string]any{} diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 4edbb6ae..751cae10 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -109,7 +109,7 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma } config.Layer.Delete(path) - config.Layer.Load(confmap.Provider(mapRes, path), nil) + config.Layer.Load(confmap.Provider(mapRes, "."), nil) } func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { From bdba73d938f8420618d73c8aa195c01e7efbb260 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:02:39 +0100 Subject: [PATCH 40/72] cleanup code for release --- internals/config/loader.go | 43 ++++++------------ internals/config/tokens.go | 30 ++++++++++++- utils/configutils/configutils.go | 76 -------------------------------- utils/configutils/transform.go | 3 -- 4 files changed, 41 insertions(+), 111 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 55443c56..45c644fc 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -14,7 +14,6 @@ import ( "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/confmap" - "github.com/knadh/koanf/v2" ) var ENV *structure.ENV = &structure.ENV{ @@ -46,7 +45,8 @@ func Load() { config.MergeLayers(defaultsConf.Layer, userConf.Layer) - Normalize() + NormalizeConfig() + NormalizeTokens() config.TemplateConfig() @@ -73,39 +73,28 @@ func LowercaseKeys(config *configutils.Config) { config.Layer.Load(confmap.Provider(data, "."), nil) } -func Normalize() { +func NormalizeConfig() { + settings := config.Layer.Get("settings") + old, ok := settings.(map[string]any) + + if !ok { + log.Warn("Could not load `settings`") + return + } + // Create temporary configs tmpConf := configutils.New() - tmpConf.Layer.Load(confmap.Provider(config.Layer.Get("settings").(map[string]any), "."), nil) + tmpConf.Layer.Load(confmap.Provider(old, "."), nil) // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) - tkConfigs := koanf.New(".") - tkConfigArray := []map[string]any{} - - for _, tkConfig := range tokenConf.Layer.Slices("tokenconfigs") { - tmpTkConf := configutils.New() - tmpTkConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) - - tmpTkConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) - - tkConfigArray = append(tkConfigArray, tkConfig.All()) - } - - // Merge token configs together into new temporary config - tkConfigs.Set("tokenconfigs", tkConfigArray) - // Lowercase actual configs LowercaseKeys(config) - LowercaseKeys(tokenConf) // Load temporary configs back into paths config.Layer.Delete("settings") config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) - - tokenConf.Layer.Delete("") - tokenConf.Layer.Load(confmap.Provider(tkConfigs.All(), "."), nil) } func InitReload() { @@ -123,8 +112,6 @@ func InitEnv() { var settings structure.SETTINGS - //config.TransformChildren("settings.message.variables", transformVariables) - config.Layer.Unmarshal("settings", &settings) ENV.SETTINGS["*"] = &settings @@ -152,8 +139,4 @@ func LoadConfig() { log.Error("Could not Load Config ", ENV.CONFIG_PATH, ": ", err.Error()) } -} - -func transformVariables(key string, value any) (string, any) { - return strings.ToUpper(key), value -} +} \ No newline at end of file diff --git a/internals/config/tokens.go b/internals/config/tokens.go index fdb6d8b3..d3ff44ce 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -4,8 +4,11 @@ import ( "strconv" "github.com/codeshelldev/secured-signal-api/internals/config/structure" + "github.com/codeshelldev/secured-signal-api/utils/configutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/confmap" + "github.com/knadh/koanf/v2" ) type TOKEN_CONFIG_ struct { @@ -25,13 +28,36 @@ func LoadTokens() { tokenConf.TemplateConfig() } +func NormalizeTokens() { + // Create temporary configs + configs := koanf.New(".") + tkConfigArray := []map[string]any{} + + for _, tkConfig := range tokenConf.Layer.Slices("tokenconfigs") { + tmpConf := configutils.New() + tmpConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) + + tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) + + tkConfigArray = append(tkConfigArray, tkConfig.All()) + } + + // Merge token configs together into new temporary config + configs.Set("tokenconfigs", tkConfigArray) + + // Lowercase actual configs + LowercaseKeys(tokenConf) + + // Load temporary configs back into paths + tokenConf.Layer.Delete("") + tokenConf.Layer.Load(confmap.Provider(configs.All(), "."), nil) +} + func InitTokens() { apiTokens := config.Layer.Strings("api.tokens") var tokenConfigs []TOKEN_CONFIG_ - //tokenConf.TransformChildrenUnderArray("tokenconfigs", "overrides.message.variables", transformVariables) - tokenConf.Layer.Unmarshal("tokenconfigs", &tokenConfigs) overrides := parseTokenConfigs(tokenConfigs) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 5d50bf86..866a1f20 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -1,7 +1,6 @@ package configutils import ( - "errors" "os" "path/filepath" "strings" @@ -174,81 +173,6 @@ func (config *Config) MergeLayers(layers ...*koanf.Koanf) { } } -// Transforms Children of path -func (config *Config) TransformChildren(path string, transform func(key string, value any) (string, any)) error { - var sub map[string]any - - if !config.Layer.Exists(path) { - return errors.New("invalid path") - } - - err := config.Layer.Unmarshal(path, &sub) - - if err != nil { - return err - } - - transformed := make(map[string]any) - - for key, val := range sub { - newKey, newVal := transform(key, val) - - transformed[newKey] = newVal - } - - config.Layer.Delete(path) - - config.Layer.Load(confmap.Provider(map[string]any{ - path: transformed, - }, "."), nil) - - return nil -} - -// Does the same thing as transformChildren() but does it for each Array Item inside of root and transforms subPath -func (config *Config) TransformChildrenUnderArray(root string, subPath string, transform func(key string, value any) (string, any)) error { - var array []map[string]any - - err := config.Layer.Unmarshal(root, &array) - if err != nil { - return err - } - - transformed := []map[string]any{} - - for _, data := range array { - tmp := New() - - tmp.Layer.Load(confmap.Provider(map[string]any{ - "item": data, - }, "."), nil) - - err := tmp.TransformChildren("item."+subPath, transform) - - if err != nil { - return err - } - - item := tmp.Layer.Get("item") - - if item != nil { - itemMap, ok := item.(map[string]any) - - if ok { - transformed = append(transformed, itemMap) - } - } - } - - config.Layer.Delete(root) - - config.Layer.Load(confmap.Provider(map[string]any{ - root: transformed, - }, "."), nil) - - return nil -} - func (config *Config) NormalizeEnv(key string, value string) (string, any) { key = strings.ToLower(key) key = strings.ReplaceAll(key, "__", ".") diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 751cae10..9820ed21 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,7 +6,6 @@ import ( "strconv" "strings" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -100,8 +99,6 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma _, res := applyTransform("", data, transformTargets, funcs) - log.Dev("Result:\n-----------------------------\n", jsonutils.ToJson(res)) - mapRes, ok := res.(map[string]any) if !ok { From f1ec3ef30fa1252ff9191fc17d0e170c2b9951d6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:16:39 +0100 Subject: [PATCH 41/72] debug duplicate / old data in result --- internals/config/structure/structure.go | 22 +++++++++++----------- utils/configutils/transform.go | 9 ++++----- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/internals/config/structure/structure.go b/internals/config/structure/structure.go index 15f0d8d3..d9f2120e 100644 --- a/internals/config/structure/structure.go +++ b/internals/config/structure/structure.go @@ -14,27 +14,27 @@ type ENV struct { } type SETTINGS struct { - ACCESS ACCESS_SETTINGS `koanf:"access" transform:"lower"` - MESSAGE MESSAGE_SETTINGS `koanf:"message" transform:"lower"` + ACCESS ACCESS_SETTINGS `koanf:"access" transform:"lower"` + MESSAGE MESSAGE_SETTINGS `koanf:"message" transform:"lower"` } type MESSAGE_SETTINGS struct { - VARIABLES map[string]any `koanf:"variables" childtransform:"upper"` - FIELD_MAPPINGS map[string][]FieldMapping `koanf:"fieldmappings" childtransform:"default"` - TEMPLATE string `koanf:"template" transform:"lower"` + VARIABLES map[string]any `koanf:"variables" childtransform:"upper"` + FIELD_MAPPINGS map[string][]FieldMapping `koanf:"fieldmappings" childtransform:"default"` + TEMPLATE string `koanf:"template" transform:"lower"` } type FieldMapping struct { - Field string `koanf:"field" transform:"lower"` - Score int `koanf:"score" transform:"lower"` + Field string `koanf:"field" transform:"lower"` + Score int `koanf:"score" transform:"lower"` } type ACCESS_SETTINGS struct { - ENDPOINTS []string `koanf:"endpoints" transform:"lower"` - FIELD_POLICIES map[string]FieldPolicy `koanf:"fieldpolicies" childtransform:"default"` + ENDPOINTS []string `koanf:"endpoints" transform:"lower"` + FIELD_POLICIES map[string]FieldPolicy `koanf:"fieldpolicies" transform:"lower" childtransform:"default"` } type FieldPolicy struct { - Value any `koanf:"value" transform:"lower"` - Action string `koanf:"action" transform:"lower"` + Value any `koanf:"value" transform:"lower"` + Action string `koanf:"action" transform:"lower"` } \ No newline at end of file diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 9820ed21..84c4e168 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,7 +6,8 @@ import ( "strconv" "strings" - log "github.com/codeshelldev/secured-signal-api/utils/logger" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" + "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -50,8 +51,6 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { transformTag := field.Tag.Get("transform") childTransformTag := field.Tag.Get("childtransform") - log.Dev("Registering ", lower, " with ", transformTag, " and ", childTransformTag) - data[lower] = TransformTarget{ Key: lower, Transform: transformTag, @@ -62,8 +61,6 @@ func GetKeyToTransformMap(value any) map[string]TransformTarget { // Recursively walk nested structs if fieldValue.Kind() == reflect.Struct || (fieldValue.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) { - log.Dev("Recursively walking ", lower) - sub := GetKeyToTransformMap(fieldValue.Interface()) for subKey, subValue := range sub { @@ -105,6 +102,8 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma return } + logger.Dev("Result:\n----------------------------------\n", jsonutils.ToJson(mapRes), "\n----------------------------------") + config.Layer.Delete(path) config.Layer.Load(confmap.Provider(mapRes, "."), nil) } From 135298b06d469a064a2f2160ebaf3c65be767fad Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:19:44 +0100 Subject: [PATCH 42/72] remove unneeded debug --- utils/configutils/configutils.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 866a1f20..7434c939 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -25,9 +25,7 @@ type Config struct { func New() *Config { return &Config{ Layer: koanf.New("."), - LoadFunc: func() { - log.Dev("Config.LoadFunc not initialized!") - }, + LoadFunc: func() {}, } } From e8b4d300186777bccb5079600a50d66279c20fba Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:27:40 +0100 Subject: [PATCH 43/72] debugging loading via confmap --- internals/config/loader.go | 2 ++ internals/config/tokens.go | 1 + 2 files changed, 3 insertions(+) diff --git a/internals/config/loader.go b/internals/config/loader.go index 45c644fc..b031fd40 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -89,6 +89,8 @@ func NormalizeConfig() { // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) + log.Dev("Transform:\n--------------------------------------\n", jsonutils.ToJson(tmpConf.Layer.All()), "\n--------------------------------------") + // Lowercase actual configs LowercaseKeys(config) diff --git a/internals/config/tokens.go b/internals/config/tokens.go index d3ff44ce..7393c0ec 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -25,6 +25,7 @@ func LoadTokens() { log.Error("Could not Load Configs in ", ENV.TOKENS_DIR, ": ", err.Error()) } + tokenConf.OnLoad(Load) tokenConf.TemplateConfig() } From 598e023d91961d07184e56474388cfaf01e212bc Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:31:57 +0100 Subject: [PATCH 44/72] compare before and after deletion --- internals/config/loader.go | 7 +++++-- utils/configutils/transform.go | 4 ---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index b031fd40..2dae5129 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -89,14 +89,17 @@ func NormalizeConfig() { // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) - log.Dev("Transform:\n--------------------------------------\n", jsonutils.ToJson(tmpConf.Layer.All()), "\n--------------------------------------") - // Lowercase actual configs LowercaseKeys(config) // Load temporary configs back into paths config.Layer.Delete("settings") + + log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) + + log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") } func InitReload() { diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 84c4e168..edecd728 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" - "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -102,8 +100,6 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma return } - logger.Dev("Result:\n----------------------------------\n", jsonutils.ToJson(mapRes), "\n----------------------------------") - config.Layer.Delete(path) config.Layer.Load(confmap.Provider(mapRes, "."), nil) } From 04af17cb82f0946f30737e1f7dd136c7a5d2b72a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:48:59 +0100 Subject: [PATCH 45/72] added custom `Delete()` for supporting unflattened paths --- internals/config/loader.go | 2 +- utils/configutils/configutils.go | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 2dae5129..f2f1b061 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -93,7 +93,7 @@ func NormalizeConfig() { LowercaseKeys(config) // Load temporary configs back into paths - config.Layer.Delete("settings") + config.Delete("settings") log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 7434c939..64a615ce 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -1,6 +1,7 @@ package configutils import ( + "errors" "os" "path/filepath" "strings" @@ -104,6 +105,24 @@ func (config *Config) Unflatten(path string) map[string]any { return res } +func (config *Config) Delete(path string) (error) { + if !config.Layer.Exists(path) { + return errors.New("path not found") + } + + data := config.Layer.All() + + if data == nil { + return errors.New("empty config") + } + + delete(data, path) + + config.Layer.Delete("") + config.Layer.Load(confmap.Provider(data, "."), nil) + + return nil +} func (config *Config) LoadDir(path string, dir string, ext string, parser koanf.Parser) error { files, err := filepath.Glob(filepath.Join(dir, "*" + ext)) From 26a4b9b21ca23aba15fd56aad6adce50080093f2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:55:31 +0100 Subject: [PATCH 46/72] unflatten data and debug --- internals/config/loader.go | 4 +--- utils/configutils/configutils.go | 7 ++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index f2f1b061..5659dc2d 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -95,11 +95,9 @@ func NormalizeConfig() { // Load temporary configs back into paths config.Delete("settings") - log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) - - log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") } func InitReload() { diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 64a615ce..92065b10 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" stringutils "github.com/codeshelldev/secured-signal-api/utils/stringutils" @@ -110,14 +111,18 @@ func (config *Config) Delete(path string) (error) { return errors.New("path not found") } - data := config.Layer.All() + data := config.Unflatten(path) if data == nil { return errors.New("empty config") } + log.Dev("Unflattened:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") + delete(data, path) + log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") + config.Layer.Delete("") config.Layer.Load(confmap.Provider(data, "."), nil) From 6cfd66b6c6fd9774c7dbaffb45873b2318c4399c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:04:44 +0100 Subject: [PATCH 47/72] fix recursive deletion --- utils/configutils/configutils.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 92065b10..c7862131 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -111,20 +111,32 @@ func (config *Config) Delete(path string) (error) { return errors.New("path not found") } - data := config.Unflatten(path) + all := config.Unflatten("") - if data == nil { + if all == nil { return errors.New("empty config") } - log.Dev("Unflattened:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") + keyParts := getKeyParts(path) - delete(data, path) + for i := 0; i < len(keyParts) - 1; i++ { + next, ok := all[keyParts[i]].(map[string]any) - log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") + if !ok { + return nil + } + + all = next + } + + log.Dev("Unflattened:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") + + delete(all, keyParts[len(keyParts)-1]) + + log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") config.Layer.Delete("") - config.Layer.Load(confmap.Provider(data, "."), nil) + config.Layer.Load(confmap.Provider(all, "."), nil) return nil } From d92e231ddee3d049029ec83f6c31ad50c64854f0 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:18:39 +0100 Subject: [PATCH 48/72] use flattened keys for deletion --- utils/configutils/configutils.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index c7862131..ca0c11fb 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -90,7 +90,7 @@ func (config *Config) Unflatten(path string) map[string]any { sub := res for i, part := range parts { - if i == len(parts)-1 { + if i == len(parts) - 1 { sub[part] = value } else { _, ok := sub[part] @@ -111,28 +111,18 @@ func (config *Config) Delete(path string) (error) { return errors.New("path not found") } - all := config.Unflatten("") + all := config.Layer.All() if all == nil { return errors.New("empty config") } - keyParts := getKeyParts(path) - - for i := 0; i < len(keyParts) - 1; i++ { - next, ok := all[keyParts[i]].(map[string]any) - - if !ok { - return nil + for key := range all { + if strings.HasPrefix(key, path + ".") || key == path { + delete(all, key) } - - all = next } - log.Dev("Unflattened:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") - - delete(all, keyParts[len(keyParts)-1]) - log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") config.Layer.Delete("") From c84a2fd20a2ff5ff0ab5ab944b9dd697be467b1a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:25:55 +0100 Subject: [PATCH 49/72] more debugging --- utils/configutils/configutils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index ca0c11fb..ed8062aa 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -112,6 +112,9 @@ func (config *Config) Delete(path string) (error) { } all := config.Layer.All() + + log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") + log.Dev("Init:Sprint():\n--------------------------------------\n", config.Layer.Sprint(), "\n--------------------------------------") if all == nil { return errors.New("empty config") From 8abdd35c1eaf0631c2ebde3810e0456608b02c19 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:32:19 +0100 Subject: [PATCH 50/72] investigating `Unflatten()` incorrectly unflattening .../ not being needed / causing issues --- utils/configutils/transform.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index edecd728..abb7cd97 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -6,6 +6,9 @@ import ( "strconv" "strings" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" + "github.com/codeshelldev/secured-signal-api/utils/logger" + log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -91,8 +94,11 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma transformTargets := GetKeyToTransformMap(structSchema) data := config.Unflatten(path) + + log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") + logger.Dev("Init:Sprint():\n--------------------------------------\n", config.Layer.Sprint(), "\n--------------------------------------") - _, res := applyTransform("", data, transformTargets, funcs) + _, res := applyTransform("", config.Layer.Get(path), transformTargets, funcs) mapRes, ok := res.(map[string]any) From fc356f599116b2111a7bad40b06d2927da3f8bb6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:44:26 +0100 Subject: [PATCH 51/72] save data in subpath --- utils/configutils/transform.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index abb7cd97..0ee323bf 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/codeshelldev/secured-signal-api/utils/jsonutils" - "github.com/codeshelldev/secured-signal-api/utils/logger" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/providers/confmap" ) @@ -95,8 +94,8 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma data := config.Unflatten(path) - log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") - logger.Dev("Init:Sprint():\n--------------------------------------\n", config.Layer.Sprint(), "\n--------------------------------------") + log.Dev("Unflatten+Get:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") + log.Dev("Get:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.Get("")), "\n--------------------------------------") _, res := applyTransform("", config.Layer.Get(path), transformTargets, funcs) @@ -106,6 +105,12 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma return } + if path != "" { + mapRes = map[string]any { + path: mapRes, + } + } + config.Layer.Delete(path) config.Layer.Load(confmap.Provider(mapRes, "."), nil) } From d24361527bf1c3f779c913287fbe7afbd6a7acb3 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:44:54 +0100 Subject: [PATCH 52/72] delete whole structure --- utils/configutils/transform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 0ee323bf..65a5fae4 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -111,7 +111,7 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma } } - config.Layer.Delete(path) + config.Layer.Delete("") config.Layer.Load(confmap.Provider(mapRes, "."), nil) } From 653b312c9148552e203de4cb05c4e4b1c8be433c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 18:59:15 +0100 Subject: [PATCH 53/72] investigate settingsdata being outside of `settings` --- internals/config/loader.go | 2 ++ utils/configutils/configutils.go | 42 +++----------------------------- utils/configutils/transform.go | 5 ++-- 3 files changed, 7 insertions(+), 42 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 5659dc2d..aecd15d4 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -74,6 +74,8 @@ func LowercaseKeys(config *configutils.Config) { } func NormalizeConfig() { + log.Dev("Normilization:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + settings := config.Layer.Get("settings") old, ok := settings.(map[string]any) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index ed8062aa..f7eafec8 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -74,63 +74,27 @@ func getPath(str string) string { return str } -func (config *Config) Unflatten(path string) map[string]any { - data := config.Layer.Get(path) - all, ok := data.(map[string]any) - - if !ok { - return nil - } - - res := map[string]any{} - - for key, value := range all { - parts := strings.Split(key, ".") - - sub := res - - for i, part := range parts { - if i == len(parts) - 1 { - sub[part] = value - } else { - _, ok := sub[part] - - if !ok { - sub[part] = map[string]any{} - } - sub = sub[part].(map[string]any) - } - } - } - - return res -} - func (config *Config) Delete(path string) (error) { if !config.Layer.Exists(path) { return errors.New("path not found") } - all := config.Layer.All() + all := config.Layer.Get("") log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") - log.Dev("Init:Sprint():\n--------------------------------------\n", config.Layer.Sprint(), "\n--------------------------------------") if all == nil { return errors.New("empty config") } - for key := range all { + for _, key := range config.Layer.Keys() { if strings.HasPrefix(key, path + ".") || key == path { - delete(all, key) + config.Layer.Delete(key) } } log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") - config.Layer.Delete("") - config.Layer.Load(confmap.Provider(all, "."), nil) - return nil } diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 65a5fae4..4110c04e 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -92,10 +92,9 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma transformTargets := GetKeyToTransformMap(structSchema) - data := config.Unflatten(path) + data := config.Layer.Get(path) - log.Dev("Unflatten+Get:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") - log.Dev("Get:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.Get("")), "\n--------------------------------------") + log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") _, res := applyTransform("", config.Layer.Get(path), transformTargets, funcs) From f70457b33518dec899cd9a26d66f92e3a18b498a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:06:10 +0100 Subject: [PATCH 54/72] searching deeper for incorrect unmarshaling of settings --- internals/config/loader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internals/config/loader.go b/internals/config/loader.go index aecd15d4..c8fe8663 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -125,6 +125,8 @@ func InitEnv() { func LoadDefaults() { _, err := defaultsConf.LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) + log.Dev("Defaults:\n--------------------------------------\n", jsonutils.ToJson(defaultsConf.Layer.All()), "\n--------------------------------------") + if err != nil { log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) } @@ -133,6 +135,8 @@ func LoadDefaults() { func LoadConfig() { _, err := userConf.LoadFile(ENV.CONFIG_PATH, yaml.Parser()) + log.Dev("User:\n--------------------------------------\n", jsonutils.ToJson(userConf.Layer.All()), "\n--------------------------------------") + if err != nil { _, fsErr := os.Stat(ENV.CONFIG_PATH) From 18cf068be7c05edcfc8f44295fe599f031a4a943 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:10:36 +0100 Subject: [PATCH 55/72] checking for issues in mergelayers --- utils/configutils/configutils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index f7eafec8..64b594d5 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -162,6 +162,8 @@ func (config *Config) MergeLayers(layers ...*koanf.Koanf) { for _, layer := range layers { config.Layer.Merge(layer) } + + log.Dev("Merge:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") } func (config *Config) NormalizeEnv(key string, value string) (string, any) { From e45eccc7e3fa0858cd8dea8ad5c33a93e31e4491 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:30:38 +0100 Subject: [PATCH 56/72] found issues at mergelayers --- utils/configutils/configutils.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 64b594d5..541feac3 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" "sync" @@ -159,7 +160,8 @@ func (config *Config) TemplateConfig() { } func (config *Config) MergeLayers(layers ...*koanf.Koanf) { - for _, layer := range layers { + for i, layer := range layers { + log.Dev("Merge[", strconv.Itoa(i), "]:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") config.Layer.Merge(layer) } From 9d4b0fd6b225f3b2869c379f87ca3d3f46fa1c54 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:35:48 +0100 Subject: [PATCH 57/72] rename to mainConf --- internals/config/loader.go | 28 ++++++++++++++-------------- internals/config/tokens.go | 4 ++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index c8fe8663..cf4e7332 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -30,7 +30,7 @@ var defaultsConf = configutils.New() var userConf = configutils.New() var tokenConf = configutils.New() -var config = configutils.New() +var mainConf = configutils.New() func Load() { InitReload() @@ -43,12 +43,12 @@ func Load() { userConf.LoadEnv() - config.MergeLayers(defaultsConf.Layer, userConf.Layer) + mainConf.MergeLayers(defaultsConf.Layer, userConf.Layer) NormalizeConfig() NormalizeTokens() - config.TemplateConfig() + mainConf.TemplateConfig() InitTokens() @@ -56,7 +56,7 @@ func Load() { log.Info("Finished Loading Configuration") - log.Dev("Loaded Config:\n" + jsonutils.ToJson(config.Layer.All())) + log.Dev("Loaded Config:\n" + jsonutils.ToJson(mainConf.Layer.All())) log.Dev("Loaded Token Configs:\n" + jsonutils.ToJson(tokenConf.Layer.All())) } @@ -74,9 +74,9 @@ func LowercaseKeys(config *configutils.Config) { } func NormalizeConfig() { - log.Dev("Normilization:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + log.Dev("Normilization:\n--------------------------------------\n", jsonutils.ToJson(mainConf.Layer.All()), "\n--------------------------------------") - settings := config.Layer.Get("settings") + settings := mainConf.Layer.Get("settings") old, ok := settings.(map[string]any) if !ok { @@ -92,14 +92,14 @@ func NormalizeConfig() { tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) // Lowercase actual configs - LowercaseKeys(config) + LowercaseKeys(mainConf) // Load temporary configs back into paths - config.Delete("settings") + mainConf.Delete("settings") - log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(mainConf.Layer.All()), "\n--------------------------------------") - config.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) + mainConf.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) } func InitReload() { @@ -109,15 +109,15 @@ func InitReload() { } func InitEnv() { - ENV.PORT = strconv.Itoa(config.Layer.Int("service.port")) + ENV.PORT = strconv.Itoa(mainConf.Layer.Int("service.port")) - ENV.LOG_LEVEL = strings.ToLower(config.Layer.String("loglevel")) + ENV.LOG_LEVEL = strings.ToLower(mainConf.Layer.String("loglevel")) - ENV.API_URL = config.Layer.String("api.url") + ENV.API_URL = mainConf.Layer.String("api.url") var settings structure.SETTINGS - config.Layer.Unmarshal("settings", &settings) + mainConf.Layer.Unmarshal("settings", &settings) ENV.SETTINGS["*"] = &settings } diff --git a/internals/config/tokens.go b/internals/config/tokens.go index 7393c0ec..2628aa06 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -55,7 +55,7 @@ func NormalizeTokens() { } func InitTokens() { - apiTokens := config.Layer.Strings("api.tokens") + apiTokens := mainConf.Layer.Strings("api.tokens") var tokenConfigs []TOKEN_CONFIG_ @@ -78,7 +78,7 @@ func InitTokens() { // Set Blocked Endpoints on Config to User Layer Value // => effectively ignoring Default Layer - config.Layer.Set("settings.access.endpoints", userConf.Layer.Strings("settings.access.endpoints")) + mainConf.Layer.Set("settings.access.endpoints", userConf.Layer.Strings("settings.access.endpoints")) } if len(apiTokens) > 0 { From 19aea10cc1df021314015778032811e5ce14c07d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:48:21 +0100 Subject: [PATCH 58/72] clear configs before reload --- internals/config/loader.go | 9 +++++++++ utils/configutils/configutils.go | 2 ++ 2 files changed, 11 insertions(+) diff --git a/internals/config/loader.go b/internals/config/loader.go index cf4e7332..e4f4077d 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -33,6 +33,8 @@ var tokenConf = configutils.New() var mainConf = configutils.New() func Load() { + Clear() + InitReload() LoadDefaults() @@ -60,6 +62,13 @@ func Load() { log.Dev("Loaded Token Configs:\n" + jsonutils.ToJson(tokenConf.Layer.All())) } +func Clear() { + defaultsConf = configutils.New() + userConf = configutils.New() + tokenConf = configutils.New() + mainConf = configutils.New() +} + func LowercaseKeys(config *configutils.Config) { data := map[string]any{} diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 541feac3..1ce96c1c 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -160,6 +160,8 @@ func (config *Config) TemplateConfig() { } func (config *Config) MergeLayers(layers ...*koanf.Koanf) { + log.Dev("Before:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + for i, layer := range layers { log.Dev("Merge[", strconv.Itoa(i), "]:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") config.Layer.Merge(layer) From 9a991302e9f8f263d46570267661c51f41f7acfa Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:52:15 +0100 Subject: [PATCH 59/72] test again with fixed reloading --- internals/config/loader.go | 4 +--- utils/configutils/configutils.go | 8 +------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index e4f4077d..fac9522a 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -83,8 +83,6 @@ func LowercaseKeys(config *configutils.Config) { } func NormalizeConfig() { - log.Dev("Normilization:\n--------------------------------------\n", jsonutils.ToJson(mainConf.Layer.All()), "\n--------------------------------------") - settings := mainConf.Layer.Get("settings") old, ok := settings.(map[string]any) @@ -104,7 +102,7 @@ func NormalizeConfig() { LowercaseKeys(mainConf) // Load temporary configs back into paths - mainConf.Delete("settings") + mainConf.Layer.Delete("settings") log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(mainConf.Layer.All()), "\n--------------------------------------") diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 1ce96c1c..f7eafec8 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -4,7 +4,6 @@ import ( "errors" "os" "path/filepath" - "strconv" "strings" "sync" @@ -160,14 +159,9 @@ func (config *Config) TemplateConfig() { } func (config *Config) MergeLayers(layers ...*koanf.Koanf) { - log.Dev("Before:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") - - for i, layer := range layers { - log.Dev("Merge[", strconv.Itoa(i), "]:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") + for _, layer := range layers { config.Layer.Merge(layer) } - - log.Dev("Merge:\n--------------------------------------\n", jsonutils.ToJson(config.Layer.All()), "\n--------------------------------------") } func (config *Config) NormalizeEnv(key string, value string) (string, any) { From 1780b9d819c07aa8aaf63f85b6575e2a5950b8d0 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 20:01:15 +0100 Subject: [PATCH 60/72] unwatch after reload --- internals/config/loader.go | 12 ++++-------- utils/configutils/configutils.go | 2 ++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index fac9522a..31cac0a0 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -26,11 +26,11 @@ var ENV *structure.ENV = &structure.ENV{ INSECURE: false, } -var defaultsConf = configutils.New() -var userConf = configutils.New() -var tokenConf = configutils.New() +var defaultsConf *configutils.Config +var userConf *configutils.Config +var tokenConf *configutils.Config -var mainConf = configutils.New() +var mainConf *configutils.Config func Load() { Clear() @@ -132,8 +132,6 @@ func InitEnv() { func LoadDefaults() { _, err := defaultsConf.LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) - log.Dev("Defaults:\n--------------------------------------\n", jsonutils.ToJson(defaultsConf.Layer.All()), "\n--------------------------------------") - if err != nil { log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) } @@ -142,8 +140,6 @@ func LoadDefaults() { func LoadConfig() { _, err := userConf.LoadFile(ENV.CONFIG_PATH, yaml.Parser()) - log.Dev("User:\n--------------------------------------\n", jsonutils.ToJson(userConf.Layer.All()), "\n--------------------------------------") - if err != nil { _, fsErr := os.Stat(ENV.CONFIG_PATH) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index f7eafec8..e5dff82e 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -62,6 +62,8 @@ func WatchFile(path string, f *file.File, loadFunc func()) { configLock.Lock() defer configLock.Unlock() + f.Unwatch() + loadFunc() }) } From a24a1bbd1f0914784703ad262e2498a39fb6f4ee Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 20:37:51 +0100 Subject: [PATCH 61/72] added `config.Load(data, path)` to replace confmap procedures --- internals/config/loader.go | 7 +++---- internals/config/tokens.go | 14 ++++++-------- utils/configutils/configutils.go | 26 +++++++++++++++++++++++--- utils/configutils/transform.go | 9 +-------- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 31cac0a0..0cf40635 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -13,7 +13,6 @@ import ( log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/confmap" ) var ENV *structure.ENV = &structure.ENV{ @@ -79,7 +78,7 @@ func LowercaseKeys(config *configutils.Config) { } config.Layer.Delete("") - config.Layer.Load(confmap.Provider(data, "."), nil) + config.Load(data, "") } func NormalizeConfig() { @@ -93,7 +92,7 @@ func NormalizeConfig() { // Create temporary configs tmpConf := configutils.New() - tmpConf.Layer.Load(confmap.Provider(old, "."), nil) + tmpConf.Load(old, "") // Apply transforms to the new configs tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) @@ -106,7 +105,7 @@ func NormalizeConfig() { log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(mainConf.Layer.All()), "\n--------------------------------------") - mainConf.Layer.Load(confmap.Provider(tmpConf.Layer.All(), "settings"), nil) + mainConf.Load(tmpConf.Layer.All(), "settings") } func InitReload() { diff --git a/internals/config/tokens.go b/internals/config/tokens.go index 2628aa06..fc818d08 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -7,7 +7,6 @@ import ( "github.com/codeshelldev/secured-signal-api/utils/configutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/confmap" "github.com/knadh/koanf/v2" ) @@ -25,33 +24,32 @@ func LoadTokens() { log.Error("Could not Load Configs in ", ENV.TOKENS_DIR, ": ", err.Error()) } - tokenConf.OnLoad(Load) tokenConf.TemplateConfig() } func NormalizeTokens() { // Create temporary configs configs := koanf.New(".") - tkConfigArray := []map[string]any{} + configArray := []map[string]any{} - for _, tkConfig := range tokenConf.Layer.Slices("tokenconfigs") { + for _, config := range tokenConf.Layer.Slices("tokenconfigs") { tmpConf := configutils.New() - tmpConf.Layer.Load(confmap.Provider(tkConfig.All(), "."), nil) + tmpConf.Load(config.All(), "") tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) - tkConfigArray = append(tkConfigArray, tkConfig.All()) + configArray = append(configArray, config.All()) } // Merge token configs together into new temporary config - configs.Set("tokenconfigs", tkConfigArray) + configs.Set("tokenconfigs", configArray) // Lowercase actual configs LowercaseKeys(tokenConf) // Load temporary configs back into paths tokenConf.Layer.Delete("") - tokenConf.Layer.Load(confmap.Provider(configs.All(), "."), nil) + tokenConf.Load(configs.All(), "") } func InitTokens() { diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index e5dff82e..a56cf735 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -76,12 +76,32 @@ func getPath(str string) string { return str } +func (config *Config) Load(data map[string]any, path string) error { + parts := strings.Split(path, ".") + + res := map[string]any{} + + for i, key := range parts { + if i == 0 { + res[key] = data + } else { + sub := map[string]any{} + + sub[key] = res + + res = sub + } + } + + return config.Layer.Load(confmap.Provider(res, "."), nil) +} + func (config *Config) Delete(path string) (error) { if !config.Layer.Exists(path) { return errors.New("path not found") } - all := config.Layer.Get("") + all := config.Layer.All() log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") @@ -125,7 +145,7 @@ func (config *Config) LoadDir(path string, dir string, ext string, parser koanf. path: array, } - return config.Layer.Load(confmap.Provider(wrapper, "."), nil) + return config.Load(wrapper, "") } func (config *Config) LoadEnv() (koanf.Provider, error) { @@ -157,7 +177,7 @@ func (config *Config) TemplateConfig() { } } - config.Layer.Load(confmap.Provider(data, "."), nil) + config.Load(data, "") } func (config *Config) MergeLayers(layers ...*koanf.Koanf) { diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 4110c04e..964c5ac1 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -8,7 +8,6 @@ import ( "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" - "github.com/knadh/koanf/providers/confmap" ) type TransformTarget struct { @@ -104,14 +103,8 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma return } - if path != "" { - mapRes = map[string]any { - path: mapRes, - } - } - config.Layer.Delete("") - config.Layer.Load(confmap.Provider(mapRes, "."), nil) + config.Load(mapRes, path) } func applyTransform(key string, value any, transformTargets map[string]TransformTarget, funcs map[string]func(string, any) (string, any)) (string, any) { From 643431b5831a75806be970ed94e9e7466b54f65c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 20:47:25 +0100 Subject: [PATCH 62/72] debugging new Load func --- utils/configutils/configutils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index a56cf735..5ef2a4e0 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -93,6 +93,8 @@ func (config *Config) Load(data map[string]any, path string) error { } } + log.Info("Load:\n--------------------------------------\n", jsonutils.ToJson(res), "\n--------------------------------------") + return config.Layer.Load(confmap.Provider(res, "."), nil) } From e6491fd815b14e27b40a92966a8dfb86c722c05a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 20:53:14 +0100 Subject: [PATCH 63/72] fix empty key --- utils/configutils/configutils.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 5ef2a4e0..1f0d41df 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -81,18 +81,22 @@ func (config *Config) Load(data map[string]any, path string) error { res := map[string]any{} - for i, key := range parts { - if i == 0 { - res[key] = data - } else { - sub := map[string]any{} - - sub[key] = res - - res = sub + if path == "" { + res = data + } else { + for i, key := range parts { + if i == 0 { + res[key] = data + } else { + sub := map[string]any{} + + sub[key] = res + + res = sub + } } } - + log.Info("Load:\n--------------------------------------\n", jsonutils.ToJson(res), "\n--------------------------------------") return config.Layer.Load(confmap.Provider(res, "."), nil) From bd087e3ac6f59bcf9de9a8fb3afc6b0275d450a9 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 31 Oct 2025 20:57:27 +0100 Subject: [PATCH 64/72] remove debugs / cleanup --- internals/config/loader.go | 4 +--- utils/configutils/configutils.go | 7 ------- utils/configutils/transform.go | 7 +------ 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 0cf40635..fbc1e5fb 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -102,9 +102,7 @@ func NormalizeConfig() { // Load temporary configs back into paths mainConf.Layer.Delete("settings") - - log.Dev("Loading:\n--------------------------------------\n", jsonutils.ToJson(mainConf.Layer.All()), "\n--------------------------------------") - + mainConf.Load(tmpConf.Layer.All(), "settings") } diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index 1f0d41df..ad4ad2ed 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -7,7 +7,6 @@ import ( "strings" "sync" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" stringutils "github.com/codeshelldev/secured-signal-api/utils/stringutils" @@ -96,8 +95,6 @@ func (config *Config) Load(data map[string]any, path string) error { } } } - - log.Info("Load:\n--------------------------------------\n", jsonutils.ToJson(res), "\n--------------------------------------") return config.Layer.Load(confmap.Provider(res, "."), nil) } @@ -108,8 +105,6 @@ func (config *Config) Delete(path string) (error) { } all := config.Layer.All() - - log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") if all == nil { return errors.New("empty config") @@ -121,8 +116,6 @@ func (config *Config) Delete(path string) (error) { } } - log.Dev("Deletion:\n--------------------------------------\n", jsonutils.ToJson(all), "\n--------------------------------------") - return nil } diff --git a/utils/configutils/transform.go b/utils/configutils/transform.go index 964c5ac1..7b812e08 100644 --- a/utils/configutils/transform.go +++ b/utils/configutils/transform.go @@ -5,9 +5,6 @@ import ( "reflect" "strconv" "strings" - - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" - log "github.com/codeshelldev/secured-signal-api/utils/logger" ) type TransformTarget struct { @@ -93,9 +90,7 @@ func (config Config) ApplyTransformFuncs(structSchema any, path string, funcs ma data := config.Layer.Get(path) - log.Dev("Init:\n--------------------------------------\n", jsonutils.ToJson(data), "\n--------------------------------------") - - _, res := applyTransform("", config.Layer.Get(path), transformTargets, funcs) + _, res := applyTransform("", data, transformTargets, funcs) mapRes, ok := res.(map[string]any) From e2cde3598018054ffc5ff5cafe25059c6b71f785 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:00:23 +0100 Subject: [PATCH 65/72] merge after normilization --- internals/config/loader.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index fbc1e5fb..76c2887c 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -44,11 +44,13 @@ func Load() { userConf.LoadEnv() - mainConf.MergeLayers(defaultsConf.Layer, userConf.Layer) - - NormalizeConfig() + NormalizeConfig(defaultsConf) + NormalizeConfig(userConf) + NormalizeTokens() + mainConf.MergeLayers(defaultsConf.Layer, userConf.Layer) + mainConf.TemplateConfig() InitTokens() @@ -81,8 +83,8 @@ func LowercaseKeys(config *configutils.Config) { config.Load(data, "") } -func NormalizeConfig() { - settings := mainConf.Layer.Get("settings") +func NormalizeConfig(config *configutils.Config) { + settings := config.Layer.Get("settings") old, ok := settings.(map[string]any) if !ok { @@ -98,12 +100,12 @@ func NormalizeConfig() { tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) // Lowercase actual configs - LowercaseKeys(mainConf) + LowercaseKeys(config) // Load temporary configs back into paths - mainConf.Layer.Delete("settings") + config.Layer.Delete("settings") - mainConf.Load(tmpConf.Layer.All(), "settings") + config.Load(tmpConf.Layer.All(), "settings") } func InitReload() { From 79832154d788eee17005c2458309225b697756d0 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:07:26 +0100 Subject: [PATCH 66/72] fix reloading for tokens --- utils/configutils/configutils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/configutils/configutils.go b/utils/configutils/configutils.go index ad4ad2ed..b0686aa8 100644 --- a/utils/configutils/configutils.go +++ b/utils/configutils/configutils.go @@ -131,6 +131,8 @@ func (config *Config) LoadDir(path string, dir string, ext string, parser koanf. for _, f := range files { tmp := New() + tmp.OnLoad(config.LoadFunc) + _, err := tmp.LoadFile(f, parser) if err != nil { From ed65286b6e84c6cdcae46b7210af2d652b0a05e6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:14:13 +0100 Subject: [PATCH 67/72] updated tokens normilization --- internals/config/loader.go | 24 ++++++++++++++---------- internals/config/tokens.go | 11 +++-------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/internals/config/loader.go b/internals/config/loader.go index 76c2887c..fee3b818 100644 --- a/internals/config/loader.go +++ b/internals/config/loader.go @@ -84,28 +84,32 @@ func LowercaseKeys(config *configutils.Config) { } func NormalizeConfig(config *configutils.Config) { - settings := config.Layer.Get("settings") - old, ok := settings.(map[string]any) + Normalize(config, "settings", &structure.SETTINGS{}) +} + +func Normalize(config *configutils.Config, path string, structure any) { + data := config.Layer.Get(path) + old, ok := data.(map[string]any) if !ok { - log.Warn("Could not load `settings`") + log.Warn("Could not load `"+path+"`") return } - // Create temporary configs + // Create temporary config tmpConf := configutils.New() tmpConf.Load(old, "") - // Apply transforms to the new configs - tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "", transformFuncs) + // Apply transforms to the new config + tmpConf.ApplyTransformFuncs(structure, "", transformFuncs) - // Lowercase actual configs + // Lowercase actual config LowercaseKeys(config) - // Load temporary configs back into paths - config.Layer.Delete("settings") + // Load temporary config back into paths + config.Layer.Delete(path) - config.Load(tmpConf.Layer.All(), "settings") + config.Load(tmpConf.Layer.All(), path) } func InitReload() { diff --git a/internals/config/tokens.go b/internals/config/tokens.go index fc818d08..eb8c2f1a 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -36,20 +36,15 @@ func NormalizeTokens() { tmpConf := configutils.New() tmpConf.Load(config.All(), "") + Normalize(tmpConf, "overrides", &structure.SETTINGS{}) + tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) - configArray = append(configArray, config.All()) + configArray = append(configArray, tmpConf.Layer.All()) } // Merge token configs together into new temporary config configs.Set("tokenconfigs", configArray) - - // Lowercase actual configs - LowercaseKeys(tokenConf) - - // Load temporary configs back into paths - tokenConf.Layer.Delete("") - tokenConf.Load(configs.All(), "") } func InitTokens() { From 5d953c1cbb84f91084aa5db74503279392a41958 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:15:07 +0100 Subject: [PATCH 68/72] fix loading of tokens after normilization --- internals/config/tokens.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internals/config/tokens.go b/internals/config/tokens.go index eb8c2f1a..06af6368 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -7,7 +7,6 @@ import ( "github.com/codeshelldev/secured-signal-api/utils/configutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/v2" ) type TOKEN_CONFIG_ struct { @@ -28,8 +27,6 @@ func LoadTokens() { } func NormalizeTokens() { - // Create temporary configs - configs := koanf.New(".") configArray := []map[string]any{} for _, config := range tokenConf.Layer.Slices("tokenconfigs") { @@ -44,7 +41,7 @@ func NormalizeTokens() { } // Merge token configs together into new temporary config - configs.Set("tokenconfigs", configArray) + tokenConf.Layer.Set("tokenconfigs", configArray) } func InitTokens() { From fafe74c332cf41e5608cf06ee3d2469a2c57818b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:20:44 +0100 Subject: [PATCH 69/72] remove leftover line --- internals/config/tokens.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internals/config/tokens.go b/internals/config/tokens.go index 06af6368..592613b0 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -35,8 +35,6 @@ func NormalizeTokens() { Normalize(tmpConf, "overrides", &structure.SETTINGS{}) - tmpConf.ApplyTransformFuncs(&structure.SETTINGS{}, "overrides", transformFuncs) - configArray = append(configArray, tmpConf.Layer.All()) } From 2d8fc178c0404818fecba9693f6e6b3be243f69f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:22:11 +0100 Subject: [PATCH 70/72] renamed struct --- internals/config/tokens.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internals/config/tokens.go b/internals/config/tokens.go index 592613b0..b8e33861 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -9,7 +9,7 @@ import ( "github.com/knadh/koanf/parsers/yaml" ) -type TOKEN_CONFIG_ struct { +type TOKEN_CONFIG struct { TOKENS []string `koanf:"tokens"` OVERRIDES structure.SETTINGS `koanf:"overrides"` } @@ -45,7 +45,7 @@ func NormalizeTokens() { func InitTokens() { apiTokens := mainConf.Layer.Strings("api.tokens") - var tokenConfigs []TOKEN_CONFIG_ + var tokenConfigs []TOKEN_CONFIG tokenConf.Layer.Unmarshal("tokenconfigs", &tokenConfigs) @@ -76,7 +76,7 @@ func InitTokens() { } } -func parseTokenConfigs(configs []TOKEN_CONFIG_) map[string]structure.SETTINGS { +func parseTokenConfigs(configs []TOKEN_CONFIG) map[string]structure.SETTINGS { settings := map[string]structure.SETTINGS{} for _, config := range configs { From 803ff89beb9823bd00aa2b49650de2c063bd2575 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:22:56 +0100 Subject: [PATCH 71/72] debugging token configs --- internals/config/tokens.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internals/config/tokens.go b/internals/config/tokens.go index b8e33861..6ce7a828 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -5,6 +5,7 @@ import ( "github.com/codeshelldev/secured-signal-api/internals/config/structure" "github.com/codeshelldev/secured-signal-api/utils/configutils" + "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -35,6 +36,8 @@ func NormalizeTokens() { Normalize(tmpConf, "overrides", &structure.SETTINGS{}) + log.Dev("Tokenconfig:\n------------------------------\n", jsonutils.ToJson(tmpConf.Layer.All()), "------------------------------") + configArray = append(configArray, tmpConf.Layer.All()) } From 93aba3e4c7510ac1a83a0cdca3000583d99a3d7b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:39:53 +0100 Subject: [PATCH 72/72] remove debug --- internals/config/tokens.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internals/config/tokens.go b/internals/config/tokens.go index 6ce7a828..69ec8518 100644 --- a/internals/config/tokens.go +++ b/internals/config/tokens.go @@ -5,7 +5,6 @@ import ( "github.com/codeshelldev/secured-signal-api/internals/config/structure" "github.com/codeshelldev/secured-signal-api/utils/configutils" - "github.com/codeshelldev/secured-signal-api/utils/jsonutils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -35,9 +34,7 @@ func NormalizeTokens() { tmpConf.Load(config.All(), "") Normalize(tmpConf, "overrides", &structure.SETTINGS{}) - - log.Dev("Tokenconfig:\n------------------------------\n", jsonutils.ToJson(tmpConf.Layer.All()), "------------------------------") - + configArray = append(configArray, tmpConf.Layer.All()) }