From 1c46123addfad5138a03305e1981c91dcfd8207a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Thu, 11 Sep 2025 17:54:33 +0200 Subject: [PATCH 01/97] remove deprecation warning --- utils/env/env.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/utils/env/env.go b/utils/env/env.go index 80657b8..e61a439 100644 --- a/utils/env/env.go +++ b/utils/env/env.go @@ -3,11 +3,9 @@ package env import ( "os" "strconv" - "strings" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" "github.com/codeshelldev/secured-signal-api/utils" - "github.com/codeshelldev/secured-signal-api/utils/docker" log "github.com/codeshelldev/secured-signal-api/utils/logger" ) @@ -108,15 +106,6 @@ func Load() { } if blockedEndpointStrArray != "" { - if strings.Contains(blockedEndpointStrArray, "[") || strings.Contains(blockedEndpointStrArray, "]") { - //! Deprecated: JSON - //TODO: Remove this in new Version - - log.Error("Invalid Blocked Endpoints: ", "JSON instead of Comma seperated String") - - docker.Exit(1) - } - ENV.BLOCKED_ENDPOINTS = utils.StringToArray(blockedEndpointStrArray) } @@ -129,15 +118,6 @@ func Load() { } if recipientsStrArray != "" { - if strings.Contains(blockedEndpointStrArray, "[") || strings.Contains(blockedEndpointStrArray, "]") { - //! Deprecated: JSON - //TODO: Remove this in new Version - - log.Error("Invalid Blocked Endpoints: ", "JSON instead of Comma seperated String") - - docker.Exit(1) - } - ENV.VARIABLES["RECIPIENTS"] = utils.StringToArray(recipientsStrArray) } } \ No newline at end of file From e8913e9c564e8d821c6fabaa3e098a47fa45afcf Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:33:34 +0200 Subject: [PATCH 02/97] added yml configs (via koanf) + env overrides + defaults.yml file (TESTING) --- Dockerfile | 5 +- docker-compose.yaml | 6 +- examples/traefik.docker-compose.yaml | 6 +- go.mod | 23 ++++- go.sum | 32 +++++++ internals/proxy/middlewares/body.go | 2 +- main.go | 8 +- tests/json_test.go | 2 +- utils/config/config.go | 135 +++++++++++++++++++++++++++ utils/config/defaults.yml | 29 ++++++ utils/env/env.go | 123 ------------------------ utils/utils.go | 24 ++++- 12 files changed, 256 insertions(+), 139 deletions(-) create mode 100644 utils/config/config.go create mode 100644 utils/config/defaults.yml delete mode 100644 utils/env/env.go diff --git a/Dockerfile b/Dockerfile index dde3acf..2a762f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ FROM alpine:latest RUN apk --no-cache add ca-certificates -ENV PORT=8880 +ENV SERVER_PORT=8880 +ENV CONFIG_PATH=/config/config.yml +ENV DEFAULTS_PATH=/app/utils/config/defaults.yml +ENV TOKENS_DIR=/config/tokens ARG TARGETOS ARG TARGETARCH diff --git a/docker-compose.yaml b/docker-compose.yaml index ab87c22..25edd98 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,10 +20,10 @@ services: aliases: - secured-signal-api environment: - SIGNAL_API_URL: http://signal-api:8080 - DEFAULT_RECIPIENTS: '[ "000", "001", "002" ]' + API__URL: http://signal-api:8080 + RECIPIENTS: 000,001,002 NUMBER: 123456789 - API_TOKEN: LOOOOOONG_STRING + API__TOKENS: LOOOOOONG_STRING ports: - "8880:8880" restart: unless-stopped diff --git a/examples/traefik.docker-compose.yaml b/examples/traefik.docker-compose.yaml index 78e8d72..7ef99b8 100644 --- a/examples/traefik.docker-compose.yaml +++ b/examples/traefik.docker-compose.yaml @@ -8,10 +8,10 @@ services: aliases: - secured-signal-api environment: - SIGNAL_API_URL: http://signal-api:8080 - DEFAULT_RECIPIENTS: '[ "000", "001", "002" ]' + API__URL: http://signal-api:8080 + DEFAULT_RECIPIENTS: 000,001,002 NUMBER: 123456789 - API_TOKEN: LOOOOOONG_STRING + API__TOKENS: LOOOOOONG_STRING labels: - traefik.enable=true - traefik.http.routers.signal-api.rule=Host(`signal-api.mydomain.com`) diff --git a/go.mod b/go.mod index a647fb6..57ab88b 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,25 @@ module github.com/codeshelldev/secured-signal-api go 1.25.1 -require go.uber.org/zap v1.27.0 +require ( + go.uber.org/zap v1.27.0 + gopkg.in/yaml.v3 v3.0.1 +) -require go.uber.org/multierr v1.11.0 // indirect +require ( + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/knadh/koanf/parsers/dotenv v1.1.0 // indirect + github.com/knadh/koanf/parsers/yaml v1.1.0 // indirect + github.com/knadh/koanf/providers/confmap v1.0.0 + github.com/knadh/koanf/providers/env v1.1.0 // indirect + github.com/knadh/koanf/providers/file v1.2.0 // indirect + github.com/knadh/koanf/v2 v2.2.2 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/sys v0.36.0 // indirect +) diff --git a/go.sum b/go.sum index 9fe9a49..4664e42 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,46 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/parsers/dotenv v1.1.0 h1:dQaM0Jw54zRsqDcaJ27pciNExuKfOXagCJW3K1h0hj0= +github.com/knadh/koanf/parsers/dotenv v1.1.0/go.mod h1:P3BQjxaIc2+SZ3n9BUceqYl95pz3qaGqYTZX0j0d/DI= +github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4= +github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= +github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= +github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= +github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc= +github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY= +github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= +github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= +github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= +github.com/knadh/koanf/v2 v2.2.2/go.mod h1:abWQc0cBXLSF/PSOMCB/SK+T13NXDsPvOksbpi5e/9Q= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internals/proxy/middlewares/body.go b/internals/proxy/middlewares/body.go index d3888e3..3cd3642 100644 --- a/internals/proxy/middlewares/body.go +++ b/internals/proxy/middlewares/body.go @@ -90,7 +90,7 @@ func getMessage(aliases []MessageAlias, data map[string]interface{}) (string, ma func processAlias(alias MessageAlias, data map[string]interface{}) (string, int, bool) { aliasKey := alias.Alias - value, ok := utils.GetJsonByPath(aliasKey, data) + value, ok := utils.GetByPath(aliasKey, data) aliasValue, isStr := value.(string) diff --git a/main.go b/main.go index b4b434c..874b722 100644 --- a/main.go +++ b/main.go @@ -7,23 +7,23 @@ import ( proxy "github.com/codeshelldev/secured-signal-api/internals/proxy" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" + config "github.com/codeshelldev/secured-signal-api/utils/config" docker "github.com/codeshelldev/secured-signal-api/utils/docker" - env "github.com/codeshelldev/secured-signal-api/utils/env" log "github.com/codeshelldev/secured-signal-api/utils/logger" ) var initHandler *httputil.ReverseProxy -var ENV env.ENV_ +var ENV config.ENV_ func main() { logLevel := os.Getenv("LOG_LEVEL") log.Init(logLevel) - env.Load() + config.Load() - ENV = env.ENV + ENV = config.ENV initHandler = proxy.Create(ENV.API_URL) diff --git a/tests/json_test.go b/tests/json_test.go index b08b9ef..4541b41 100644 --- a/tests/json_test.go +++ b/tests/json_test.go @@ -107,7 +107,7 @@ func TestJsonPath(t *testing.T) { key := c.key expected := c.expected - got, ok := utils.GetJsonByPath(key, data) + got, ok := utils.GetByPath(key, data) if !ok || got.(string) != expected { t.Error("Expected: ", key, " == ", expected, "; Got: ", got) diff --git a/utils/config/config.go b/utils/config/config.go new file mode 100644 index 0000000..328c189 --- /dev/null +++ b/utils/config/config.go @@ -0,0 +1,135 @@ +package config + +import ( + "os" + "strconv" + "strings" + + middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" + utils "github.com/codeshelldev/secured-signal-api/utils" + log "github.com/codeshelldev/secured-signal-api/utils/logger" + + "github.com/knadh/koanf/parsers/dotenv" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/confmap" + "github.com/knadh/koanf/providers/env" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/v2" +) + +type ENV_ struct { + CONFIG_PATH string + DEFAULTS_PATH string + TOKENS_DIR string + PORT string + API_URL string + API_TOKENS []string + BLOCKED_ENDPOINTS []string + VARIABLES map[string]any + MESSAGE_ALIASES []middlewares.MessageAlias +} + +var ENV ENV_ = ENV_{ + CONFIG_PATH: os.Getenv("CONFIG_PATH"), + DEFAULTS_PATH: os.Getenv("DEFAULTS_PATH"), + TOKENS_DIR: os.Getenv("TOKENS_DIR"), +} + +var config = koanf.New(".") + +func LoadIntoENV() { + ENV.PORT = strconv.Itoa(config.Int("server.port")) + + ENV.API_URL = config.String("api.url") + ENV.API_TOKENS = config.Strings("api.tokens") + + ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") + + ENV.VARIABLES = config.Get("variables").(map[string]any) + ENV.MESSAGE_ALIASES = config.Get("messagealiases").([]middlewares.MessageAlias) + + ENV.VARIABLES["NUMBER"] = config.String("number") + ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") +} + +func Load() { + LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) + LoadFile(ENV.CONFIG_PATH, yaml.Parser()) + + LoadDotEnv() + + normalizeKeys() + + LoadIntoENV() +} + +func LoadFile(path string, parser koanf.Parser) (*file.File) { + f := file.Provider(path) + + err := config.Load(f, parser) + + if err != nil { + log.Fatal("Error loading ", path, ": ", err.Error()) + } + + f.Watch(func(event interface{}, err error) { + if err != nil { + return + } + + log.Info("Config changed, Reloading...") + + Load() + }) + + return f +} + +func LoadDotEnv() (*env.Env) { + e := env.ProviderWithValue("", ".", normalizeEnv) + + err := config.Load(e, dotenv.Parser()) + + if err != nil { + log.Fatal("Error loading env: ", err.Error()) + } + + return e +} + +func normalizeKeys() { + data := map[string]any{} + + for _, key := range config.Keys() { + lower := strings.ToLower(key) + + data[lower] = config.Get(key) + } + config.Load(confmap.Provider(data, "."), nil) +} + +func normalizeEnv(key string, value string) (string, any) { + key = strings.ToLower(strings.ReplaceAll(key, "__", ".")) + + if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { + data, err := utils.GetJsonSafe[any](value) + + if data != nil && err == nil { + return key, data + } + } + + if strings.Contains(value, ",") { + items := utils.StringToArray(value) + + return key, items + } + + intValue, intErr := strconv.Atoi(value) + + if intErr == nil { + return key, intValue + } + + return key, value +} \ No newline at end of file diff --git a/utils/config/defaults.yml b/utils/config/defaults.yml new file mode 100644 index 0000000..c046721 --- /dev/null +++ b/utils/config/defaults.yml @@ -0,0 +1,29 @@ +server: + port: 8880 + +api: + tokens: + url: + +messageAliases: + [ + { alias: msg, score: 100 }, + { alias: content, score: 99 }, + { alias: description, score: 98 }, + { alias: text, score: 20 }, + { alias: body, score: 15 }, + { alias: summary, score: 10 }, + { alias: details, score: 9 }, + { alias: payload, score: 2 }, + { alias: data, score: 1 }, + ] + +blockedEndpoints: + - /v1/about + - /v1/configuration + - /v1/devices + - /v1/register + - /v1/unregister + - /v1/qrcodelink + - /v1/accounts + - /v1/contacts diff --git a/utils/env/env.go b/utils/env/env.go deleted file mode 100644 index e61a439..0000000 --- a/utils/env/env.go +++ /dev/null @@ -1,123 +0,0 @@ -package env - -import ( - "os" - "strconv" - - middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" - "github.com/codeshelldev/secured-signal-api/utils" - log "github.com/codeshelldev/secured-signal-api/utils/logger" -) - -type ENV_ struct { - PORT string - API_URL string - API_TOKENS []string - BLOCKED_ENDPOINTS []string - VARIABLES map[string]any - MESSAGE_ALIASES []middlewares.MessageAlias -} - -var ENV ENV_ = ENV_{ - BLOCKED_ENDPOINTS: []string { - "/v1/about", - "/v1/configuration", - "/v1/devices", - "/v1/register", - "/v1/unregister", - "/v1/qrcodelink", - "/v1/accounts", - "/v1/contacts", - }, - VARIABLES: map[string]any { - "RECIPIENTS": []string{}, - "NUMBER": os.Getenv("NUMBER"), - }, - MESSAGE_ALIASES: []middlewares.MessageAlias{ - { - Alias: "msg", - Score: 100, - }, - { - Alias: "content", - Score: 99, - }, - { - Alias: "description", - Score: 98, - }, - { - Alias: "text", - Score: 20, - }, - { - Alias: "body", - Score: 15, - }, - { - Alias: "summary", - Score: 10, - }, - { - Alias: "details", - Score: 9, - }, - { - Alias: "payload", - Score: 2, - }, - { - Alias: "data", - Score: 1, - }, - }, -} - -func Load() { - ENV.PORT = os.Getenv("PORT") - ENV.API_URL = os.Getenv("SIGNAL_API_URL") - - apiToken := os.Getenv("API_TOKENS") - - if apiToken == "" { - apiToken = os.Getenv("API_TOKEN") - } - - blockedEndpointStrArray := os.Getenv("BLOCKED_ENDPOINTS") - recipientsStrArray := os.Getenv("RECIPIENTS") - - messageAliasesJSON := os.Getenv("MESSAGE_ALIASES") - variablesJSON := os.Getenv("VARIABLES") - - log.Info("Loaded Environment Variables") - - apiTokens := utils.StringToArray(apiToken) - - if apiTokens == nil { - log.Warn("No API TOKEN provided this is NOT recommended") - - log.Info("Disabling Security Features due to incomplete Congfiguration") - - ENV.BLOCKED_ENDPOINTS = []string{} - } else { - log.Debug("Registered " + strconv.Itoa(len(apiTokens)) + " Tokens") - - ENV.API_TOKENS = apiTokens - } - - if blockedEndpointStrArray != "" { - ENV.BLOCKED_ENDPOINTS = utils.StringToArray(blockedEndpointStrArray) - } - - if messageAliasesJSON != "" { - ENV.MESSAGE_ALIASES = utils.GetJson[[]middlewares.MessageAlias](messageAliasesJSON) - } - - if variablesJSON != "" { - ENV.VARIABLES = utils.GetJson[map[string]any](variablesJSON) - } - - if recipientsStrArray != "" { - ENV.VARIABLES["RECIPIENTS"] = utils.StringToArray(recipientsStrArray) - } -} \ No newline at end of file diff --git a/utils/utils.go b/utils/utils.go index cd746b3..806e115 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -10,6 +10,8 @@ import ( "regexp" "strconv" "strings" + + "gopkg.in/yaml.v3" ) func StringToArray(sliceStr string) []string { @@ -30,7 +32,7 @@ func StringToArray(sliceStr string) []string { return items } -func GetJsonByPath(path string, data interface{}) (interface{}, bool) { +func GetByPath(path string, data any) (any, bool) { // Split into parts by `.` and `[]` re := regexp.MustCompile(`\.|\[|\]`) @@ -93,6 +95,26 @@ func GetJson[T any](jsonStr string) (T) { return result } +func GetYmlSafe[T any](ymlStr string) (T, error) { + var result T + + err := yaml.Unmarshal([]byte(ymlStr), &result) + + return result, err +} + +func GetYml[T any](ymlStr string) (T) { + var result T + + err := yaml.Unmarshal([]byte(ymlStr), &result) + + if err != nil { + // JSON is empty + } + + return result +} + func ToJsonSafe[T any](obj T) (string, error) { bytes, err := json.Marshal(obj) From 52714f80c6791e88a914950dfb9740520d3219d7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:39:16 +0200 Subject: [PATCH 03/97] fix missing defaults file --- .dockerignore | 1 + Dockerfile | 4 +++- {utils/config => config}/defaults.yml | 0 3 files changed, 4 insertions(+), 1 deletion(-) rename {utils/config => config}/defaults.yml (100%) diff --git a/.dockerignore b/.dockerignore index 0c6693d..f3a5350 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ *.yaml LICENSE *.md +!config \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2a762f3..cb45659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,10 @@ FROM alpine:latest RUN apk --no-cache add ca-certificates ENV SERVER_PORT=8880 + +ENV DEFAULTS_PATH=/app/config/defaults.yml + ENV CONFIG_PATH=/config/config.yml -ENV DEFAULTS_PATH=/app/utils/config/defaults.yml ENV TOKENS_DIR=/config/tokens ARG TARGETOS diff --git a/utils/config/defaults.yml b/config/defaults.yml similarity index 100% rename from utils/config/defaults.yml rename to config/defaults.yml From 855395118ff9afe4c8486abaf7ec030090ce6930 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:40:56 +0200 Subject: [PATCH 04/97] update workflow --- .github/workflows/docker-image-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml index 21bd4a8..4105f80 100644 --- a/.github/workflows/docker-image-dev.yml +++ b/.github/workflows/docker-image-dev.yml @@ -6,6 +6,7 @@ on: - dev paths: - "**/*.go" + - "config/*.yml" jobs: update: From 87b4a4c344f9f34c13f6779c0b4b9cc18507757a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:44:06 +0200 Subject: [PATCH 05/97] updated defaults --- config/defaults.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/config/defaults.yml b/config/defaults.yml index c046721..942479e 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -11,10 +11,16 @@ messageAliases: { alias: content, score: 99 }, { alias: description, score: 98 }, { alias: text, score: 20 }, - { alias: body, score: 15 }, - { alias: summary, score: 10 }, - { alias: details, score: 9 }, - { alias: payload, score: 2 }, + { alias: summary, score: 15 }, + { alias: details, score: 14 }, + { alias: data.message, score: 10 }, + { alias: data.content, score: 9 }, + { alias: data.description, score: 8 }, + { alias: data.text, score: 7 }, + { alias: data.summary, score: 6 }, + { alias: data.details, score: 5 }, + { alias: payload, score: 3 }, + { alias: body, score: 2 }, { alias: data, score: 1 }, ] From 7c2221a371eadc41d50357c29f41785702fac90c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:47:09 +0200 Subject: [PATCH 06/97] changed dockerignore --- .dockerignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index f3a5350..9074701 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,4 @@ *.yaml LICENSE *.md -!config \ No newline at end of file +!config/*.yml \ No newline at end of file From ac4614fb1eb6ad4778acb67bb4ffd1ff7513be26 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:47:46 +0200 Subject: [PATCH 07/97] formatting --- config/defaults.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/defaults.yml b/config/defaults.yml index 942479e..50d626d 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -13,12 +13,14 @@ messageAliases: { alias: text, score: 20 }, { alias: summary, score: 15 }, { alias: details, score: 14 }, + { alias: data.message, score: 10 }, { alias: data.content, score: 9 }, { alias: data.description, score: 8 }, { alias: data.text, score: 7 }, { alias: data.summary, score: 6 }, { alias: data.details, score: 5 }, + { alias: payload, score: 3 }, { alias: body, score: 2 }, { alias: data, score: 1 }, From 4c84f9b8ac5735f307d4b76b1aa28b9595fafa17 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:55:49 +0200 Subject: [PATCH 08/97] copy config --- .dockerignore | 3 ++- .github/templates/README.template.md | 18 +++++++++--------- Dockerfile | 2 ++ LICENSE => LICENSE.md | 0 4 files changed, 13 insertions(+), 10 deletions(-) rename LICENSE => LICENSE.md (100%) diff --git a/.dockerignore b/.dockerignore index 9074701..538c10c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ .git .github *.yaml -LICENSE *.md +!LICENSE.md +!README.md !config/*.yml \ No newline at end of file diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index f3066e4..7a5a54a 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -6,7 +6,7 @@ Get the latest version of the `docker-compose.yaml` file: -And add secure Token(s) to `API_TOKEN` / `API_TOKENS`. See [API_TOKEN(s)](#api-tokens) +And add secure Token(s) to `API__TOKEN` / `API__TOKENS`. See [API\_\_TOKEN(s)](#api-tokens) > [!IMPORTANT] > This Documentation will be using `sec-signal-api:8880` as the service host, @@ -47,7 +47,7 @@ Secured Signal API provides 3 Ways to Authenticate ### Bearer -To Authenticate add `Authorization: Bearer API_TOKEN` to your request Headers +To Authenticate add `Authorization: Bearer API__TOKEN` to your request Headers ### Basic Auth @@ -55,12 +55,12 @@ To use Basic Auth as Authorization Method add `Authorization: Basic BASE64_STRIN User is `api` (LOWERCASE) -Formatting for `BASE64_STRING` = `user:API_TOKEN`. +Formatting for `BASE64_STRING` = `user:API__TOKEN`. example: ```bash -echo "api:API_TOKEN" | base64 +echo "api:API__TOKEN" | base64 ``` => `YXBpOkFQSV9LRVkK` @@ -73,7 +73,7 @@ in this case you can use **Query Auth**. Here is a simple example: ```bash -curl -X POST http://sec-signal-api:8880/v2/send?@authorization=API_TOKEN +curl -X POST http://sec-signal-api:8880/v2/send?@authorization=API__TOKEN ``` Notice the `@` infront of `authorization`. See [Formatting](#format). @@ -83,7 +83,7 @@ Notice the `@` infront of `authorization`. See [Formatting](#format). To send a message to 1234567: ```bash -curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer API_TOKEN" -d '{"message": "Hello World!", "recipients": ["1234567"]}' http://sec-signal-api:8880/v2/send +curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer API__TOKEN" -d '{"message": "Hello World!", "recipients": ["1234567"]}' http://sec-signal-api:8880/v2/send ``` ### Advanced @@ -142,13 +142,13 @@ you have to add `@` in front of any KeyValue Pair assignment. ### API Token(s) -Both `API_TOKEN` and `API_TOKENS` support multiple Tokens seperated by a `,` **Comma**. +Both `API__TOKEN` and `API__TOKENS` support multiple Tokens seperated by a `,` **Comma**. During Authentication Secured Signal API will try to match the given Token against the list of Tokens inside of these Variables. ```yaml environment: - API_TOKEN: "token1, token2, token3" - API_TOKENS: "token1, token2, token3" + API__TOKEN: "token1, token2, token3" + API__TOKENS: "token1, token2, token3" ``` > [!IMPORTANT] diff --git a/Dockerfile b/Dockerfile index cb45659..00371d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,8 @@ ARG TARGETARCH WORKDIR /app +COPY /app/config ./config + COPY dist/${TARGETOS}/${TARGETARCH}/app . RUN ls diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md From dec145426b48f7e83c0a5b64282b82307e99223d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:58:00 +0200 Subject: [PATCH 09/97] added alias for API_TOKENS (without 'S) --- utils/config/config.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index 328c189..682f0c8 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -41,7 +41,14 @@ func LoadIntoENV() { ENV.PORT = strconv.Itoa(config.Int("server.port")) ENV.API_URL = config.String("api.url") - ENV.API_TOKENS = config.Strings("api.tokens") + + apiTokens := config.Strings("api.tokens") + + if len(apiTokens) <= 0 { + apiTokens = config.Strings("api.token") + } + + ENV.API_TOKENS = apiTokens ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") From fdbcdc16ee0a7a03ab32ea2a0b29e42d2d9329a4 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:03:52 +0200 Subject: [PATCH 10/97] fix ? --- Dockerfile | 2 +- utils/config/config.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 00371d0..a16a7b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG TARGETARCH WORKDIR /app -COPY /app/config ./config +COPY ./config . COPY dist/${TARGETOS}/${TARGETARCH}/app . diff --git a/utils/config/config.go b/utils/config/config.go index 682f0c8..83f0960 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -60,14 +60,21 @@ func LoadIntoENV() { } func Load() { + log.Debug("Loading Config ", ENV.DEFAULTS_PATH) + LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) + + log.Debug("Loading Config ", ENV.CONFIG_PATH) LoadFile(ENV.CONFIG_PATH, yaml.Parser()) + log.Debug("Loading DotEnv") LoadDotEnv() normalizeKeys() LoadIntoENV() + + log.Info("Finished Loading Configuration") } func LoadFile(path string, parser koanf.Parser) (*file.File) { From e6236d031f0f6221fede5ae075b39a5b36dc0f0a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:09:15 +0200 Subject: [PATCH 11/97] safely exit on config error --- utils/config/config.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index 83f0960..f5c3789 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -4,9 +4,11 @@ import ( "os" "strconv" "strings" + "time" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" utils "github.com/codeshelldev/secured-signal-api/utils" + "github.com/codeshelldev/secured-signal-api/utils/docker" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/dotenv" @@ -83,7 +85,11 @@ func LoadFile(path string, parser koanf.Parser) (*file.File) { err := config.Load(f, parser) if err != nil { - log.Fatal("Error loading ", path, ": ", err.Error()) + log.Error("Error loading ", path, ": ", err.Error()) + + time.Sleep(5*1000*1000) + + docker.Exit(1) } f.Watch(func(event interface{}, err error) { From be242f9051347435b0280c14d28f791a27a4561b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:12:36 +0200 Subject: [PATCH 12/97] debugging --- utils/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index f5c3789..11ab696 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -87,7 +87,7 @@ func LoadFile(path string, parser koanf.Parser) (*file.File) { if err != nil { log.Error("Error loading ", path, ": ", err.Error()) - time.Sleep(5*1000*1000) + time.Sleep(10 * time.Second) docker.Exit(1) } From b8cb21cd7cf2b7d6d64250ee387580d3843a0d86 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:17:05 +0200 Subject: [PATCH 13/97] debug --- utils/config/config.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 11ab696..18feee3 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -8,7 +8,6 @@ import ( middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" utils "github.com/codeshelldev/secured-signal-api/utils" - "github.com/codeshelldev/secured-signal-api/utils/docker" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/dotenv" @@ -89,7 +88,7 @@ func LoadFile(path string, parser koanf.Parser) (*file.File) { time.Sleep(10 * time.Second) - docker.Exit(1) + os.Exit(1) } f.Watch(func(event interface{}, err error) { From 743d3bf4e0d7c52bfaaf97cc07b4884b4d26c3bf Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:19:23 +0200 Subject: [PATCH 14/97] rename interface{} to any --- internals/proxy/middlewares/body.go | 6 +++--- internals/proxy/middlewares/template.go | 8 ++++---- utils/config/config.go | 5 +++-- utils/query/query.go | 10 +++++----- utils/request/request.go | 14 +++++++------- utils/templating/templating.go | 8 ++++---- utils/utils.go | 4 ++-- 7 files changed, 28 insertions(+), 27 deletions(-) diff --git a/internals/proxy/middlewares/body.go b/internals/proxy/middlewares/body.go index 3cd3642..57badc5 100644 --- a/internals/proxy/middlewares/body.go +++ b/internals/proxy/middlewares/body.go @@ -33,7 +33,7 @@ func (data BodyMiddleware) Use() http.Handler { } var modifiedBody bool - var bodyData map[string]interface{} + var bodyData map[string]any if !body.Empty { bodyData = body.Data @@ -70,7 +70,7 @@ func (data BodyMiddleware) Use() http.Handler { }) } -func getMessage(aliases []MessageAlias, data map[string]interface{}) (string, map[string]interface{}) { +func getMessage(aliases []MessageAlias, data map[string]any) (string, map[string]any) { var content string var best int @@ -87,7 +87,7 @@ func getMessage(aliases []MessageAlias, data map[string]interface{}) (string, ma return content, data } -func processAlias(alias MessageAlias, data map[string]interface{}) (string, int, bool) { +func processAlias(alias MessageAlias, data map[string]any) (string, int, bool) { aliasKey := alias.Alias value, ok := utils.GetByPath(aliasKey, data) diff --git a/internals/proxy/middlewares/template.go b/internals/proxy/middlewares/template.go index bb7e67a..1029718 100644 --- a/internals/proxy/middlewares/template.go +++ b/internals/proxy/middlewares/template.go @@ -16,7 +16,7 @@ import ( type TemplateMiddleware struct { Next http.Handler - Variables map[string]interface{} + Variables map[string]any } func (data TemplateMiddleware) Use() http.Handler { @@ -30,7 +30,7 @@ func (data TemplateMiddleware) Use() http.Handler { log.Error("Could not get Request Body: ", err.Error()) } - bodyData := map[string]interface{}{} + bodyData := map[string]any{} var modifiedBody bool @@ -100,7 +100,7 @@ func (data TemplateMiddleware) Use() http.Handler { }) } -func TemplateBody(data map[string]interface{}, VARIABLES any) (map[string]interface{}, bool, error) { +func TemplateBody(data map[string]any, VARIABLES any) (map[string]any, bool, error) { var modified bool templatedData, err := templating.RenderJSONTemplate("body", data, VARIABLES) @@ -139,7 +139,7 @@ func TemplatePath(reqUrl *url.URL, VARIABLES any) (string, bool, error) { return reqPath, modified, nil } -func TemplateQuery(reqUrl *url.URL, data map[string]interface{}, VARIABLES any) (string, map[string]interface{}, bool, error) { +func TemplateQuery(reqUrl *url.URL, data map[string]any, VARIABLES any) (string, map[string]any, bool, error) { var modified bool decodedQuery, _ := url.QueryUnescape(reqUrl.RawQuery) diff --git a/utils/config/config.go b/utils/config/config.go index 18feee3..c6b5d5b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -8,6 +8,7 @@ import ( middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" utils "github.com/codeshelldev/secured-signal-api/utils" + "github.com/codeshelldev/secured-signal-api/utils/docker" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/dotenv" @@ -88,10 +89,10 @@ func LoadFile(path string, parser koanf.Parser) (*file.File) { time.Sleep(10 * time.Second) - os.Exit(1) + docker.Exit(1) } - f.Watch(func(event interface{}, err error) { + f.Watch(func(event any, err error) { if err != nil { return } diff --git a/utils/query/query.go b/utils/query/query.go index 4fc3a55..1e9c9fb 100644 --- a/utils/query/query.go +++ b/utils/query/query.go @@ -44,8 +44,8 @@ func tryParseInt(str string) (int, bool) { return 0, false } -func ParseTypedQueryValues(values []string) interface{} { - var result interface{} +func ParseTypedQueryValues(values []string) any { + var result any raw := values[0] @@ -60,7 +60,7 @@ func ParseTypedQueryValues(values []string) interface{} { parts := strings.Split(raw, ",") - var list []interface{} + var list []any for _, part := range parts { _intValue, _isInt := tryParseInt(part) @@ -81,8 +81,8 @@ func ParseTypedQueryValues(values []string) interface{} { return result } -func ParseTypedQuery(query string, matchPrefix string) (map[string]interface{}) { - addedData := map[string]interface{}{} +func ParseTypedQuery(query string, matchPrefix string) (map[string]any) { + addedData := map[string]any{} queryData := ParseRawQuery(query) diff --git a/utils/request/request.go b/utils/request/request.go index 723e586..7bcacbf 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -19,7 +19,7 @@ const ( type BodyType string type Body struct { - Data map[string]interface{} + Data map[string]any Raw []byte Empty bool } @@ -28,7 +28,7 @@ func (body Body) ToString() string { return string(body.Raw) } -func CreateBody(data map[string]interface{}) (Body, error) { +func CreateBody(data map[string]any) (Body, error) { if len(data) <= 0 { err := errors.New("empty data map") @@ -51,8 +51,8 @@ func CreateBody(data map[string]interface{}) (Body, error) { }, nil } -func GetJsonData(body []byte) (map[string]interface{}, error) { - var data map[string]interface{} +func GetJsonData(body []byte) (map[string]any, error) { + var data map[string]any err := json.Unmarshal(body, &data) @@ -64,8 +64,8 @@ func GetJsonData(body []byte) (map[string]interface{}, error) { return data, nil } -func GetFormData(body []byte) (map[string]interface{}, error) { - data := map[string]interface{}{} +func GetFormData(body []byte) (map[string]any, error) { + data := map[string]any{} queryData := query.ParseRawQuery(string(body)) @@ -110,7 +110,7 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { return Body{Empty: true}, nil } - var data map[string]interface{} + var data map[string]any switch GetBodyType(req) { case Json: diff --git a/utils/templating/templating.go b/utils/templating/templating.go index 70f8710..69dc621 100644 --- a/utils/templating/templating.go +++ b/utils/templating/templating.go @@ -9,12 +9,12 @@ import ( "text/template" ) -func normalize(value interface{}) string { +func normalize(value any) string { switch str := value.(type) { case []string: return "[" + strings.Join(str, ",") + "]" - case []interface{}: + case []any: items := make([]string, len(str)) for i, item := range str { @@ -27,7 +27,7 @@ func normalize(value interface{}) string { } } -func normalizeJSON(value interface{}) string { +func normalizeJSON(value any) string { jsonBytes, err := json.Marshal(value) if err != nil { @@ -63,7 +63,7 @@ func CreateTemplateWithFunc(name string, funcMap template.FuncMap) (*template.Te return template.New(name).Funcs(funcMap) } -func RenderJSONTemplate(name string, data map[string]interface{}, variables any) (map[string]interface{}, error) { +func RenderJSONTemplate(name string, data map[string]any, variables any) (map[string]any, error) { jsonBytes, err := json.Marshal(data) if err != nil { diff --git a/utils/utils.go b/utils/utils.go index 806e115..3558290 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -51,7 +51,7 @@ func GetByPath(path string, data any) (any, bool) { for _, key := range cleaned { switch currentDataType := current.(type) { // Case: Dictionary - case map[string]interface{}: + case map[string]any: value, ok := currentDataType[key] if !ok { return nil, false @@ -59,7 +59,7 @@ func GetByPath(path string, data any) (any, bool) { current = value // Case: Array - case []interface{}: + case []any: index, err := strconv.Atoi(key) if err != nil || index < 0 || index >= len(currentDataType) { From ccaf801396cd12eb517471bb7b27e3b159b7efc8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:21:52 +0200 Subject: [PATCH 15/97] fixed defaults file copying --- Dockerfile | 2 +- utils/utils.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a16a7b1..1aab5cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG TARGETARCH WORKDIR /app -COPY ./config . +COPY ./config ./config/ COPY dist/${TARGETOS}/${TARGETARCH}/app . diff --git a/utils/utils.go b/utils/utils.go index 3558290..8a4d4ae 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -89,7 +89,7 @@ func GetJson[T any](jsonStr string) (T) { err := json.Unmarshal([]byte(jsonStr), &result) if err != nil { - // JSON is empty + // YML is empty } return result @@ -109,7 +109,7 @@ func GetYml[T any](ymlStr string) (T) { err := yaml.Unmarshal([]byte(ymlStr), &result) if err != nil { - // JSON is empty + // YML is empty } return result From 84a8b4bfba257949dae8d1fc2ed9bed025f68c8f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:30:53 +0200 Subject: [PATCH 16/97] check if config is missing --- utils/config/config.go | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index c6b5d5b..00d8855 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,14 +1,14 @@ package config import ( + "errors" + "io/fs" "os" "strconv" "strings" - "time" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" utils "github.com/codeshelldev/secured-signal-api/utils" - "github.com/codeshelldev/secured-signal-api/utils/docker" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/dotenv" @@ -62,12 +62,25 @@ func LoadIntoENV() { } func Load() { - log.Debug("Loading Config ", ENV.DEFAULTS_PATH) + log.Debug("Loading Default Config ", ENV.DEFAULTS_PATH) - LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) + defErr := LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) + + if defErr != nil { + log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) + } log.Debug("Loading Config ", ENV.CONFIG_PATH) - LoadFile(ENV.CONFIG_PATH, yaml.Parser()) + + conErr := LoadFile(ENV.CONFIG_PATH, yaml.Parser()) + + if conErr != nil { + _, err := os.Stat(ENV.CONFIG_PATH) + + if !errors.Is(err, fs.ErrNotExist) { + log.Error("Could not Load Config ", ENV.CONFIG_PATH, ": ", conErr.Error()) + } + } log.Debug("Loading DotEnv") LoadDotEnv() @@ -79,17 +92,13 @@ func Load() { log.Info("Finished Loading Configuration") } -func LoadFile(path string, parser koanf.Parser) (*file.File) { +func LoadFile(path string, parser koanf.Parser) error { f := file.Provider(path) err := config.Load(f, parser) if err != nil { - log.Error("Error loading ", path, ": ", err.Error()) - - time.Sleep(10 * time.Second) - - docker.Exit(1) + return err } f.Watch(func(event any, err error) { @@ -102,10 +111,10 @@ func LoadFile(path string, parser koanf.Parser) (*file.File) { Load() }) - return f + return err } -func LoadDotEnv() (*env.Env) { +func LoadDotEnv() error { e := env.ProviderWithValue("", ".", normalizeEnv) err := config.Load(e, dotenv.Parser()) @@ -114,7 +123,7 @@ func LoadDotEnv() (*env.Env) { log.Fatal("Error loading env: ", err.Error()) } - return e + return err } func normalizeKeys() { From 60a9d0c2efb6279ac895bd82587428d52f037efb Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:37:50 +0200 Subject: [PATCH 17/97] update env to v2 --- go.mod | 1 + go.sum | 2 ++ utils/config/config.go | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 57ab88b..6cf6e21 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/knadh/koanf/parsers/yaml v1.1.0 // indirect github.com/knadh/koanf/providers/confmap v1.0.0 github.com/knadh/koanf/providers/env v1.1.0 // indirect + github.com/knadh/koanf/providers/env/v2 v2.0.0 github.com/knadh/koanf/providers/file v1.2.0 // indirect github.com/knadh/koanf/v2 v2.2.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/go.sum b/go.sum index 4664e42..3e6711c 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5z github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc= github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY= +github.com/knadh/koanf/providers/env/v2 v2.0.0 h1:Ad5H3eun722u+FvchiIcEIJZsZ2M6oxCkgZfWN5B5KY= +github.com/knadh/koanf/providers/env/v2 v2.0.0/go.mod h1:1g01PE+Ve1gBfWNNw2wmULRP0tc8RJrjn5p2N/jNCIc= github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= diff --git a/utils/config/config.go b/utils/config/config.go index 00d8855..96fc411 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -14,7 +14,7 @@ import ( "github.com/knadh/koanf/parsers/dotenv" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/confmap" - "github.com/knadh/koanf/providers/env" + "github.com/knadh/koanf/providers/env/v2" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" ) @@ -62,7 +62,7 @@ func LoadIntoENV() { } func Load() { - log.Debug("Loading Default Config ", ENV.DEFAULTS_PATH) + log.Debug("Loading Config ", ENV.DEFAULTS_PATH) defErr := LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) @@ -115,7 +115,7 @@ func LoadFile(path string, parser koanf.Parser) error { } func LoadDotEnv() error { - e := env.ProviderWithValue("", ".", normalizeEnv) + e := env.Provider("", ".", env.Op) err := config.Load(e, dotenv.Parser()) From 55d729986c40ea3e9b7032ff6ab9463ffa3d480b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:39:14 +0200 Subject: [PATCH 18/97] fix? --- utils/config/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index 96fc411..546c867 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -115,7 +115,9 @@ func LoadFile(path string, parser koanf.Parser) error { } func LoadDotEnv() error { - e := env.Provider("", ".", env.Op) + e := env.Provider(".", env.Opt{ + TransformFunc: normalizeEnv, + }) err := config.Load(e, dotenv.Parser()) From 3d140e744388a7c824d3e4d325393a333f481abe Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:43:14 +0200 Subject: [PATCH 19/97] removed unnecessary parser (dotEnv) --- go.mod | 9 +++------ go.sum | 15 ++++++--------- utils/config/config.go | 11 +++++------ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 6cf6e21..fc1c638 100644 --- a/go.mod +++ b/go.mod @@ -10,15 +10,12 @@ require ( require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.3.0 // indirect - github.com/joho/godotenv v1.5.1 // indirect github.com/knadh/koanf/maps v0.1.2 // indirect - github.com/knadh/koanf/parsers/dotenv v1.1.0 // indirect - github.com/knadh/koanf/parsers/yaml v1.1.0 // indirect + github.com/knadh/koanf/parsers/yaml v1.1.0 github.com/knadh/koanf/providers/confmap v1.0.0 - github.com/knadh/koanf/providers/env v1.1.0 // indirect github.com/knadh/koanf/providers/env/v2 v2.0.0 - github.com/knadh/koanf/providers/file v1.2.0 // indirect - github.com/knadh/koanf/v2 v2.2.2 // indirect + github.com/knadh/koanf/providers/file v1.2.0 + github.com/knadh/koanf/v2 v2.2.2 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index 3e6711c..166ba14 100644 --- a/go.sum +++ b/go.sum @@ -4,33 +4,30 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= -github.com/knadh/koanf/parsers/dotenv v1.1.0 h1:dQaM0Jw54zRsqDcaJ27pciNExuKfOXagCJW3K1h0hj0= -github.com/knadh/koanf/parsers/dotenv v1.1.0/go.mod h1:P3BQjxaIc2+SZ3n9BUceqYl95pz3qaGqYTZX0j0d/DI= github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4= github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= -github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc= -github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY= github.com/knadh/koanf/providers/env/v2 v2.0.0 h1:Ad5H3eun722u+FvchiIcEIJZsZ2M6oxCkgZfWN5B5KY= github.com/knadh/koanf/providers/env/v2 v2.0.0/go.mod h1:1g01PE+Ve1gBfWNNw2wmULRP0tc8RJrjn5p2N/jNCIc= github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= github.com/knadh/koanf/v2 v2.2.2/go.mod h1:abWQc0cBXLSF/PSOMCB/SK+T13NXDsPvOksbpi5e/9Q= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -41,8 +38,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/utils/config/config.go b/utils/config/config.go index 546c867..95aa4de 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -11,7 +11,6 @@ import ( utils "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" - "github.com/knadh/koanf/parsers/dotenv" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/confmap" "github.com/knadh/koanf/providers/env/v2" @@ -39,7 +38,7 @@ var ENV ENV_ = ENV_{ var config = koanf.New(".") -func LoadIntoENV() { +func InitEnv() { ENV.PORT = strconv.Itoa(config.Int("server.port")) ENV.API_URL = config.String("api.url") @@ -83,11 +82,11 @@ func Load() { } log.Debug("Loading DotEnv") - LoadDotEnv() + LoadEnv() normalizeKeys() - LoadIntoENV() + InitEnv() log.Info("Finished Loading Configuration") } @@ -114,12 +113,12 @@ func LoadFile(path string, parser koanf.Parser) error { return err } -func LoadDotEnv() error { +func LoadEnv() error { e := env.Provider(".", env.Opt{ TransformFunc: normalizeEnv, }) - err := config.Load(e, dotenv.Parser()) + err := config.Load(e, nil) if err != nil { log.Fatal("Error loading env: ", err.Error()) From 8680d7893e337f7836c6e39e6e3d696f69349149 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:49:35 +0200 Subject: [PATCH 20/97] check configs if nil before applying type conversion --- utils/config/config.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 95aa4de..ecb522b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -53,8 +53,19 @@ func InitEnv() { ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") - ENV.VARIABLES = config.Get("variables").(map[string]any) - ENV.MESSAGE_ALIASES = config.Get("messagealiases").([]middlewares.MessageAlias) + messageAliases := config.Get("messagealiases") + + if messageAliases != nil { + ENV.MESSAGE_ALIASES = messageAliases.([]middlewares.MessageAlias) + } + + variables := config.Get("variables") + + if variables != nil { + ENV.VARIABLES = variables.(map[string]any) + } else { + ENV.VARIABLES = map[string]any{} + } ENV.VARIABLES["NUMBER"] = config.String("number") ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") From edaa95c629fa0433208ef9ac88681a95d3ef48c7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:00:53 +0200 Subject: [PATCH 21/97] fix empty variables --- utils/config/config.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index ecb522b..d23d638 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -34,6 +34,8 @@ var ENV ENV_ = ENV_{ CONFIG_PATH: os.Getenv("CONFIG_PATH"), DEFAULTS_PATH: os.Getenv("DEFAULTS_PATH"), TOKENS_DIR: os.Getenv("TOKENS_DIR"), + MESSAGE_ALIASES: []middlewares.MessageAlias{}, + VARIABLES: map[string]any{}, } var config = koanf.New(".") @@ -53,19 +55,9 @@ func InitEnv() { ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") - messageAliases := config.Get("messagealiases") + config.Unmarshal("messagealiases", &ENV.MESSAGE_ALIASES) - if messageAliases != nil { - ENV.MESSAGE_ALIASES = messageAliases.([]middlewares.MessageAlias) - } - - variables := config.Get("variables") - - if variables != nil { - ENV.VARIABLES = variables.(map[string]any) - } else { - ENV.VARIABLES = map[string]any{} - } + config.Unmarshal("variables", &ENV.VARIABLES) ENV.VARIABLES["NUMBER"] = config.String("number") ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") From 0774fcdbf0d6b3bd24aec64bbd115c7271a44056 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:49:48 +0200 Subject: [PATCH 22/97] add insecurity (disable sec if token missing) --- utils/config/config.go | 62 ++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index d23d638..80612da 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -25,6 +25,7 @@ type ENV_ struct { PORT string API_URL string API_TOKENS []string + INSECURE bool BLOCKED_ENDPOINTS []string VARIABLES map[string]any MESSAGE_ALIASES []middlewares.MessageAlias @@ -34,11 +35,17 @@ var ENV ENV_ = ENV_{ CONFIG_PATH: os.Getenv("CONFIG_PATH"), DEFAULTS_PATH: os.Getenv("DEFAULTS_PATH"), TOKENS_DIR: os.Getenv("TOKENS_DIR"), + API_TOKENS: []string{}, + BLOCKED_ENDPOINTS: []string{}, MESSAGE_ALIASES: []middlewares.MessageAlias{}, VARIABLES: map[string]any{}, + INSECURE: false, } -var config = koanf.New(".") +var defaultsLayer = koanf.New(".") +var userLayer = koanf.New(".") + +var config *koanf.Koanf func InitEnv() { ENV.PORT = strconv.Itoa(config.Int("server.port")) @@ -49,24 +56,39 @@ func InitEnv() { if len(apiTokens) <= 0 { apiTokens = config.Strings("api.token") + + if len(apiTokens) <= 0 { + log.Warn("No API TOKEN provided this is NOT recommended") + + log.Info("Disabling Security Features due to incomplete Congfiguration") + + ENV.INSECURE = true + + // Set Blocked Endpoints on Config to User Layer Value + // => effectively ignoring Default Layer + config.Set("blockedendpoints", userLayer.Strings("blockeendpoints")) + } } - ENV.API_TOKENS = apiTokens + if len(apiTokens) > 0 { + log.Debug("Registered " + strconv.Itoa(len(apiTokens)) + " Tokens") - ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") + ENV.API_TOKENS = apiTokens + } config.Unmarshal("messagealiases", &ENV.MESSAGE_ALIASES) - config.Unmarshal("variables", &ENV.VARIABLES) ENV.VARIABLES["NUMBER"] = config.String("number") ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") + + ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") } func Load() { log.Debug("Loading Config ", ENV.DEFAULTS_PATH) - defErr := LoadFile(ENV.DEFAULTS_PATH, yaml.Parser()) + defPro, defErr := LoadFile(ENV.DEFAULTS_PATH, defaultsLayer, yaml.Parser()) if defErr != nil { log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) @@ -74,7 +96,7 @@ func Load() { log.Debug("Loading Config ", ENV.CONFIG_PATH) - conErr := LoadFile(ENV.CONFIG_PATH, yaml.Parser()) + conPro, conErr := LoadFile(ENV.CONFIG_PATH, userLayer, yaml.Parser()) if conErr != nil { _, err := os.Stat(ENV.CONFIG_PATH) @@ -85,22 +107,25 @@ func Load() { } log.Debug("Loading DotEnv") - LoadEnv() - normalizeKeys() + envPro, _ := LoadEnv(userLayer) + + config = mergeLayers(defPro, conPro, envPro) + + normalizeKeys(config) InitEnv() log.Info("Finished Loading Configuration") } -func LoadFile(path string, parser koanf.Parser) error { +func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Provider, error) { f := file.Provider(path) err := config.Load(f, parser) if err != nil { - return err + return nil, err } f.Watch(func(event any, err error) { @@ -113,10 +138,10 @@ func LoadFile(path string, parser koanf.Parser) error { Load() }) - return err + return f, err } -func LoadEnv() error { +func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { e := env.Provider(".", env.Opt{ TransformFunc: normalizeEnv, }) @@ -127,10 +152,19 @@ func LoadEnv() error { log.Fatal("Error loading env: ", err.Error()) } - return err + return e, err +} + +func mergeLayers(defPro koanf.Provider, conPro koanf.Provider, envPro koanf.Provider) *koanf.Koanf { + final := koanf.New(".") + _ = final.Load(defPro, nil) + _ = final.Load(conPro, nil) + _ = final.Load(envPro, nil) + + return final } -func normalizeKeys() { +func normalizeKeys(config *koanf.Koanf) { data := map[string]any{} for _, key := range config.Keys() { From 8651ac3d7771bc373ff326c54521eab4fa41a929 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 19:01:10 +0200 Subject: [PATCH 23/97] debugging --- utils/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index 80612da..1a792f6 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -48,7 +48,7 @@ var userLayer = koanf.New(".") var config *koanf.Koanf func InitEnv() { - ENV.PORT = strconv.Itoa(config.Int("server.port")) + ENV.PORT = strconv.Itoa(defaultsLayer.Int("server.port")) ENV.API_URL = config.String("api.url") From 15ecc31801dae52821c517f97b7064a85d61af70 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 19:05:26 +0200 Subject: [PATCH 24/97] fixed merging issues --- utils/config/config.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 1a792f6..d99a5a1 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -48,7 +48,7 @@ var userLayer = koanf.New(".") var config *koanf.Koanf func InitEnv() { - ENV.PORT = strconv.Itoa(defaultsLayer.Int("server.port")) + ENV.PORT = strconv.Itoa(config.Int("server.port")) ENV.API_URL = config.String("api.url") @@ -88,7 +88,7 @@ func InitEnv() { func Load() { log.Debug("Loading Config ", ENV.DEFAULTS_PATH) - defPro, defErr := LoadFile(ENV.DEFAULTS_PATH, defaultsLayer, yaml.Parser()) + _, defErr := LoadFile(ENV.DEFAULTS_PATH, defaultsLayer, yaml.Parser()) if defErr != nil { log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) @@ -96,7 +96,7 @@ func Load() { log.Debug("Loading Config ", ENV.CONFIG_PATH) - conPro, conErr := LoadFile(ENV.CONFIG_PATH, userLayer, yaml.Parser()) + _, conErr := LoadFile(ENV.CONFIG_PATH, userLayer, yaml.Parser()) if conErr != nil { _, err := os.Stat(ENV.CONFIG_PATH) @@ -108,9 +108,9 @@ func Load() { log.Debug("Loading DotEnv") - envPro, _ := LoadEnv(userLayer) + LoadEnv(userLayer) - config = mergeLayers(defPro, conPro, envPro) + config = mergeLayers() normalizeKeys(config) @@ -155,11 +155,11 @@ func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { return e, err } -func mergeLayers(defPro koanf.Provider, conPro koanf.Provider, envPro koanf.Provider) *koanf.Koanf { +func mergeLayers() *koanf.Koanf { final := koanf.New(".") - _ = final.Load(defPro, nil) - _ = final.Load(conPro, nil) - _ = final.Load(envPro, nil) + + final.Merge(defaultsLayer) + final.Merge(userLayer) return final } From fec07639271e5dfabfb519006f987f23c7b571ef Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 19:08:57 +0200 Subject: [PATCH 25/97] fix number handling --- utils/config/config.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index d99a5a1..b8590ea 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -192,10 +192,13 @@ func normalizeEnv(key string, value string) (string, any) { return key, items } - intValue, intErr := strconv.Atoi(value) + // Treat `+` prefixed Values as strings + if !strings.HasPrefix(value, "+") { + intValue, intErr := strconv.Atoi(value) - if intErr == nil { - return key, intValue + if intErr == nil { + return key, intValue + } } return key, value From 285f4a08efd7922d7fb420ae7fe8427d191dab75 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 19:28:19 +0200 Subject: [PATCH 26/97] =?UTF-8?q?added=20"stringing.go"=20=F0=9F=A4=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/stringing/stringing.go | 1 + 1 file changed, 1 insertion(+) create mode 100644 utils/stringing/stringing.go diff --git a/utils/stringing/stringing.go b/utils/stringing/stringing.go new file mode 100644 index 0000000..23ea081 --- /dev/null +++ b/utils/stringing/stringing.go @@ -0,0 +1 @@ +package stringing From 7065f68abdf65efe44fd815ba66133e76eb9d5d5 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 19:29:01 +0200 Subject: [PATCH 27/97] revert --- utils/stringing/stringing.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 utils/stringing/stringing.go diff --git a/utils/stringing/stringing.go b/utils/stringing/stringing.go deleted file mode 100644 index 23ea081..0000000 --- a/utils/stringing/stringing.go +++ /dev/null @@ -1 +0,0 @@ -package stringing From 5791b3d155438d746107289841efd78efdb32141 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 22:17:38 +0200 Subject: [PATCH 28/97] updated string to type * added central file for str operations and funcs * fixed and added tests * (fixed) README --- .github/templates/README.template.md | 55 ++++++----- tests/string_test.go | 78 +++++++++++++++ utils/config/config.go | 27 +----- utils/query/query.go | 53 +---------- utils/safestrings/safestrings.go | 136 +++++++++++++++++++++++++++ utils/utils.go | 19 ---- 6 files changed, 251 insertions(+), 117 deletions(-) create mode 100644 tests/string_test.go create mode 100644 utils/safestrings/safestrings.go diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 7a5a54a..90f443a 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -6,11 +6,11 @@ Get the latest version of the `docker-compose.yaml` file: -And add secure Token(s) to `API__TOKEN` / `API__TOKENS`. See [API\_\_TOKEN(s)](#api-tokens) +And add secure Token(s) to `API__TOKEN` / `API__TOKENS`. See [API TOKEN(s)](#api-tokens) > [!IMPORTANT] > This Documentation will be using `sec-signal-api:8880` as the service host, -> this **won't work**, instead use your containers IP + Port. +> this **is just for simplicty**, instead use your containers or hosts IP + Port. > Or a hostname if applicable. See [Reverse Proxy](#reverse-proxy) ```yaml @@ -47,7 +47,7 @@ Secured Signal API provides 3 Ways to Authenticate ### Bearer -To Authenticate add `Authorization: Bearer API__TOKEN` to your request Headers +To Authenticate add `Authorization: Bearer API_TOKEN` to your request Headers ### Basic Auth @@ -55,12 +55,12 @@ To use Basic Auth as Authorization Method add `Authorization: Basic BASE64_STRIN User is `api` (LOWERCASE) -Formatting for `BASE64_STRING` = `user:API__TOKEN`. +Formatting for `BASE64_STRING` = `user:API_TOKEN`. example: ```bash -echo "api:API__TOKEN" | base64 +echo "api:API_TOKEN" | base64 ``` => `YXBpOkFQSV9LRVkK` @@ -73,17 +73,17 @@ in this case you can use **Query Auth**. Here is a simple example: ```bash -curl -X POST http://sec-signal-api:8880/v2/send?@authorization=API__TOKEN +curl -X POST http://sec-signal-api:8880/v2/send?@authorization=API_TOKEN ``` -Notice the `@` infront of `authorization`. See [Formatting](#format). +Notice the `@` infront of `authorization`. See [KeyValue Pair Injection](#keyvalue-pair-injection). ### Example To send a message to 1234567: ```bash -curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer API__TOKEN" -d '{"message": "Hello World!", "recipients": ["1234567"]}' http://sec-signal-api:8880/v2/send +curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer API_TOKEN" -d '{"message": "Hello World!", "recipients": ["1234567"]}' http://sec-signal-api:8880/v2/send ``` ### Advanced @@ -121,34 +121,41 @@ http://sec-signal-api:8880/v1/receive/{{.NUMBER}} In some cases you may not be able to access / modify the Request Body, in that case specify needed values in the Request Query: -Supported types include **strings**, **ints** and **arrays** - `http://sec-signal-api:8880/?@key=value` -| type | example | -| :--------- | :------ | -| string | abc | -| int | 123 | -| array | [1,2,3] | -| array(int) | 1,2,3 | -| array(str) | a,b,c | - -##### Format - In order to differentiate Injection Queries and _regular_ Queries you have to add `@` in front of any KeyValue Pair assignment. +Supported types include **strings**, **ints** and **arrays**. See [Formatting](#string-to-type). + ## Environment Variables +### String To Type + +In the Environment the only allowed type is a string so to not have to always use a json string you can use the following types, +if you format them correctly... + +| type | example | +| :--------- | :---------------- | +| string | abc | +| string | +123 | +| int | 123 | +| int | -123 | +| json | {"a":"b","c":"d"} | +| array(int) | [1,2,3] | +| array(str) | [a,b,c] | + +This formatting applies to almost every situation where the only (allowed) Input Type is a string and other Output Types are needed. + ### API Token(s) -Both `API__TOKEN` and `API__TOKENS` support multiple Tokens seperated by a `,` **Comma**. +Both `API__TOKEN` and `API__TOKENS` support multiple Tokens seperated by a `,` **Comma** and `[]` **Brackets**. See [Formatting](#string-to-type). During Authentication Secured Signal API will try to match the given Token against the list of Tokens inside of these Variables. ```yaml environment: - API__TOKEN: "token1, token2, token3" - API__TOKENS: "token1, token2, token3" + API__TOKEN: [token1, token2, token3] + API__TOKENS: [token1, token2, token3] ``` > [!IMPORTANT] @@ -215,7 +222,7 @@ Set this Environment Variable to automatically provide default Recipients: ```yaml environment: RECIPIENTS: | - user.id, 000, 001, group.id, + [ user.id, 000, 001, group.id ] ``` example: diff --git a/tests/string_test.go b/tests/string_test.go new file mode 100644 index 0000000..e7d79b0 --- /dev/null +++ b/tests/string_test.go @@ -0,0 +1,78 @@ +package tests + +import ( + "reflect" + "testing" + + "github.com/codeshelldev/secured-signal-api/utils/safestrings" +) + +func TestStringEscaping(t *testing.T) { + str1 := `\#` + + res1 := safestrings.IsEscaped(str1, "#") + + if !res1 { + t.Error("Expected: ", str1, " == true", "; Got: ", str1, " == ", res1) + } + + str2 := "#" + + res2 := safestrings.IsEscaped(str2, "#") + + if res2 { + t.Error("Expected: ", str2, " == false", "; Got: ", str2, " == ", res2) + } + + str3 := `#\#` + + res3 := safestrings.Contains(str3, "#") + + if !res3 { + t.Error("Expected: ", str3, " == true", "; Got: ", str3, " == ", res3) + } +} + +func TestStringEnclosement(t *testing.T) { + str1 := "[enclosed]" + + res1 := safestrings.IsEnclosedBy(str1, `[`, `]`) + + if !res1 { + t.Error("Expected: ", str1, " == true", "; Got: ", str1, " == ", res1) + } + + str2 := `\[enclosed]` + + res2 := safestrings.IsEnclosedBy(str2, `[`, `]`) + + if res2 { + t.Error("Expected: ", str2, " == false", "; Got: ", str2, " == ", res2) + } +} + +func TestStringToType(t *testing.T) { + str1 := `[item1,item2]` + + res1 := safestrings.ToType(str1) + + if reflect.TypeOf(res1) != reflect.TypeFor[[]string]() { + t.Error("Expected: ", str1, " == []string", "; Got: ", str1, " == ", reflect.TypeOf(res1)) + } + + str2 := `1` + + res2 := safestrings.ToType(str2) + + if reflect.TypeOf(res2) != reflect.TypeFor[int]() { + t.Error("Expected: ", str2, " == int", "; Got: ", str2, " == ", reflect.TypeOf(res2)) + } + + str3 := `{ "key": "value" }` + + res3 := safestrings.ToType(str3) + + if reflect.TypeOf(res3) != reflect.TypeFor[map[string]any]() { + t.Error("Expected: ", str3, " == map[string]any", "; Got: ", str3, " == ", reflect.TypeOf(res3)) + } +} \ No newline at end of file diff --git a/utils/config/config.go b/utils/config/config.go index b8590ea..81b8beb 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -8,8 +8,8 @@ import ( "strings" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" - utils "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" + "github.com/codeshelldev/secured-signal-api/utils/safestrings" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/confmap" @@ -178,28 +178,5 @@ func normalizeKeys(config *koanf.Koanf) { func normalizeEnv(key string, value string) (string, any) { key = strings.ToLower(strings.ReplaceAll(key, "__", ".")) - if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { - data, err := utils.GetJsonSafe[any](value) - - if data != nil && err == nil { - return key, data - } - } - - if strings.Contains(value, ",") { - items := utils.StringToArray(value) - - return key, items - } - - // Treat `+` prefixed Values as strings - if !strings.HasPrefix(value, "+") { - intValue, intErr := strconv.Atoi(value) - - if intErr == nil { - return key, intValue - } - } - - return key, value + return key, safestrings.ToType(value) } \ No newline at end of file diff --git a/utils/query/query.go b/utils/query/query.go index 1e9c9fb..98a50ce 100644 --- a/utils/query/query.go +++ b/utils/query/query.go @@ -1,9 +1,9 @@ package query import ( - "regexp" - "strconv" "strings" + + "github.com/codeshelldev/secured-signal-api/utils/safestrings" ) func ParseRawQuery(raw string) map[string][]string { @@ -30,55 +30,10 @@ func ParseRawQuery(raw string) map[string][]string { return result } -func tryParseInt(str string) (int, bool) { - isInt, err := regexp.MatchString(`^\d+$`, str) - - if isInt && err == nil { - intValue, err := strconv.Atoi(str) - - if err == nil { - return intValue, true - } - } - - return 0, false -} - func ParseTypedQueryValues(values []string) any { - var result any - - raw := values[0] - - intValue, isInt := tryParseInt(raw) + raw := values[len(values)-1] - if strings.Contains(raw, ",") || (strings.Contains(raw, "[") && strings.Contains(raw, "]")) { - if strings.Contains(raw, "[") && strings.Contains(raw, "]") { - escapedStr := strings.ReplaceAll(raw, "[", "") - escapedStr = strings.ReplaceAll(escapedStr, "]", "") - raw = escapedStr - } - - parts := strings.Split(raw, ",") - - var list []any - - for _, part := range parts { - _intValue, _isInt := tryParseInt(part) - - if _isInt { - list = append(list, _intValue) - } else { - list = append(list, part) - } - } - result = list - } else if isInt { - result = intValue - } else { - result = raw - } - - return result + return safestrings.ToType(raw) } func ParseTypedQuery(query string, matchPrefix string) (map[string]any) { diff --git a/utils/safestrings/safestrings.go b/utils/safestrings/safestrings.go new file mode 100644 index 0000000..650b362 --- /dev/null +++ b/utils/safestrings/safestrings.go @@ -0,0 +1,136 @@ +package safestrings + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/codeshelldev/secured-signal-api/utils" +) + +func ToType(str string) any { + cleaned := strings.TrimSpace(str) + + //* Try JSON + if IsEnclosedBy(cleaned, `[`, `]`) || IsEnclosedBy(cleaned, `{`, `}`) { + data, err := utils.GetJsonSafe[any](str) + + if data != nil && err == nil { + return data + } + } + + //* Try String Slice + if Contains(str, ",") && IsEnclosedBy(cleaned, `[`, `]`) { + bracketsless := strings.ReplaceAll(str, "[", "") + bracketsless = strings.ReplaceAll(bracketsless, "]", "") + + data := ToArray(bracketsless) + + if data != nil { + if len(data) > 0 { + return data + } + } + } + + //* Try Number + if !strings.HasPrefix(cleaned, "+") { + intValue, intErr := strconv.Atoi(cleaned) + + if intErr == nil { + return intValue + } + } + + fmt.Print(str) + + return str +} + +func Contains(str string, match string) bool { + return !IsEscaped(str, match) +} + +// Checks if a string is Enclosed by `char` and are not Escaped +func IsEnclosedBy(str string, charA, charB string) bool { + if NeedsEscapeForRegex(rune(charA[0])) { + charA = `\` + charA + } + + if NeedsEscapeForRegex(rune(charB[0])) { + charB = `\` + charB + } + + regexStr := `(^|[^\\])(\\\\)*(` + charA + `)(.*?)(^|[^\\])(\\\\)*(` + charB + ")" + + re := regexp.MustCompile(regexStr) + + matches := re.FindAllStringSubmatchIndex(str, -1) + + filtered := [][]int{} + + for _, match := range matches { + start := match[len(match)-2] + end := match[len(match)-1] + char := str[start:end] + + if char != `\` { + filtered = append(filtered, match) + } + } + + return len(filtered) > 0 +} + +// Checks if a string is completly Escaped with `\` +func IsEscaped(str string, char string) bool { + if NeedsEscapeForRegex(rune(char[0])) { + char = `\` + char + } + + regexStr := `(^|[^\\])(\\\\)*(` + char + ")" + + re := regexp.MustCompile(regexStr) + + matches := re.FindAllStringSubmatchIndex(str, -1) + + filtered := [][]int{} + + for _, match := range matches { + start := match[len(match)-2] + end := match[len(match)-1] + char := str[start:end] + + if char != `\` { + filtered = append(filtered, match) + } + } + + return len(filtered) == 0 +} + +func NeedsEscapeForRegex(char rune) bool { + special := `.+*?()|[]{}^$\\` + + return strings.ContainsRune(special, char) +} + +func ToArray(sliceStr string) []string { + if sliceStr == "" { + return nil + } + + rawItems := strings.Split(sliceStr, ",") + items := make([]string, 0, len(rawItems)) + + for _, item := range rawItems { + trimmed := strings.TrimSpace(item) + if trimmed != "" { + items = append(items, trimmed) + } + } + + return items +} \ No newline at end of file diff --git a/utils/utils.go b/utils/utils.go index 8a4d4ae..2159e1b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -9,29 +9,10 @@ import ( "encoding/json" "regexp" "strconv" - "strings" "gopkg.in/yaml.v3" ) -func StringToArray(sliceStr string) []string { - if sliceStr == "" { - return nil - } - - rawItems := strings.Split(sliceStr, ",") - items := make([]string, 0, len(rawItems)) - - for _, item := range rawItems { - trimmed := strings.TrimSpace(item) - if trimmed != "" { - items = append(items, trimmed) - } - } - - return items -} - func GetByPath(path string, data any) (any, bool) { // Split into parts by `.` and `[]` re := regexp.MustCompile(`\.|\[|\]`) From 468468cb59c544289635252d1f157d7cf0de5767 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 22:22:01 +0200 Subject: [PATCH 29/97] debugging RECIPIENTS --- utils/config/config.go | 2 ++ utils/safestrings/safestrings.go | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 81b8beb..a4f3a5f 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -82,6 +82,8 @@ func InitEnv() { ENV.VARIABLES["NUMBER"] = config.String("number") ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") + log.Debug("Recipients: ", ENV.VARIABLES["RECIPIENTS"].([]string)[0]) + ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") } diff --git a/utils/safestrings/safestrings.go b/utils/safestrings/safestrings.go index 650b362..29621b6 100644 --- a/utils/safestrings/safestrings.go +++ b/utils/safestrings/safestrings.go @@ -1,7 +1,6 @@ package safestrings import ( - "fmt" "regexp" "strconv" "strings" @@ -44,8 +43,6 @@ func ToType(str string) any { } } - fmt.Print(str) - return str } From 2b882a5e667ec0c73de10a13841c698d347df40d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 22:28:36 +0200 Subject: [PATCH 30/97] assume `[item]` to become an array --- utils/config/config.go | 2 -- utils/safestrings/safestrings.go | 10 ++++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index a4f3a5f..81b8beb 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -82,8 +82,6 @@ func InitEnv() { ENV.VARIABLES["NUMBER"] = config.String("number") ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") - log.Debug("Recipients: ", ENV.VARIABLES["RECIPIENTS"].([]string)[0]) - ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") } diff --git a/utils/safestrings/safestrings.go b/utils/safestrings/safestrings.go index 29621b6..811e7b7 100644 --- a/utils/safestrings/safestrings.go +++ b/utils/safestrings/safestrings.go @@ -21,11 +21,17 @@ func ToType(str string) any { } //* Try String Slice - if Contains(str, ",") && IsEnclosedBy(cleaned, `[`, `]`) { + if IsEnclosedBy(cleaned, `[`, `]`) { bracketsless := strings.ReplaceAll(str, "[", "") bracketsless = strings.ReplaceAll(bracketsless, "]", "") - data := ToArray(bracketsless) + var data []string + + if Contains(str, ",") { + data = ToArray(bracketsless) + } else { + data = []string{bracketsless} + } if data != nil { if len(data) > 0 { From 38f365ba05935cf0bc8114036ef178d2f8c1f2f6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 22:36:20 +0200 Subject: [PATCH 31/97] update README before config-README-update --- .github/templates/README.template.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 90f443a..b81a5b4 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -145,7 +145,12 @@ if you format them correctly... | array(int) | [1,2,3] | | array(str) | [a,b,c] | -This formatting applies to almost every situation where the only (allowed) Input Type is a string and other Output Types are needed. +> [!TIP] +> This formatting applies to almost every situation where the only (allowed) Input Type is a string and other Output Types are needed. + +> [!NOTE] +> If you have a string that should not be turned into any other type, then you will need to escape the `[]` or `{}` (also `-`) with a `\` **Backslash** > **Double Backslashes** do exist in that case you could just leave them out completly. +> An **Odd** number of **Backslashes** **escape** the character in front of them and an **Even** number leave the character **as-is**. ### API Token(s) From f3f7be1b26bf55605d48d7f3958056b7a4f9acbd Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 22:39:47 +0200 Subject: [PATCH 32/97] update README --- .github/templates/README.template.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index b81a5b4..13b51ec 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -132,6 +132,9 @@ Supported types include **strings**, **ints** and **arrays**. See [Formatting](# ### String To Type +> [!TIP] +> This formatting applies to almost every situation where the only (allowed) Input Type is a string and other Output Types are needed. + In the Environment the only allowed type is a string so to not have to always use a json string you can use the following types, if you format them correctly... @@ -145,11 +148,8 @@ if you format them correctly... | array(int) | [1,2,3] | | array(str) | [a,b,c] | -> [!TIP] -> This formatting applies to almost every situation where the only (allowed) Input Type is a string and other Output Types are needed. - > [!NOTE] -> If you have a string that should not be turned into any other type, then you will need to escape the `[]` or `{}` (also `-`) with a `\` **Backslash** > **Double Backslashes** do exist in that case you could just leave them out completly. +> If you have a string that should not be turned into any other type, then you will need to escape all Type Denotations, `[]` or `{}` (also `-`) with a `\` **Backslash** > **Double Backslashes** do exist in that case you could just leave them out completly. > An **Odd** number of **Backslashes** **escape** the character in front of them and an **Even** number leave the character **as-is**. ### API Token(s) From a5a101bb46f2eae2f0b8c19c688ab229132aefed Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 23:52:45 +0200 Subject: [PATCH 33/97] Update .dockerignore --- .dockerignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 538c10c..c23627f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,4 @@ .github *.yaml *.md -!LICENSE.md -!README.md -!config/*.yml \ No newline at end of file +!config/*.yml From 8dd6279bd815756fe84de1903ad83e4810d94694 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 23:53:01 +0200 Subject: [PATCH 34/97] Rename LICENSE.md to LICENSE --- LICENSE.md => LICENSE | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE.md => LICENSE (100%) diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE From 401ed2dd518a43d66bf37ffc5a6ed4dec432b468 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 12 Sep 2025 23:53:28 +0200 Subject: [PATCH 35/97] Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1aab5cf..9c1aee2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,10 +13,10 @@ ARG TARGETARCH WORKDIR /app -COPY ./config ./config/ +COPY . . COPY dist/${TARGETOS}/${TARGETARCH}/app . RUN ls -CMD ["./app"] \ No newline at end of file +CMD ["./app"] From e9959f66d610d9e76cb4667e0d2032cd4fab0560 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 11:22:11 +0200 Subject: [PATCH 36/97] added templating for configs **testing** --- config/defaults.yml | 8 +++--- main.go | 6 +++++ utils/config/config.go | 57 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/config/defaults.yml b/config/defaults.yml index 50d626d..d0a67a5 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -1,9 +1,7 @@ server: port: 8880 -api: - tokens: - url: +logLevel: "INFO" messageAliases: [ @@ -26,6 +24,10 @@ messageAliases: { alias: data, score: 1 }, ] +variables: + recipients: ${RECIPIENTS} + number: ${NUMBER} + blockedEndpoints: - /v1/about - /v1/configuration diff --git a/main.go b/main.go index 874b722..73b4040 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,12 @@ func main() { ENV = config.ENV + logLevel = ENV.LOG_LEVEL + + if logLevel != "" { + log.Init(logLevel) + } + initHandler = proxy.Create(ENV.API_URL) body_m4 := middlewares.BodyMiddleware{ diff --git a/utils/config/config.go b/utils/config/config.go index 81b8beb..d8084f9 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -22,6 +22,7 @@ type ENV_ struct { CONFIG_PATH string DEFAULTS_PATH string TOKENS_DIR string + LOG_LEVEL string PORT string API_URL string API_TOKENS []string @@ -49,6 +50,8 @@ var config *koanf.Koanf func InitEnv() { ENV.PORT = strconv.Itoa(config.Int("server.port")) + + ENV.LOG_LEVEL = config.String("loglevel") ENV.API_URL = config.String("api.url") @@ -77,10 +80,12 @@ func InitEnv() { } config.Unmarshal("messagealiases", &ENV.MESSAGE_ALIASES) - config.Unmarshal("variables", &ENV.VARIABLES) - ENV.VARIABLES["NUMBER"] = config.String("number") - ENV.VARIABLES["RECIPIENTS"] = config.Strings("recipients") + transformChildren(config, "variables", func(key string, value any) (string, any) { + return strings.ToUpper(key), value + }) + + config.Unmarshal("variables", &ENV.VARIABLES) ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") } @@ -114,6 +119,8 @@ func Load() { normalizeKeys(config) + templateConfig(config) + InitEnv() log.Info("Finished Loading Configuration") @@ -141,6 +148,20 @@ func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Prov return f, err } +func templateConfig(config *koanf.Koanf) { + data := config.All() + + for key, value := range data { + str, isStr := value.(string) + + if isStr { + data[key] = os.ExpandEnv(str) + } + } + + config.Load(confmap.Provider(data, "."), nil) +} + func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { e := env.Provider(".", env.Opt{ TransformFunc: normalizeEnv, @@ -175,8 +196,36 @@ func normalizeKeys(config *koanf.Koanf) { config.Load(confmap.Provider(data, "."), nil) } +func transformChildren(config *koanf.Koanf, prefix string, transform func(key string, value any) (string, any)) error { + var sub map[string]any + if err := config.Unmarshal(prefix, &sub); err != nil { + return err + } + + transformed := make(map[string]any) + for key, val := range sub { + newKey, newVal := transform(key, val) + + transformed[newKey] = newVal + } + + // Remove the old subtree by overwriting with empty map + config.Load(confmap.Provider(map[string]any{ + prefix: map[string]any{}, + }, "."), nil) + + // Load the normalized subtree back in + config.Load(confmap.Provider(map[string]any{ + prefix: transformed, + }, "."), nil) + + return nil +} + func normalizeEnv(key string, value string) (string, any) { - key = strings.ToLower(strings.ReplaceAll(key, "__", ".")) + key = strings.ToLower(key) + key = strings.ReplaceAll(key, "__", ".") + key = strings.ReplaceAll(key, "_", "") return key, safestrings.ToType(value) } \ No newline at end of file From 23daaf39150e65d508a4b63c30ebd9a123bc2ade Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 11:35:22 +0200 Subject: [PATCH 37/97] fix empty templating --- utils/config/config.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index d8084f9..74b0518 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -155,7 +155,11 @@ func templateConfig(config *koanf.Koanf) { str, isStr := value.(string) if isStr { - data[key] = os.ExpandEnv(str) + templated := os.ExpandEnv(str) + + if templated != "" { + data[key] = templated + } } } From ea9efd0bd4cbeb27956de7885c9dbe9e4156d7af Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 11:39:08 +0200 Subject: [PATCH 38/97] debug config --- main.go | 6 ++---- utils/config/config.go | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 73b4040..4c1021b 100644 --- a/main.go +++ b/main.go @@ -25,10 +25,8 @@ func main() { ENV = config.ENV - logLevel = ENV.LOG_LEVEL - - if logLevel != "" { - log.Init(logLevel) + if ENV.LOG_LEVEL != "" { + log.Init(ENV.LOG_LEVEL) } initHandler = proxy.Create(ENV.API_URL) diff --git a/utils/config/config.go b/utils/config/config.go index 74b0518..303f941 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -8,6 +8,7 @@ import ( "strings" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" + "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/safestrings" @@ -124,6 +125,8 @@ func Load() { InitEnv() log.Info("Finished Loading Configuration") + + log.Debug(utils.ToJson(config.All())) } func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Provider, error) { From 23cf8daed6d11a8731cdd2d653615637a3a33243 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 11:49:53 +0200 Subject: [PATCH 39/97] lower Log Level --- utils/logger/logger.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/logger/logger.go b/utils/logger/logger.go index 78887b3..5aac5ee 100644 --- a/utils/logger/logger.go +++ b/utils/logger/logger.go @@ -47,6 +47,8 @@ func Init(level string) { } func getLogLevel(level string) zapcore.Level { + level = strings.ToLower(level) + switch level { case "info": return zapcore.InfoLevel From 16871782afb630ff8952837818052bc5ec04d574 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 11:58:23 +0200 Subject: [PATCH 40/97] debug ENV --- utils/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/config/config.go b/utils/config/config.go index 303f941..b38eb70 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -127,6 +127,7 @@ func Load() { log.Info("Finished Loading Configuration") log.Debug(utils.ToJson(config.All())) + log.Debug(utils.ToJson(ENV)) } func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Provider, error) { From 64c3f58fc1ccca4c4db56970760ff6838a8daa6d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 12:01:51 +0200 Subject: [PATCH 41/97] overwrite config --- utils/config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/config/config.go b/utils/config/config.go index b38eb70..10cf1b8 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -201,6 +201,8 @@ func normalizeKeys(config *koanf.Koanf) { data[lower] = config.Get(key) } + + config.Delete("") config.Load(confmap.Provider(data, "."), nil) } From f0896d27e39892544d914ca9ac8b16016b8808b5 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 12:16:39 +0200 Subject: [PATCH 42/97] lock Load() during Watch --- utils/config/config.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 10cf1b8..53ce899 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -6,6 +6,7 @@ import ( "os" "strconv" "strings" + "sync" middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" "github.com/codeshelldev/secured-signal-api/utils" @@ -49,6 +50,8 @@ var userLayer = koanf.New(".") var config *koanf.Koanf +var configLock sync.Mutex + func InitEnv() { ENV.PORT = strconv.Itoa(config.Int("server.port")) @@ -146,6 +149,9 @@ func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Prov log.Info("Config changed, Reloading...") + configLock.Lock() + defer configLock.Unlock() + Load() }) @@ -201,7 +207,7 @@ func normalizeKeys(config *koanf.Koanf) { data[lower] = config.Get(key) } - + config.Delete("") config.Load(confmap.Provider(data, "."), nil) } @@ -218,13 +224,11 @@ func transformChildren(config *koanf.Koanf, prefix string, transform func(key st transformed[newKey] = newVal } - - // Remove the old subtree by overwriting with empty map + config.Load(confmap.Provider(map[string]any{ prefix: map[string]any{}, }, "."), nil) - // Load the normalized subtree back in config.Load(confmap.Provider(map[string]any{ prefix: transformed, }, "."), nil) From f81251ee2d4170ccf2e9f519e7ef58920f0aa40b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:08:34 +0200 Subject: [PATCH 43/97] updated README and surrounding files --- .github/templates/README.template.md | 151 +++++++++++++++------------ Dockerfile | 2 +- config/defaults.yml | 2 +- docker-compose.yaml | 4 +- examples/config.yml | 17 +++ 5 files changed, 105 insertions(+), 71 deletions(-) create mode 100644 examples/config.yml diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 13b51ec..18b1c73 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -1,22 +1,22 @@ Secure Proxy for Signal REST API -
Secure Proxy for Signal REST API
+
Secure Proxy for Signal Messenger REST API
-## Installation +## Getting Started Get the latest version of the `docker-compose.yaml` file: -And add secure Token(s) to `API__TOKEN` / `API__TOKENS`. See [API TOKEN(s)](#api-tokens) +```yaml +{ { file.docker-compose.yaml } } +``` + +And add secure Token(s) to `api.tokens`. See [API TOKEN(s)](#api-tokens). > [!IMPORTANT] > This Documentation will be using `sec-signal-api:8880` as the service host, > this **is just for simplicty**, instead use your containers or hosts IP + Port. > Or a hostname if applicable. See [Reverse Proxy](#reverse-proxy) -```yaml -{ { file.docker-compose.yaml } } -``` - ### Reverse proxy Take a look at the [traefik](https://github.com/traefik/traefik) implementation: @@ -90,9 +90,7 @@ curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer API_T #### Placeholders -If you are not comfortable / don't want to hardcode your Number and/or Recipients in you may use **Placeholders** in your Request. - -Built-in Placeholders: `{{ .NUMBER }}` and `{{ .RECIPIENTS }}` +If you are not comfortable / don't want to hardcode your Number for example and/or Recipients in you, may use **Placeholders** in your Request. See [Custom Variables](#variables). These Placeholders can be used in the Request Query or the Body of a Request like so: @@ -128,15 +126,42 @@ you have to add `@` in front of any KeyValue Pair assignment. Supported types include **strings**, **ints** and **arrays**. See [Formatting](#string-to-type). -## Environment Variables +## Configuration + +There are multiple ways to configure Secured Signal API, you can optionally use `config.yml` aswell as Environment Variables to override the config. + +### Config File + +Config files allow **YML** formatting and also `${ENV}` to get Environment Variables. + +To change the internal config file location set `CONFIG_PATH` in your **Environment** to an absolute path including the filename.extension. (default: `/config/config.yml`) + +This example config shows all of the individual settings that can be applied: + +```yaml +{ { file.examples/config.yml } } +``` + +### Environment + +Suppose you want to set a new [Placeholder](#placeholders) `NUMBER` in your Environment... + +```yaml +environment: + VARIABLES__NUMBER: "000" +``` + +This would internally be converted into `variables.number` matching the config formatting. + +> [!IMPORTANT] +> Underscores `_` are removed during Conversion, Double Underscores `__` on the other hand convert the Variable into a nested Object (`__` replaced by `.`) ### String To Type > [!TIP] > This formatting applies to almost every situation where the only (allowed) Input Type is a string and other Output Types are needed. -In the Environment the only allowed type is a string so to not have to always use a json string you can use the following types, -if you format them correctly... +If you are using Environment Variables as an example you won't be able to specify an Array or a Dictionary of items, in that case you can provide a specifically formatted string which will be translated into the correct type... | type | example | | :--------- | :---------------- | @@ -149,22 +174,25 @@ if you format them correctly... | array(str) | [a,b,c] | > [!NOTE] -> If you have a string that should not be turned into any other type, then you will need to escape all Type Denotations, `[]` or `{}` (also `-`) with a `\` **Backslash** > **Double Backslashes** do exist in that case you could just leave them out completly. +> If you have a string that should not be turned into any other type, then you will need to escape all Type Denotations, `[]` or `{}` (also `-`) with a `\` **Backslash**. +> **Double Backslashes** do exist but you could just leave them out completly. > An **Odd** number of **Backslashes** **escape** the character in front of them and an **Even** number leave the character **as-is**. ### API Token(s) -Both `API__TOKEN` and `API__TOKENS` support multiple Tokens seperated by a `,` **Comma** and `[]` **Brackets**. See [Formatting](#string-to-type). During Authentication Secured Signal API will try to match the given Token against the list of Tokens inside of these Variables. +> [!NOTE] +> Both `api.token` and `api.tokens` support multiple Tokens. + ```yaml -environment: - API__TOKEN: [token1, token2, token3] - API__TOKENS: [token1, token2, token3] +api: + token: [token1, token2, token3] + tokens: [token1, token2, token3] ``` > [!IMPORTANT] -> It is highly recommended to set this Environment Variable +> It is highly recommended use API Tokens > _What if I just don't?_ @@ -172,7 +200,7 @@ Secured Signal API will still work, but important Security Features won't be ava like Blocked Endpoints and any sort of Auth. > [!NOTE] -> Blocked Endpoints can be reactivated by manually setting them in the Environment +> Blocked Endpoints can be reactivated by manually configuring them ### Blocked Endpoints @@ -189,53 +217,27 @@ Because Secured Signal API is just a Proxy you can use all of the [Signal REST A | **/v1/accounts** | | **/v1/contacts** | -These Endpoints are blocked by default due to Security Risks, but can be modified by setting `BLOCKED_ENDPOINTS` to a Comma seperated List: +These Endpoints are blocked by default due to Security Risks, but can be modified by setting `blockedEndpoints` in your config: ```yaml -environment: - BLOCKED_ENDPOINTS: | - /v1/register, - /v1/unregister, - /v1/qrcodelink, - /v1/contacts, +blockedEndpoints: [/v1/register, /v1/unregister, /v1/qrcodelink, /v1/contacts] ``` -#### Variables - -By default Secured Signal API provides the following Placeholders: +### Variables -- **NUMBER** = _ENV_: `NUMBER` -- **RECIPIENTS** = _ENV_: `RECIPIENTS` +Placeholders can be added under `variables` and can then be referenced in the Body, Query or URL. +See [Placeholders](#placeholders). -### Customization - -Placeholders can be added by setting `VARIABLES` inside your Environment. - -```yaml -environment: - VARIABLES: | - "NUMBER2": "002", - "GROUP_CHAT_1": [ - "user.id", "000", "001", "group.id" - ] -``` - -### Recipients - -Set this Environment Variable to automatically provide default Recipients: +> [!NOTE] +> Every Placeholder Key will be converted into an Uppercase String. +> Example: `number` becomes `NUMBER` in `{{.NUMBER}}` ```yaml -environment: - RECIPIENTS: | - [ user.id, 000, 001, group.id ] -``` - -example: - -```json -{ - "recipients": "{{.RECIPIENTS}}" -} +variables: + number: "001", + recipients: [ + "user.id", "000", "001", "group.id" + ] ``` ### Message Aliases @@ -256,18 +258,33 @@ To improve compatibility with other services Secured Signal API provides aliases Secured Signal API will pick the best scoring Message Alias (if available) to extract the correct message from the Request Body. -Message Aliases can be added by setting `MESSAGE_ALIASES` to a valid json array containing dictionaries of `alias`, the json key to be used for lookup (use `.` dots for using values from a nested dictionary and `[i]` to get values from an array): +Message Aliases can be added by setting `messageAliases` in your config: ```yaml -environment: - MESSAGE_ALIASES: | - [ - { "alias": "msg", "score": 80 }, - { "alias": "data.message", "score": 79 }, - { "alias": "array[0].message", "score": 78 }, - ] +messageAliases: + [ + { alias: "msg", score: 80 }, + { alias: "data.message", score: 79 }, + { alias: "array[0].message", score: 78 }, + ] ``` +### Port + +To change the Port which Secured Signal API uses, you need to set `server.port` in your config. (default: `8880`) + +### Log Level + +To change the Log Level set `logLevel` to: (default: `info`) + +| Level | +| ------- | +| `info` | +| `debug` | +| `warn` | +| `error` | +| `fatal` | + ## Contributing Found a bug? Want to change or add something? diff --git a/Dockerfile b/Dockerfile index 9c1aee2..109e2c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:latest RUN apk --no-cache add ca-certificates -ENV SERVER_PORT=8880 +ENV SERVER__PORT=8880 ENV DEFAULTS_PATH=/app/config/defaults.yml diff --git a/config/defaults.yml b/config/defaults.yml index d0a67a5..db61d52 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -1,7 +1,7 @@ server: port: 8880 -logLevel: "INFO" +logLevel: INFO messageAliases: [ diff --git a/docker-compose.yaml b/docker-compose.yaml index 25edd98..9979186 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,8 +21,8 @@ services: - secured-signal-api environment: API__URL: http://signal-api:8080 - RECIPIENTS: 000,001,002 - NUMBER: 123456789 + VARIABLES__RECIPIENTS: 000,001,002 + VARIABLES__NUMBER: 123456789 API__TOKENS: LOOOOOONG_STRING ports: - "8880:8880" diff --git a/examples/config.yml b/examples/config.yml new file mode 100644 index 0000000..f8e24c3 --- /dev/null +++ b/examples/config.yml @@ -0,0 +1,17 @@ +# Example Config (all configurations shown) + +api: + port: 8880 + url: http://signal-api:8080 + tokens: [token1, token2] + +logLevel: INFO + +variables: + number: "000" + recipients: ["001", "group.id", "user.id"] + +messageAliases: [{ alias: "msg", score: 100 }] + +blockedEndpoints: + - /v1/about From 7d58ff2aed9b19982645a6c0e438513198b6ddf2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:09:24 +0200 Subject: [PATCH 44/97] Update docker-image-dev.yml --- .github/workflows/docker-image-dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml index 4105f80..21bd4a8 100644 --- a/.github/workflows/docker-image-dev.yml +++ b/.github/workflows/docker-image-dev.yml @@ -6,7 +6,6 @@ on: - dev paths: - "**/*.go" - - "config/*.yml" jobs: update: From e427018bf02d39f0f8b2c7862a2dd118f46d93d7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:16:49 +0200 Subject: [PATCH 45/97] added `dev` log level --- .github/templates/README.template.md | 1 + utils/config/config.go | 4 ++-- utils/logger/logger.go | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 18b1c73..9dea71f 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -284,6 +284,7 @@ To change the Log Level set `logLevel` to: (default: `info`) | `warn` | | `error` | | `fatal` | +| `dev` | ## Contributing diff --git a/utils/config/config.go b/utils/config/config.go index 53ce899..7719287 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -129,8 +129,8 @@ func Load() { log.Info("Finished Loading Configuration") - log.Debug(utils.ToJson(config.All())) - log.Debug(utils.ToJson(ENV)) + log.Dev(utils.ToJson(config.All())) + log.Dev(utils.ToJson(ENV)) } func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Provider, error) { diff --git a/utils/logger/logger.go b/utils/logger/logger.go index 5aac5ee..871ba94 100644 --- a/utils/logger/logger.go +++ b/utils/logger/logger.go @@ -9,9 +9,12 @@ import ( ) var _log *zap.Logger +var _logLevel = "" func Init(level string) { - logLevel := getLogLevel(level) + _logLevel = strings.ToLower(level) + + logLevel := getLogLevel(_logLevel) cfg := zap.Config{ Level: zap.NewAtomicLevelAt(logLevel), @@ -47,13 +50,13 @@ func Init(level string) { } func getLogLevel(level string) zapcore.Level { - level = strings.ToLower(level) - switch level { case "info": return zapcore.InfoLevel case "debug": return zapcore.DebugLevel + case "dev": + return zapcore.DebugLevel case "warn": return zapcore.WarnLevel case "error": @@ -73,6 +76,12 @@ func Debug(msg ...string) { _log.Debug(strings.Join(msg, "")) } +func Dev(msg ...string) { + if _logLevel == "dev" { + _log.Debug(strings.Join(msg, "")) + } +} + func Error(msg ...string) { _log.Error(strings.Join(msg, "")) } From ad7c084ad5dbd3a70ab733494c18da5202e20b2e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 15:13:06 +0200 Subject: [PATCH 46/97] added token configs * removed `api.token` in favor of `api.tokens` * added configs for tokens to override settings from main config --- .github/templates/README.template.md | 20 +++- examples/token.yml | 7 ++ utils/config/config.go | 161 +++++++-------------------- utils/config/loader.go | 118 ++++++++++++++++++++ utils/config/token-config.go | 63 +++++++++++ 5 files changed, 241 insertions(+), 128 deletions(-) create mode 100644 examples/token.yml create mode 100644 utils/config/loader.go create mode 100644 utils/config/token-config.go diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 9dea71f..57924fe 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -10,7 +10,7 @@ Get the latest version of the `docker-compose.yaml` file: { { file.docker-compose.yaml } } ``` -And add secure Token(s) to `api.tokens`. See [API TOKEN(s)](#api-tokens). +And add secure Token(s) to `api.tokens`. See [API TOKENs](#api-tokens). > [!IMPORTANT] > This Documentation will be using `sec-signal-api:8880` as the service host, @@ -130,7 +130,7 @@ Supported types include **strings**, **ints** and **arrays**. See [Formatting](# There are multiple ways to configure Secured Signal API, you can optionally use `config.yml` aswell as Environment Variables to override the config. -### Config File +### Config Files Config files allow **YML** formatting and also `${ENV}` to get Environment Variables. @@ -142,6 +142,18 @@ This example config shows all of the individual settings that can be applied: { { file.examples/config.yml } } ``` +#### Token Configs + +You can also override the `config.yml` file for each individual token by adding configs under `TOKENS_PATH` (default: `config/tokens/`) + +This way you can permission tokens by further restricting or adding [Endpoints](#blocked-endpoints), [Placeholders](#variables), etc. + +Here is an example: + +```yaml +{ { file.examples/token.yml } } +``` + ### Environment Suppose you want to set a new [Placeholder](#placeholders) `NUMBER` in your Environment... @@ -182,12 +194,8 @@ If you are using Environment Variables as an example you won't be able to specif During Authentication Secured Signal API will try to match the given Token against the list of Tokens inside of these Variables. -> [!NOTE] -> Both `api.token` and `api.tokens` support multiple Tokens. - ```yaml api: - token: [token1, token2, token3] tokens: [token1, token2, token3] ``` diff --git a/examples/token.yml b/examples/token.yml new file mode 100644 index 0000000..b9b384c --- /dev/null +++ b/examples/token.yml @@ -0,0 +1,7 @@ +token: LOOOONG_STRING + +overrides: + variables: # Disable Placeholder + blockedEndpoints: # Disable Sending + - /v2/send + messageAliases: # Disable Aliases diff --git a/utils/config/config.go b/utils/config/config.go index 7719287..b5f1a22 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,161 +1,92 @@ package config import ( - "errors" - "io/fs" "os" + "path/filepath" "strconv" "strings" "sync" - middlewares "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" - "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/safestrings" - "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/confmap" "github.com/knadh/koanf/providers/env/v2" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/v2" ) -type ENV_ struct { - CONFIG_PATH string - DEFAULTS_PATH string - TOKENS_DIR string - LOG_LEVEL string - PORT string - API_URL string - API_TOKENS []string - INSECURE bool - BLOCKED_ENDPOINTS []string - VARIABLES map[string]any - MESSAGE_ALIASES []middlewares.MessageAlias -} - -var ENV ENV_ = ENV_{ - CONFIG_PATH: os.Getenv("CONFIG_PATH"), - DEFAULTS_PATH: os.Getenv("DEFAULTS_PATH"), - TOKENS_DIR: os.Getenv("TOKENS_DIR"), - API_TOKENS: []string{}, - BLOCKED_ENDPOINTS: []string{}, - MESSAGE_ALIASES: []middlewares.MessageAlias{}, - VARIABLES: map[string]any{}, - INSECURE: false, -} - var defaultsLayer = koanf.New(".") var userLayer = koanf.New(".") +var tokensLayer = koanf.New(".") var config *koanf.Koanf var configLock sync.Mutex -func InitEnv() { - ENV.PORT = strconv.Itoa(config.Int("server.port")) - - ENV.LOG_LEVEL = config.String("loglevel") - - ENV.API_URL = config.String("api.url") - - apiTokens := config.Strings("api.tokens") - - if len(apiTokens) <= 0 { - apiTokens = config.Strings("api.token") - - if len(apiTokens) <= 0 { - log.Warn("No API TOKEN provided this is NOT recommended") +func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Provider, error) { + f := file.Provider(path) - log.Info("Disabling Security Features due to incomplete Congfiguration") + err := config.Load(f, parser) - ENV.INSECURE = true + if err != nil { + return nil, err + } - // Set Blocked Endpoints on Config to User Layer Value - // => effectively ignoring Default Layer - config.Set("blockedendpoints", userLayer.Strings("blockeendpoints")) + f.Watch(func(event any, err error) { + if err != nil { + return } - } - if len(apiTokens) > 0 { - log.Debug("Registered " + strconv.Itoa(len(apiTokens)) + " Tokens") + log.Info(path, " changed, Reloading...") - ENV.API_TOKENS = apiTokens - } - - config.Unmarshal("messagealiases", &ENV.MESSAGE_ALIASES) + configLock.Lock() + defer configLock.Unlock() - transformChildren(config, "variables", func(key string, value any) (string, any) { - return strings.ToUpper(key), value + Load() }) - config.Unmarshal("variables", &ENV.VARIABLES) - - ENV.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") + return f, err } -func Load() { - log.Debug("Loading Config ", ENV.DEFAULTS_PATH) - - _, defErr := LoadFile(ENV.DEFAULTS_PATH, defaultsLayer, yaml.Parser()) +func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) error { + files, err := filepath.Glob(filepath.Join(dir, "*.yml")) - if defErr != nil { - log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) - } - - log.Debug("Loading Config ", ENV.CONFIG_PATH) + if err != nil { + return err + } - _, conErr := LoadFile(ENV.CONFIG_PATH, userLayer, yaml.Parser()) + for i, file := range files { + tmp := koanf.New(".") - if conErr != nil { - _, err := os.Stat(ENV.CONFIG_PATH) + _, err := LoadFile(file, tmp, parser) - if !errors.Is(err, fs.ErrNotExist) { - log.Error("Could not Load Config ", ENV.CONFIG_PATH, ": ", conErr.Error()) + if err != nil { + return err } - } - - log.Debug("Loading DotEnv") - - LoadEnv(userLayer) - - config = mergeLayers() - normalizeKeys(config) - - templateConfig(config) - - InitEnv() - - log.Info("Finished Loading Configuration") + config.Set(path + "." + strconv.Itoa(i), tmp.All()) + } - log.Dev(utils.ToJson(config.All())) - log.Dev(utils.ToJson(ENV)) + return nil } -func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Provider, error) { - f := file.Provider(path) +func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { + e := env.Provider(".", env.Opt{ + TransformFunc: normalizeEnv, + }) - err := config.Load(f, parser) + err := config.Load(e, nil) if err != nil { - return nil, err + log.Fatal("Error loading env: ", err.Error()) } - f.Watch(func(event any, err error) { - if err != nil { - return - } - - log.Info("Config changed, Reloading...") - - configLock.Lock() - defer configLock.Unlock() - - Load() - }) + return e, err +} - return f, err +func mergeConfig(path string, mergeInto *koanf.Koanf, mergeFrom *koanf.Koanf) { + mergeInto.MergeAt(mergeFrom, path) } func templateConfig(config *koanf.Koanf) { @@ -176,20 +107,6 @@ func templateConfig(config *koanf.Koanf) { config.Load(confmap.Provider(data, "."), nil) } -func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { - e := env.Provider(".", env.Opt{ - TransformFunc: normalizeEnv, - }) - - err := config.Load(e, nil) - - if err != nil { - log.Fatal("Error loading env: ", err.Error()) - } - - return e, err -} - func mergeLayers() *koanf.Koanf { final := koanf.New(".") diff --git a/utils/config/loader.go b/utils/config/loader.go new file mode 100644 index 0000000..573950a --- /dev/null +++ b/utils/config/loader.go @@ -0,0 +1,118 @@ +package config + +import ( + "errors" + "io/fs" + "os" + "strconv" + "strings" + + "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" + "github.com/codeshelldev/secured-signal-api/utils" + log "github.com/codeshelldev/secured-signal-api/utils/logger" + "github.com/knadh/koanf/parsers/yaml" +) + +type ENV_ struct { + CONFIG_PATH string + DEFAULTS_PATH string + TOKENS_DIR string + LOG_LEVEL string + PORT string + API_URL string + API_TOKENS []string + SETTINGS map[string]*SETTING_ + INSECURE bool +} + +type SETTING_ struct { + BLOCKED_ENDPOINTS []string + VARIABLES map[string]any + MESSAGE_ALIASES []middlewares.MessageAlias +} + +var ENV ENV_ = ENV_{ + CONFIG_PATH: os.Getenv("CONFIG_PATH"), + DEFAULTS_PATH: os.Getenv("DEFAULTS_PATH"), + TOKENS_DIR: os.Getenv("TOKENS_DIR"), + API_TOKENS: []string{}, + SETTINGS: map[string]*SETTING_{ + "*": &SETTING_{ + BLOCKED_ENDPOINTS: []string{}, + MESSAGE_ALIASES: []middlewares.MessageAlias{}, + VARIABLES: map[string]any{}, + }, + }, + INSECURE: false, +} + +func InitEnv() { + ENV.PORT = strconv.Itoa(config.Int("server.port")) + + ENV.LOG_LEVEL = config.String("loglevel") + + ENV.API_URL = config.String("api.url") + + InitTokens() + + defaultSettings := ENV.SETTINGS["*"] + + config.Unmarshal("messagealiases", &defaultSettings.MESSAGE_ALIASES) + + transformChildren(config, "variables", func(key string, value any) (string, any) { + return strings.ToUpper(key), value + }) + + config.Unmarshal("variables", &defaultSettings.VARIABLES) + + defaultSettings.BLOCKED_ENDPOINTS = config.Strings("blockedendpoints") +} + +func Load() { + LoadDefaults() + + LoadConfig() + + LoadTokens() + + log.Debug("Loading DotEnv") + + LoadEnv(userLayer) + + config = mergeLayers() + + normalizeKeys(config) + + templateConfig(config) + + InitEnv() + + log.Info("Finished Loading Configuration") + + log.Dev(utils.ToJson(config.All())) + log.Dev(utils.ToJson(ENV)) +} + +func LoadDefaults() { + log.Debug("Loading Config ", ENV.DEFAULTS_PATH) + + _, defErr := LoadFile(ENV.DEFAULTS_PATH, defaultsLayer, yaml.Parser()) + + if defErr != nil { + log.Warn("Could not Load Defaults", ENV.DEFAULTS_PATH) + } +} + +func LoadConfig() { + log.Debug("Loading Config ", ENV.CONFIG_PATH) + + _, conErr := LoadFile(ENV.CONFIG_PATH, userLayer, yaml.Parser()) + + if conErr != nil { + _, err := os.Stat(ENV.CONFIG_PATH) + + if !errors.Is(err, fs.ErrNotExist) { + log.Error("Could not Load Config ", ENV.CONFIG_PATH, ": ", conErr.Error()) + } + } +} \ No newline at end of file diff --git a/utils/config/token-config.go b/utils/config/token-config.go new file mode 100644 index 0000000..e30a86d --- /dev/null +++ b/utils/config/token-config.go @@ -0,0 +1,63 @@ +package config + +import ( + "strconv" + + log "github.com/codeshelldev/secured-signal-api/utils/logger" + "github.com/knadh/koanf/parsers/yaml" +) + +type TOKEN_CONFIG_ struct { + TOKEN string `koanf:"token"` + OVERRIDES *SETTING_ `koanf:"overrides"` +} + +func LoadTokens() { + log.Debug("Loading Configs ", ENV.TOKENS_DIR) + + LoadDir("tokens", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) +} + +func InitTokens() { + apiTokens := config.Strings("api.tokens") + + var tokenConfigs []TOKEN_CONFIG_ + + tokensLayer.Unmarshal("tokens", &tokenConfigs) + + overrides := ParseTokenConfigs(tokenConfigs) + + for token, override := range overrides { + apiTokens = append(apiTokens, token) + + ENV.SETTINGS[token] = override + } + + if len(apiTokens) <= 0 { + log.Warn("No API TOKEN provided this is NOT recommended") + + log.Info("Disabling Security Features due to incomplete Congfiguration") + + ENV.INSECURE = true + + // Set Blocked Endpoints on Config to User Layer Value + // => effectively ignoring Default Layer + config.Set("blockedendpoints", userLayer.Strings("blockeendpoints")) + } + + if len(apiTokens) > 0 { + log.Debug("Registered " + strconv.Itoa(len(apiTokens)) + " Tokens") + + ENV.API_TOKENS = apiTokens + } +} + +func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]*SETTING_) { + settings := map[string]*SETTING_{} + + for _, config := range configs { + settings[config.TOKEN] = config.OVERRIDES + } + + return settings +} \ No newline at end of file From 936d0742bf8f6a65d184aea31b2c252df06b67f8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:25:30 +0200 Subject: [PATCH 47/97] added settings overrides --- internals/proxy/middlewares/auth.go | 25 +++++++-------- internals/proxy/middlewares/body.go | 16 ++++------ internals/proxy/middlewares/common.go | 36 ++++++++++++++++++++++ internals/proxy/middlewares/endpoints.go | 6 ++-- internals/proxy/middlewares/template.go | 10 +++--- internals/proxy/middlewares/types/types.go | 6 ++++ main.go | 14 +++------ utils/config/loader.go | 10 +++--- 8 files changed, 77 insertions(+), 46 deletions(-) create mode 100644 internals/proxy/middlewares/common.go create mode 100644 internals/proxy/middlewares/types/types.go diff --git a/internals/proxy/middlewares/auth.go b/internals/proxy/middlewares/auth.go index a5676aa..7c7339c 100644 --- a/internals/proxy/middlewares/auth.go +++ b/internals/proxy/middlewares/auth.go @@ -1,28 +1,20 @@ package middlewares import ( + "context" "encoding/base64" "net/http" "slices" "strings" + "github.com/codeshelldev/secured-signal-api/utils/config" log "github.com/codeshelldev/secured-signal-api/utils/logger" ) type AuthMiddleware struct { - Next http.Handler - Tokens []string + Next http.Handler } -type authType string - -const ( - Bearer authType = "Bearer" - Basic authType = "Basic" - Query authType = "Query" - None authType = "None" -) - func getAuthType(str string) authType { switch str { case "Bearer": @@ -40,7 +32,7 @@ func isValidToken(tokens []string, match string) (bool) { func (data AuthMiddleware) Use() http.Handler { next := data.Next - tokens := data.Tokens + tokens := config.ENV.API_TOKENS return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if len(tokens) <= 0 { @@ -54,13 +46,15 @@ func (data AuthMiddleware) Use() http.Handler { var authType authType = None + var authToken string + success := false if authHeader != "" { authBody := strings.Split(authHeader, " ") authType = getAuthType(authBody[0]) - authToken := authBody[1] + authToken = authBody[1] switch authType { case Bearer: @@ -88,7 +82,7 @@ func (data AuthMiddleware) Use() http.Handler { } else if authQuery != "" { authType = Query - authToken := strings.TrimSpace(authQuery) + authToken = strings.TrimSpace(authQuery) if isValidToken(tokens, authToken) { success = true @@ -109,6 +103,9 @@ func (data AuthMiddleware) Use() http.Handler { return } + ctx := context.WithValue(req.Context(), tokenKey, authToken) + req = req.WithContext(ctx) + next.ServeHTTP(w, req) }) } diff --git a/internals/proxy/middlewares/body.go b/internals/proxy/middlewares/body.go index 57badc5..6f4351f 100644 --- a/internals/proxy/middlewares/body.go +++ b/internals/proxy/middlewares/body.go @@ -6,26 +6,22 @@ import ( "net/http" "strconv" + middlewareTypes "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares/types" "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" request "github.com/codeshelldev/secured-signal-api/utils/request" ) -type MessageAlias struct { - Alias string - Score int -} - type BodyMiddleware struct { - Next http.Handler - MessageAliases []MessageAlias + Next http.Handler } func (data BodyMiddleware) Use() http.Handler { next := data.Next - messageAliases := data.MessageAliases return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + messageAliases := GetSettings(req).MESSAGE_ALIASES + body, err := request.GetReqBody(w, req) if err != nil { @@ -70,7 +66,7 @@ func (data BodyMiddleware) Use() http.Handler { }) } -func getMessage(aliases []MessageAlias, data map[string]any) (string, map[string]any) { +func getMessage(aliases []middlewareTypes.MessageAlias, data map[string]any) (string, map[string]any) { var content string var best int @@ -87,7 +83,7 @@ func getMessage(aliases []MessageAlias, data map[string]any) (string, map[string return content, data } -func processAlias(alias MessageAlias, data map[string]any) (string, int, bool) { +func processAlias(alias middlewareTypes.MessageAlias, data map[string]any) (string, int, bool) { aliasKey := alias.Alias value, ok := utils.GetByPath(aliasKey, data) diff --git a/internals/proxy/middlewares/common.go b/internals/proxy/middlewares/common.go new file mode 100644 index 0000000..df97e61 --- /dev/null +++ b/internals/proxy/middlewares/common.go @@ -0,0 +1,36 @@ +package middlewares + +import ( + "net/http" + + "github.com/codeshelldev/secured-signal-api/utils/config" +) + +type Context struct { + Next http.Handler +} + +type authType string + +const ( + Bearer authType = "Bearer" + Basic authType = "Basic" + Query authType = "Query" + None authType = "None" +) + +type contextKey string + +const tokenKey contextKey = "token" + +func GetSettings(req *http.Request) config.SETTING_ { + token := req.Context().Value(tokenKey).(string) + + settings := config.ENV.SETTINGS[token] + + if settings == nil { + settings = config.ENV.SETTINGS["*"] + } + + return *settings +} \ No newline at end of file diff --git a/internals/proxy/middlewares/endpoints.go b/internals/proxy/middlewares/endpoints.go index 6a14c7e..11f1954 100644 --- a/internals/proxy/middlewares/endpoints.go +++ b/internals/proxy/middlewares/endpoints.go @@ -9,17 +9,17 @@ import ( type EndpointsMiddleware struct { Next http.Handler - BlockedEndpoints []string } func (data EndpointsMiddleware) Use() http.Handler { next := data.Next - BLOCKED_ENDPOINTS := data.BlockedEndpoints return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blockedEndpoints := GetSettings(req).BLOCKED_ENDPOINTS + reqPath := req.URL.Path - if slices.Contains(BLOCKED_ENDPOINTS, reqPath) { + if slices.Contains(blockedEndpoints, reqPath) { log.Warn("User tried to access blocked endpoint: ", reqPath) http.Error(w, "Forbidden", http.StatusForbidden) return diff --git a/internals/proxy/middlewares/template.go b/internals/proxy/middlewares/template.go index 1029718..26300c6 100644 --- a/internals/proxy/middlewares/template.go +++ b/internals/proxy/middlewares/template.go @@ -16,14 +16,14 @@ import ( type TemplateMiddleware struct { Next http.Handler - Variables map[string]any } func (data TemplateMiddleware) Use() http.Handler { next := data.Next - VARIABLES := data.Variables return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + variables := GetSettings(req).VARIABLES + body, err := request.GetReqBody(w, req) if err != nil { @@ -37,7 +37,7 @@ func (data TemplateMiddleware) Use() http.Handler { if !body.Empty { var modified bool - bodyData, modified, err = TemplateBody(body.Data, VARIABLES) + bodyData, modified, err = TemplateBody(body.Data, variables) if err != nil { log.Error("Error Templating JSON: ", err.Error()) @@ -51,7 +51,7 @@ func (data TemplateMiddleware) Use() http.Handler { if req.URL.RawQuery != "" { var modified bool - req.URL.RawQuery, bodyData, modified, err = TemplateQuery(req.URL, bodyData, VARIABLES) + req.URL.RawQuery, bodyData, modified, err = TemplateQuery(req.URL, bodyData, variables) if err != nil { log.Error("Error Templating Query: ", err.Error()) @@ -85,7 +85,7 @@ func (data TemplateMiddleware) Use() http.Handler { if req.URL.Path != "" { var modified bool - req.URL.Path, modified, err = TemplatePath(req.URL, VARIABLES) + req.URL.Path, modified, err = TemplatePath(req.URL, variables) if err != nil { log.Error("Error Templating Path: ", err.Error()) diff --git a/internals/proxy/middlewares/types/types.go b/internals/proxy/middlewares/types/types.go new file mode 100644 index 0000000..596687f --- /dev/null +++ b/internals/proxy/middlewares/types/types.go @@ -0,0 +1,6 @@ +package middlewareTypes + +type MessageAlias struct { + Alias string + Score int +} \ No newline at end of file diff --git a/main.go b/main.go index 4c1021b..9a8cdb5 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( var initHandler *httputil.ReverseProxy -var ENV config.ENV_ +var ENV *config.ENV_ func main() { logLevel := os.Getenv("LOG_LEVEL") @@ -32,27 +32,23 @@ func main() { initHandler = proxy.Create(ENV.API_URL) body_m4 := middlewares.BodyMiddleware{ - Next: initHandler, - MessageAliases: ENV.MESSAGE_ALIASES, + Next: initHandler, } temp_m3 := middlewares.TemplateMiddleware{ - Next: body_m4.Use(), - Variables: ENV.VARIABLES, + Next: body_m4.Use(), } endp_m2 := middlewares.EndpointsMiddleware{ - Next: temp_m3.Use(), - BlockedEndpoints: ENV.BLOCKED_ENDPOINTS, + Next: temp_m3.Use(), } auth_m1 := middlewares.AuthMiddleware{ Next: endp_m2.Use(), - Tokens: ENV.API_TOKENS, } log_m0 := middlewares.LogMiddleware{ - Next: auth_m1.Use(), + Next: auth_m1.Use(), } log.Info("Initialized Proxy Handler") diff --git a/utils/config/loader.go b/utils/config/loader.go index 573950a..af7c42c 100644 --- a/utils/config/loader.go +++ b/utils/config/loader.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares" + middlewareTypes "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares/types" "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" @@ -28,18 +28,18 @@ type ENV_ struct { type SETTING_ struct { BLOCKED_ENDPOINTS []string VARIABLES map[string]any - MESSAGE_ALIASES []middlewares.MessageAlias + MESSAGE_ALIASES []middlewareTypes.MessageAlias } -var ENV ENV_ = ENV_{ +var ENV *ENV_ = &ENV_{ CONFIG_PATH: os.Getenv("CONFIG_PATH"), DEFAULTS_PATH: os.Getenv("DEFAULTS_PATH"), TOKENS_DIR: os.Getenv("TOKENS_DIR"), API_TOKENS: []string{}, SETTINGS: map[string]*SETTING_{ - "*": &SETTING_{ + "*": { BLOCKED_ENDPOINTS: []string{}, - MESSAGE_ALIASES: []middlewares.MessageAlias{}, + MESSAGE_ALIASES: []middlewareTypes.MessageAlias{}, VARIABLES: map[string]any{}, }, }, From 8e6d593b27d1e0ca173af1def67ebbec465c8574 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:33:20 +0200 Subject: [PATCH 48/97] debugging --- utils/config/token-config.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index e30a86d..adffea0 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -8,14 +8,14 @@ import ( ) type TOKEN_CONFIG_ struct { - TOKEN string `koanf:"token"` + TOKENS []string `koanf:"tokens"` OVERRIDES *SETTING_ `koanf:"overrides"` } func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) - LoadDir("tokens", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) + LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) } func InitTokens() { @@ -23,7 +23,7 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - tokensLayer.Unmarshal("tokens", &tokenConfigs) + tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) overrides := ParseTokenConfigs(tokenConfigs) @@ -56,7 +56,9 @@ func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]*SETTING_) { settings := map[string]*SETTING_{} for _, config := range configs { - settings[config.TOKEN] = config.OVERRIDES + for _, token := range config.TOKENS { + settings[token] = config.OVERRIDES + } } return settings From 8ab4ee51080fe86a803c9a617351fbf87364d53d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:37:13 +0200 Subject: [PATCH 49/97] debugging+ --- internals/proxy/middlewares/types/types.go | 4 ++-- utils/config/token-config.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internals/proxy/middlewares/types/types.go b/internals/proxy/middlewares/types/types.go index 596687f..b22a473 100644 --- a/internals/proxy/middlewares/types/types.go +++ b/internals/proxy/middlewares/types/types.go @@ -1,6 +1,6 @@ package middlewareTypes type MessageAlias struct { - Alias string - Score int + Alias string `koanf:"alias"` + Score int `koanf:"score"` } \ No newline at end of file diff --git a/utils/config/token-config.go b/utils/config/token-config.go index adffea0..f311361 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -3,6 +3,7 @@ package config import ( "strconv" + "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -16,6 +17,8 @@ func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) + + log.Dev(utils.ToJson(tokensLayer.All())) } func InitTokens() { From 7762f0eec2b355e2d7d3337acbb54dcf56964d9e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:39:56 +0200 Subject: [PATCH 50/97] debugging++ --- utils/config/token-config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index f311361..e7e41bf 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -28,6 +28,8 @@ func InitTokens() { tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) + log.Dev(utils.ToJson(tokenConfigs)) + overrides := ParseTokenConfigs(tokenConfigs) for token, override := range overrides { From 8ca08d0b2851ad79ff838fc0f95af674d67b6a4d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:57:11 +0200 Subject: [PATCH 51/97] debug --- utils/config/loader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/config/loader.go b/utils/config/loader.go index af7c42c..740fdec 100644 --- a/utils/config/loader.go +++ b/utils/config/loader.go @@ -26,9 +26,9 @@ type ENV_ struct { } type SETTING_ struct { - BLOCKED_ENDPOINTS []string - VARIABLES map[string]any - MESSAGE_ALIASES []middlewareTypes.MessageAlias + BLOCKED_ENDPOINTS []string `koanf:"blockedendpoints"` + VARIABLES map[string]any `koanf:"variables"` + MESSAGE_ALIASES []middlewareTypes.MessageAlias `koanf:"messagealiases"` } var ENV *ENV_ = &ENV_{ From 7c6a7848b727cddcc59edfb1ea915230e1b4c35d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:03:34 +0200 Subject: [PATCH 52/97] fixed dirloading --- utils/config/config.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index b5f1a22..9383a89 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,7 +3,6 @@ package config import ( "os" "path/filepath" - "strconv" "strings" "sync" @@ -56,7 +55,9 @@ func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) return err } - for i, file := range files { + data := []map[string]any{} + + for _, file := range files { tmp := koanf.New(".") _, err := LoadFile(file, tmp, parser) @@ -65,9 +66,11 @@ func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) return err } - config.Set(path + "." + strconv.Itoa(i), tmp.All()) + data = append(data, tmp.All()) } + config.Set(path, data) + return nil } From 2cd9473fde2f7a5ee773f81aac9709c51501e555 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:07:40 +0200 Subject: [PATCH 53/97] debug --- internals/proxy/middlewares/body.go | 2 +- internals/proxy/middlewares/common.go | 2 +- internals/proxy/middlewares/endpoints.go | 2 +- internals/proxy/middlewares/template.go | 2 +- utils/config/token-config.go | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internals/proxy/middlewares/body.go b/internals/proxy/middlewares/body.go index 6f4351f..518bff6 100644 --- a/internals/proxy/middlewares/body.go +++ b/internals/proxy/middlewares/body.go @@ -20,7 +20,7 @@ func (data BodyMiddleware) Use() http.Handler { next := data.Next return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - messageAliases := GetSettings(req).MESSAGE_ALIASES + messageAliases := getSettings(req).MESSAGE_ALIASES body, err := request.GetReqBody(w, req) diff --git a/internals/proxy/middlewares/common.go b/internals/proxy/middlewares/common.go index df97e61..c98f29c 100644 --- a/internals/proxy/middlewares/common.go +++ b/internals/proxy/middlewares/common.go @@ -23,7 +23,7 @@ type contextKey string const tokenKey contextKey = "token" -func GetSettings(req *http.Request) config.SETTING_ { +func getSettings(req *http.Request) config.SETTING_ { token := req.Context().Value(tokenKey).(string) settings := config.ENV.SETTINGS[token] diff --git a/internals/proxy/middlewares/endpoints.go b/internals/proxy/middlewares/endpoints.go index 11f1954..fb0d400 100644 --- a/internals/proxy/middlewares/endpoints.go +++ b/internals/proxy/middlewares/endpoints.go @@ -15,7 +15,7 @@ func (data EndpointsMiddleware) Use() http.Handler { next := data.Next return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - blockedEndpoints := GetSettings(req).BLOCKED_ENDPOINTS + blockedEndpoints := getSettings(req).BLOCKED_ENDPOINTS reqPath := req.URL.Path diff --git a/internals/proxy/middlewares/template.go b/internals/proxy/middlewares/template.go index 26300c6..f8a4b0e 100644 --- a/internals/proxy/middlewares/template.go +++ b/internals/proxy/middlewares/template.go @@ -22,7 +22,7 @@ func (data TemplateMiddleware) Use() http.Handler { next := data.Next return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - variables := GetSettings(req).VARIABLES + variables := getSettings(req).VARIABLES body, err := request.GetReqBody(w, req) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index e7e41bf..ab9e0cd 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -10,7 +10,7 @@ import ( type TOKEN_CONFIG_ struct { TOKENS []string `koanf:"tokens"` - OVERRIDES *SETTING_ `koanf:"overrides"` + OVERRIDES SETTING_ `koanf:"overrides"` } func LoadTokens() { @@ -35,7 +35,7 @@ func InitTokens() { for token, override := range overrides { apiTokens = append(apiTokens, token) - ENV.SETTINGS[token] = override + *ENV.SETTINGS[token] = override } if len(apiTokens) <= 0 { @@ -57,8 +57,8 @@ func InitTokens() { } } -func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]*SETTING_) { - settings := map[string]*SETTING_{} +func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]SETTING_) { + settings := map[string]SETTING_{} for _, config := range configs { for _, token := range config.TOKENS { From 397e4ad59ef5a60178398ef29c0d1da5f753a405 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:13:27 +0200 Subject: [PATCH 54/97] fix? --- utils/config/token-config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index ab9e0cd..074a24f 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -35,7 +35,7 @@ func InitTokens() { for token, override := range overrides { apiTokens = append(apiTokens, token) - *ENV.SETTINGS[token] = override + ENV.SETTINGS[token] = &override } if len(apiTokens) <= 0 { From b32cbc2fcbca5f4507f8d0ea3d27f3f9951fc3af Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:29:13 +0200 Subject: [PATCH 55/97] testing --- internals/proxy/middlewares/common.go | 18 +++++++++++------- utils/config/token-config.go | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internals/proxy/middlewares/common.go b/internals/proxy/middlewares/common.go index c98f29c..7dc12a2 100644 --- a/internals/proxy/middlewares/common.go +++ b/internals/proxy/middlewares/common.go @@ -23,14 +23,18 @@ type contextKey string const tokenKey contextKey = "token" -func getSettings(req *http.Request) config.SETTING_ { - token := req.Context().Value(tokenKey).(string) +func getSettings(req *http.Request) *config.SETTING_ { + token, ok := req.Context().Value(tokenKey).(string) - settings := config.ENV.SETTINGS[token] + if !ok { + token = "*" + } - if settings == nil { - settings = config.ENV.SETTINGS["*"] - } + settings, exists := config.ENV.SETTINGS[token] - return *settings + if !exists || settings == nil { + settings = config.ENV.SETTINGS["*"] + } + + return settings } \ No newline at end of file diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 074a24f..4a5f373 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -24,7 +24,7 @@ func LoadTokens() { func InitTokens() { apiTokens := config.Strings("api.tokens") - var tokenConfigs []TOKEN_CONFIG_ + tokenConfigs := []TOKEN_CONFIG_{} tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) From 291d3902e35c3236b001c46ae777747e933e3e37 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:33:13 +0200 Subject: [PATCH 56/97] fix --- internals/proxy/middlewares/body.go | 6 +++++- internals/proxy/middlewares/common.go | 6 +++++- internals/proxy/middlewares/endpoints.go | 6 +++++- internals/proxy/middlewares/template.go | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/internals/proxy/middlewares/body.go b/internals/proxy/middlewares/body.go index 518bff6..67619d8 100644 --- a/internals/proxy/middlewares/body.go +++ b/internals/proxy/middlewares/body.go @@ -20,7 +20,11 @@ func (data BodyMiddleware) Use() http.Handler { next := data.Next return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - messageAliases := getSettings(req).MESSAGE_ALIASES + messageAliases := getSettingsByReq(req).MESSAGE_ALIASES + + if messageAliases == nil { + messageAliases = getSettings("*").MESSAGE_ALIASES + } body, err := request.GetReqBody(w, req) diff --git a/internals/proxy/middlewares/common.go b/internals/proxy/middlewares/common.go index 7dc12a2..1c04a2f 100644 --- a/internals/proxy/middlewares/common.go +++ b/internals/proxy/middlewares/common.go @@ -23,13 +23,17 @@ type contextKey string const tokenKey contextKey = "token" -func getSettings(req *http.Request) *config.SETTING_ { +func getSettingsByReq(req *http.Request) *config.SETTING_ { token, ok := req.Context().Value(tokenKey).(string) if !ok { token = "*" } + return getSettings(token) +} + +func getSettings(token string) *config.SETTING_ { settings, exists := config.ENV.SETTINGS[token] if !exists || settings == nil { diff --git a/internals/proxy/middlewares/endpoints.go b/internals/proxy/middlewares/endpoints.go index fb0d400..173fff2 100644 --- a/internals/proxy/middlewares/endpoints.go +++ b/internals/proxy/middlewares/endpoints.go @@ -15,7 +15,11 @@ func (data EndpointsMiddleware) Use() http.Handler { next := data.Next return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - blockedEndpoints := getSettings(req).BLOCKED_ENDPOINTS + blockedEndpoints := getSettingsByReq(req).BLOCKED_ENDPOINTS + + if blockedEndpoints == nil { + blockedEndpoints = getSettings("*").BLOCKED_ENDPOINTS + } reqPath := req.URL.Path diff --git a/internals/proxy/middlewares/template.go b/internals/proxy/middlewares/template.go index f8a4b0e..83d5a3a 100644 --- a/internals/proxy/middlewares/template.go +++ b/internals/proxy/middlewares/template.go @@ -22,7 +22,11 @@ func (data TemplateMiddleware) Use() http.Handler { next := data.Next return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - variables := getSettings(req).VARIABLES + variables := getSettingsByReq(req).VARIABLES + + if variables == nil { + variables = getSettings("*").VARIABLES + } body, err := request.GetReqBody(w, req) From 0ee32909abc5ee1b370071ddf64e8e6f28ab5b95 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:42:23 +0200 Subject: [PATCH 57/97] debugging json templating --- utils/templating/templating.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/templating/templating.go b/utils/templating/templating.go index 69dc621..f630ffa 100644 --- a/utils/templating/templating.go +++ b/utils/templating/templating.go @@ -7,6 +7,8 @@ import ( "regexp" "strings" "text/template" + + "github.com/codeshelldev/secured-signal-api/utils/logger" ) func normalize(value any) string { @@ -67,6 +69,7 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st jsonBytes, err := json.Marshal(data) if err != nil { + logger.Dev("72"+err.Error()) return nil, err } @@ -86,6 +89,7 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st jsonStr, err := ParseTemplate(templt, tmplStr, variables) if err != nil { + logger.Dev("92:"+err.Error()) return nil, err } From d4f46c716fff19f13e74d2421992faa39afe4879 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:45:55 +0200 Subject: [PATCH 58/97] again --- utils/templating/templating.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/templating/templating.go b/utils/templating/templating.go index f630ffa..6a44154 100644 --- a/utils/templating/templating.go +++ b/utils/templating/templating.go @@ -97,6 +97,7 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st re, err = regexp.Compile(`"<<(.*?)>>"`) if err != nil { + logger.Dev("100:"+err.Error()) return nil, err } @@ -105,6 +106,7 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st err = json.Unmarshal([]byte(jsonStr), &data) if err != nil { + logger.Dev("109:"+err.Error()) return nil, err } From 46f1e845e7de14ad53a8d1e799af23bc0c601eff Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:57:00 +0200 Subject: [PATCH 59/97] fix? --- tests/json_test.go | 18 +++++++++--------- utils/templating/templating.go | 13 +++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/json_test.go b/tests/json_test.go index 4541b41..1e92182 100644 --- a/tests/json_test.go +++ b/tests/json_test.go @@ -8,7 +8,7 @@ import ( ) func TestJsonTemplating(t *testing.T) { - variables := map[string]interface{}{ + variables := map[string]any{ "array": []string{ "item0", "item1", @@ -28,17 +28,17 @@ func TestJsonTemplating(t *testing.T) { "key2": "{{.int}}" }` - data := utils.GetJson[map[string]interface{}](json) + data := utils.GetJson[map[string]any](json) - expected := map[string]interface{}{ - "dict": map[string]interface{}{ + expected := map[string]any{ + "dict": map[string]any{ "key": "val", }, - "dictArray": []interface{}{ - map[string]interface{}{"key": "val"}, - map[string]interface{}{"key": []interface{}{ "item0", "item1" }}, + "dictArray": []any{ + map[string]any{"key": "val"}, + map[string]any{"key": []any{ "item0", "item1" }}, }, - "key1": []interface{}{ "item0", "item1" }, + "key1": []any{ "item0", "item1" }, "key2": 4, } @@ -71,7 +71,7 @@ func TestJsonPath(t *testing.T) { "key": "val" }` - data := utils.GetJson[map[string]interface{}](json) + data := utils.GetJson[map[string]any](json) cases := []struct{ key string diff --git a/utils/templating/templating.go b/utils/templating/templating.go index 6a44154..a7f3da9 100644 --- a/utils/templating/templating.go +++ b/utils/templating/templating.go @@ -30,15 +30,16 @@ func normalize(value any) string { } func normalizeJSON(value any) string { - jsonBytes, err := json.Marshal(value) + switch value.(type) { + case []any, []string, map[string]any, int, float64, bool: + object, _ := json.Marshal(value) - if err != nil { - return "INVALID:JSON" - } + return "<<" + string(object) + ">>" - return "<<" + string(jsonBytes) + ">>" + default: + return value.(string) + } } - func ParseTemplate(templt *template.Template, tmplStr string, variables any) (string, error) { tmpl, err := templt.Parse(tmplStr) From 2a2907b4ccc42cf24221ebb298afb87f7217c6b0 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:00:04 +0200 Subject: [PATCH 60/97] =?UTF-8?q?debugging=20=F0=9F=91=8E=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internals/proxy/middlewares/template.go | 2 ++ utils/templating/templating.go | 1 + 2 files changed, 3 insertions(+) diff --git a/internals/proxy/middlewares/template.go b/internals/proxy/middlewares/template.go index 83d5a3a..d8ee4fb 100644 --- a/internals/proxy/middlewares/template.go +++ b/internals/proxy/middlewares/template.go @@ -107,6 +107,8 @@ func (data TemplateMiddleware) Use() http.Handler { func TemplateBody(data map[string]any, VARIABLES any) (map[string]any, bool, error) { var modified bool + log.Dev(utils.ToJson(data)) + templatedData, err := templating.RenderJSONTemplate("body", data, VARIABLES) if err != nil { diff --git a/utils/templating/templating.go b/utils/templating/templating.go index a7f3da9..f8e0478 100644 --- a/utils/templating/templating.go +++ b/utils/templating/templating.go @@ -40,6 +40,7 @@ func normalizeJSON(value any) string { return value.(string) } } + func ParseTemplate(templt *template.Template, tmplStr string, variables any) (string, error) { tmpl, err := templt.Parse(tmplStr) From d03040a2356b93ae45cf3111d39b61cfd025de35 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:08:25 +0200 Subject: [PATCH 61/97] debugggggggging --- utils/request/request.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/utils/request/request.go b/utils/request/request.go index 7bcacbf..69ded2e 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" + "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/query" ) @@ -98,6 +99,8 @@ func GetBody(req *http.Request) ([]byte, error) { func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { bytes, err := GetBody(req) + logger.Dev(string(bytes)) + var isEmpty bool if err != nil { @@ -113,22 +116,22 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { var data map[string]any switch GetBodyType(req) { - case Json: - data, err = GetJsonData(bytes) + case Json: + data, err = GetJsonData(bytes) - if err != nil { - http.Error(w, "Bad Request: invalid JSON", http.StatusBadRequest) + if err != nil { + http.Error(w, "Bad Request: invalid JSON", http.StatusBadRequest) - return Body{Empty: true}, err - } - case Form: - data, err = GetFormData(bytes) + return Body{Empty: true}, err + } + case Form: + data, err = GetFormData(bytes) - if err != nil { - http.Error(w, "Bad Request: invalid Form", http.StatusBadRequest) + if err != nil { + http.Error(w, "Bad Request: invalid Form", http.StatusBadRequest) - return Body{Empty: true}, err - } + return Body{Empty: true}, err + } } isEmpty = len(data) <= 0 From 6bee92ae33516f57894bc7a7f940b998373c85c2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:13:03 +0200 Subject: [PATCH 62/97] DeBuGgInG --- utils/request/request.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/request/request.go b/utils/request/request.go index 69ded2e..0f59ee1 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" + "github.com/codeshelldev/secured-signal-api/utils" "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/query" ) @@ -99,8 +100,6 @@ func GetBody(req *http.Request) ([]byte, error) { func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { bytes, err := GetBody(req) - logger.Dev(string(bytes)) - var isEmpty bool if err != nil { @@ -134,6 +133,8 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { } } + logger.Dev(utils.ToJson(data)) + isEmpty = len(data) <= 0 return Body{ @@ -156,6 +157,6 @@ func GetBodyType(req *http.Request) BodyType { case strings.HasPrefix(contentType, "application/x-www-form-urlencoded"): return Form default: - return Unknown + return Json } } \ No newline at end of file From ba70a35c15a60f7a91cd8b6664fc267fed49cf8c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:17:30 +0200 Subject: [PATCH 63/97] debugging... --- utils/request/request.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/request/request.go b/utils/request/request.go index 0f59ee1..b267dbe 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -7,7 +7,6 @@ import ( "net/http" "strings" - "github.com/codeshelldev/secured-signal-api/utils" "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/query" ) @@ -54,6 +53,8 @@ func CreateBody(data map[string]any) (Body, error) { } func GetJsonData(body []byte) (map[string]any, error) { + logger.Dev(string(body)) + var data map[string]any err := json.Unmarshal(body, &data) @@ -133,8 +134,6 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { } } - logger.Dev(utils.ToJson(data)) - isEmpty = len(data) <= 0 return Body{ From 6a6655be8e08d41e664c36e77052fa0ed79e4944 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:29:25 +0200 Subject: [PATCH 64/97] more debugging --- utils/request/request.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/request/request.go b/utils/request/request.go index b267dbe..6998e61 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" + "github.com/codeshelldev/secured-signal-api/utils" "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/query" ) @@ -53,8 +54,6 @@ func CreateBody(data map[string]any) (Body, error) { } func GetJsonData(body []byte) (map[string]any, error) { - logger.Dev(string(body)) - var data map[string]any err := json.Unmarshal(body, &data) @@ -136,6 +135,8 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { isEmpty = len(data) <= 0 + logger.Dev(utils.ToJson(data)) + return Body{ Raw: bytes, Data: data, @@ -146,6 +147,8 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { func GetBodyType(req *http.Request) BodyType { contentType := req.Header.Get("Content-Type") + logger.Dev(contentType) + switch { case strings.HasPrefix(contentType, "application/json"): return Json From f9cee1d4d0b294f2cd419287f68954d38d096d23 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 18:35:59 +0200 Subject: [PATCH 65/97] ... Who knew that `curl` wouldn't be able to identify / add Content-Type automatically... Debugging was for nothing... --- utils/request/request.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/utils/request/request.go b/utils/request/request.go index 6998e61..cf26ab5 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -7,8 +7,6 @@ import ( "net/http" "strings" - "github.com/codeshelldev/secured-signal-api/utils" - "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/query" ) @@ -135,8 +133,6 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { isEmpty = len(data) <= 0 - logger.Dev(utils.ToJson(data)) - return Body{ Raw: bytes, Data: data, @@ -147,8 +143,6 @@ func GetReqBody(w http.ResponseWriter, req *http.Request) (Body, error) { func GetBodyType(req *http.Request) BodyType { contentType := req.Header.Get("Content-Type") - logger.Dev(contentType) - switch { case strings.HasPrefix(contentType, "application/json"): return Json @@ -159,6 +153,6 @@ func GetBodyType(req *http.Request) BodyType { case strings.HasPrefix(contentType, "application/x-www-form-urlencoded"): return Form default: - return Json + return Unknown } } \ No newline at end of file From cb34e710e33c63674e46f743cfedf02312047328 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 19:05:25 +0200 Subject: [PATCH 66/97] test --- internals/proxy/middlewares/template.go | 2 -- utils/config/config.go | 11 ++++------ utils/config/token-config.go | 28 ++++++++++++------------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/internals/proxy/middlewares/template.go b/internals/proxy/middlewares/template.go index d8ee4fb..83d5a3a 100644 --- a/internals/proxy/middlewares/template.go +++ b/internals/proxy/middlewares/template.go @@ -107,8 +107,6 @@ func (data TemplateMiddleware) Use() http.Handler { func TemplateBody(data map[string]any, VARIABLES any) (map[string]any, bool, error) { var modified bool - log.Dev(utils.ToJson(data)) - templatedData, err := templating.RenderJSONTemplate("body", data, VARIABLES) if err != nil { diff --git a/utils/config/config.go b/utils/config/config.go index 9383a89..5c888ed 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -17,7 +17,6 @@ import ( var defaultsLayer = koanf.New(".") var userLayer = koanf.New(".") -var tokensLayer = koanf.New(".") var config *koanf.Koanf @@ -48,11 +47,11 @@ func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Prov return f, err } -func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) error { +func LoadDir(dir string, parser koanf.Parser) []map[string]any { files, err := filepath.Glob(filepath.Join(dir, "*.yml")) if err != nil { - return err + return nil } data := []map[string]any{} @@ -63,15 +62,13 @@ func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) _, err := LoadFile(file, tmp, parser) if err != nil { - return err + return nil } data = append(data, tmp.All()) } - config.Set(path, data) - - return nil + return data } func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 4a5f373..8fa3472 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -3,34 +3,28 @@ package config import ( "strconv" + middlewareTypes "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares/types" "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) -type TOKEN_CONFIG_ struct { - TOKENS []string `koanf:"tokens"` - OVERRIDES SETTING_ `koanf:"overrides"` -} +var tokens []map[string]any func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) - LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) + LoadDir(ENV.TOKENS_DIR, yaml.Parser()) - log.Dev(utils.ToJson(tokensLayer.All())) + log.Dev(utils.ToJson(tokens)) } func InitTokens() { apiTokens := config.Strings("api.tokens") - tokenConfigs := []TOKEN_CONFIG_{} - - tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) - - log.Dev(utils.ToJson(tokenConfigs)) + log.Dev(utils.ToJson(tokens)) - overrides := ParseTokenConfigs(tokenConfigs) + overrides := ParseTokenConfigs(tokens) for token, override := range overrides { apiTokens = append(apiTokens, token) @@ -57,12 +51,16 @@ func InitTokens() { } } -func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]SETTING_) { +func ParseTokenConfigs(configs []map[string]any) (map[string]SETTING_) { settings := map[string]SETTING_{} for _, config := range configs { - for _, token := range config.TOKENS { - settings[token] = config.OVERRIDES + for _, token := range config["tokens"].([]string) { + settings[token] = SETTING_{ + BLOCKED_ENDPOINTS: config["override.blockedendpoints"].([]string), + VARIABLES: config["overrides.variables"].(map[string]any), + MESSAGE_ALIASES: config["overrides.messagealiases"].([]middlewareTypes.MessageAlias), + } } } From 2d1b245c52bad8c87375db355604d66c7ba1ce2f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 19:08:52 +0200 Subject: [PATCH 67/97] =?UTF-8?q?=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/config/token-config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 8fa3472..bcd822b 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -14,7 +14,7 @@ var tokens []map[string]any func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) - LoadDir(ENV.TOKENS_DIR, yaml.Parser()) + tokens = LoadDir(ENV.TOKENS_DIR, yaml.Parser()) log.Dev(utils.ToJson(tokens)) } From c1ec5a6bfaa73e7523826851088b9c478154f8bd Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 19:51:48 +0200 Subject: [PATCH 68/97] testing changes --- utils/config/config.go | 30 ++++++++++++++++++------------ utils/config/token-config.go | 28 +++++++++++++++------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 5c888ed..f366551 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strconv" "strings" "sync" @@ -17,6 +18,7 @@ import ( var defaultsLayer = koanf.New(".") var userLayer = koanf.New(".") +var tokensLayer = koanf.New(".") var config *koanf.Koanf @@ -31,6 +33,12 @@ func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Prov return nil, err } + WatchFile(path, f) + + return f, err +} + +func WatchFile(path string, f *file.File) { f.Watch(func(event any, err error) { if err != nil { return @@ -43,32 +51,30 @@ func LoadFile(path string, config *koanf.Koanf, parser koanf.Parser) (koanf.Prov Load() }) - - return f, err } -func LoadDir(dir string, parser koanf.Parser) []map[string]any { +func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) []map[string]any { files, err := filepath.Glob(filepath.Join(dir, "*.yml")) if err != nil { return nil } - data := []map[string]any{} - - for _, file := range files { + for i, f := range files { tmp := koanf.New(".") - _, err := LoadFile(file, tmp, parser) + LoadFile(f, config, parser) - if err != nil { - return nil + wrapper := map[string]any{ + path: map[string]any{ + strconv.Itoa(i): tmp.Raw(), + }, } - data = append(data, tmp.All()) - } + config.Load(confmap.Provider(wrapper, ""), nil); + } - return data + return nil } func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { diff --git a/utils/config/token-config.go b/utils/config/token-config.go index bcd822b..074a24f 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -3,28 +3,34 @@ package config import ( "strconv" - middlewareTypes "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares/types" "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) -var tokens []map[string]any +type TOKEN_CONFIG_ struct { + TOKENS []string `koanf:"tokens"` + OVERRIDES SETTING_ `koanf:"overrides"` +} func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) - tokens = LoadDir(ENV.TOKENS_DIR, yaml.Parser()) + LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) - log.Dev(utils.ToJson(tokens)) + log.Dev(utils.ToJson(tokensLayer.All())) } func InitTokens() { apiTokens := config.Strings("api.tokens") - log.Dev(utils.ToJson(tokens)) + var tokenConfigs []TOKEN_CONFIG_ + + tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) + + log.Dev(utils.ToJson(tokenConfigs)) - overrides := ParseTokenConfigs(tokens) + overrides := ParseTokenConfigs(tokenConfigs) for token, override := range overrides { apiTokens = append(apiTokens, token) @@ -51,16 +57,12 @@ func InitTokens() { } } -func ParseTokenConfigs(configs []map[string]any) (map[string]SETTING_) { +func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]SETTING_) { settings := map[string]SETTING_{} for _, config := range configs { - for _, token := range config["tokens"].([]string) { - settings[token] = SETTING_{ - BLOCKED_ENDPOINTS: config["override.blockedendpoints"].([]string), - VARIABLES: config["overrides.variables"].(map[string]any), - MESSAGE_ALIASES: config["overrides.messagealiases"].([]middlewareTypes.MessageAlias), - } + for _, token := range config.TOKENS { + settings[token] = config.OVERRIDES } } From 534204adb629983d6bcf2da913964fc057ecb737 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:01:05 +0200 Subject: [PATCH 69/97] fix --- utils/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index f366551..0de2543 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -63,7 +63,7 @@ func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) for i, f := range files { tmp := koanf.New(".") - LoadFile(f, config, parser) + LoadFile(f, tmp, parser) wrapper := map[string]any{ path: map[string]any{ From 0c71ceddcc7b7d097434d05e35126990316030f7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:16:30 +0200 Subject: [PATCH 70/97] testing --- utils/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index 0de2543..37a7976 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -71,7 +71,7 @@ func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) }, } - config.Load(confmap.Provider(wrapper, ""), nil); + config.Load(confmap.Provider(wrapper, "."), nil); } return nil From a87cd1749d384b24fac33e9dffa003ef01f11750 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:25:24 +0200 Subject: [PATCH 71/97] debug --- utils/config/token-config.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 074a24f..19a93db 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -24,19 +24,17 @@ func LoadTokens() { func InitTokens() { apiTokens := config.Strings("api.tokens") - var tokenConfigs []TOKEN_CONFIG_ + tokensConfig := tokensLayer.Get("tokenConfigs") - tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) + log.Dev(utils.ToJson(tokensConfig)) - log.Dev(utils.ToJson(tokenConfigs)) - - overrides := ParseTokenConfigs(tokenConfigs) + /*overrides := ParseTokenConfigs(tokenConfigs) for token, override := range overrides { apiTokens = append(apiTokens, token) ENV.SETTINGS[token] = &override - } + }*/ if len(apiTokens) <= 0 { log.Warn("No API TOKEN provided this is NOT recommended") From 7d9fd2dee011cf5885bcd66989f6c68c79ed7a85 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:29:36 +0200 Subject: [PATCH 72/97] still brute-force searching for solution --- utils/config/config.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 37a7976..e350ec9 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,7 +3,6 @@ package config import ( "os" "path/filepath" - "strconv" "strings" "sync" @@ -53,28 +52,32 @@ func WatchFile(path string, f *file.File) { }) } -func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) []map[string]any { +func LoadDir(path string, dir string, config *koanf.Koanf, parser koanf.Parser) error { files, err := filepath.Glob(filepath.Join(dir, "*.yml")) if err != nil { return nil } - for i, f := range files { + var array []any + + for _, f := range files { tmp := koanf.New(".") - LoadFile(f, tmp, parser) + _, err := LoadFile(f, tmp, parser) - wrapper := map[string]any{ - path: map[string]any{ - strconv.Itoa(i): tmp.Raw(), - }, + if err != nil { + return err } - config.Load(confmap.Provider(wrapper, "."), nil); + array = append(array, tmp.Raw()) + } + + wrapper := map[string]any{ + path: array, } - return nil + return config.Load(confmap.Provider(wrapper, "."), nil) } func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { From ebfd40583f7c97fde744d1dbeb50d4e04f6b7767 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:31:38 +0200 Subject: [PATCH 73/97] finally fixed?! --- utils/config/token-config.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 19a93db..074a24f 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -24,17 +24,19 @@ func LoadTokens() { func InitTokens() { apiTokens := config.Strings("api.tokens") - tokensConfig := tokensLayer.Get("tokenConfigs") + var tokenConfigs []TOKEN_CONFIG_ - log.Dev(utils.ToJson(tokensConfig)) + tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) - /*overrides := ParseTokenConfigs(tokenConfigs) + log.Dev(utils.ToJson(tokenConfigs)) + + overrides := ParseTokenConfigs(tokenConfigs) for token, override := range overrides { apiTokens = append(apiTokens, token) ENV.SETTINGS[token] = &override - }*/ + } if len(apiTokens) <= 0 { log.Warn("No API TOKEN provided this is NOT recommended") From c9fc738f7a6d1b2a079f7eb05df19ce1e1f560d8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 20:40:07 +0200 Subject: [PATCH 74/97] nil-check at value normilization --- utils/templating/templating.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/utils/templating/templating.go b/utils/templating/templating.go index f8e0478..37d4ec3 100644 --- a/utils/templating/templating.go +++ b/utils/templating/templating.go @@ -7,8 +7,6 @@ import ( "regexp" "strings" "text/template" - - "github.com/codeshelldev/secured-signal-api/utils/logger" ) func normalize(value any) string { @@ -30,7 +28,11 @@ func normalize(value any) string { } func normalizeJSON(value any) string { - switch value.(type) { + if value == nil { + return "" + } + + switch value.(type) { case []any, []string, map[string]any, int, float64, bool: object, _ := json.Marshal(value) @@ -71,7 +73,6 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st jsonBytes, err := json.Marshal(data) if err != nil { - logger.Dev("72"+err.Error()) return nil, err } @@ -91,7 +92,6 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st jsonStr, err := ParseTemplate(templt, tmplStr, variables) if err != nil { - logger.Dev("92:"+err.Error()) return nil, err } @@ -99,7 +99,6 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st re, err = regexp.Compile(`"<<(.*?)>>"`) if err != nil { - logger.Dev("100:"+err.Error()) return nil, err } @@ -108,7 +107,6 @@ func RenderJSONTemplate(name string, data map[string]any, variables any) (map[st err = json.Unmarshal([]byte(jsonStr), &data) if err != nil { - logger.Dev("109:"+err.Error()) return nil, err } From ac03e3030044d901bcda9f81e420f9340df48bfe Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:07:39 +0200 Subject: [PATCH 75/97] normalize VARIABLES --- utils/config/config.go | 36 ++++++++++++++++++++++++++++-------- utils/config/loader.go | 4 ++-- utils/config/token-config.go | 14 +++++++------- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index e350ec9..08b654e 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strconv" "strings" "sync" @@ -94,10 +95,6 @@ func LoadEnv(config *koanf.Koanf) (koanf.Provider, error) { return e, err } -func mergeConfig(path string, mergeInto *koanf.Koanf, mergeFrom *koanf.Koanf) { - mergeInto.MergeAt(mergeFrom, path) -} - func templateConfig(config *koanf.Koanf) { data := config.All() @@ -138,9 +135,10 @@ func normalizeKeys(config *koanf.Koanf) { config.Load(confmap.Provider(data, "."), nil) } -func transformChildren(config *koanf.Koanf, prefix string, transform func(key string, value any) (string, any)) error { +// Transforms Children of path +func transformChildren(config *koanf.Koanf, path string, transform func(key string, value any) (string, any)) error { var sub map[string]any - if err := config.Unmarshal(prefix, &sub); err != nil { + if err := config.Unmarshal(path, &sub); err != nil { return err } @@ -152,16 +150,38 @@ func transformChildren(config *koanf.Koanf, prefix string, transform func(key st } config.Load(confmap.Provider(map[string]any{ - prefix: map[string]any{}, + path: map[string]any{}, }, "."), nil) config.Load(confmap.Provider(map[string]any{ - prefix: transformed, + path: transformed, }, "."), nil) return nil } +// Does the same thing as transformChildren() but does it for each Array Item inside of root and transforms subPath +func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath string, transform func(key string, value any) (string, any)) error { + var tokens []map[string]any + + err := config.Unmarshal(root, &tokens) + + if err != nil { + return err + } + + for i := range tokens { + p := root + "." + strconv.Itoa(i) + "." + subPath + + if err := transformChildren(config, p, transform); err != nil { + return err + } + } + + return nil +} + + func normalizeEnv(key string, value string) (string, any) { key = strings.ToLower(key) key = strings.ReplaceAll(key, "__", ".") diff --git a/utils/config/loader.go b/utils/config/loader.go index 740fdec..dc4e118 100644 --- a/utils/config/loader.go +++ b/utils/config/loader.go @@ -53,8 +53,6 @@ func InitEnv() { ENV.API_URL = config.String("api.url") - InitTokens() - defaultSettings := ENV.SETTINGS["*"] config.Unmarshal("messagealiases", &defaultSettings.MESSAGE_ALIASES) @@ -81,6 +79,8 @@ func Load() { config = mergeLayers() + InitTokens() + normalizeKeys(config) templateConfig(config) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 074a24f..a0e383c 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -2,8 +2,8 @@ package config import ( "strconv" + "strings" - "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -17,8 +17,6 @@ func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) - - log.Dev(utils.ToJson(tokensLayer.All())) } func InitTokens() { @@ -26,11 +24,13 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) + transformChildrenUnderArray(config, "tokenConfigs", "override.variables", func(key string, value any) (string, any) { + return strings.ToUpper(key), value + }) - log.Dev(utils.ToJson(tokenConfigs)) + tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) - overrides := ParseTokenConfigs(tokenConfigs) + overrides := parseTokenConfigs(tokenConfigs) for token, override := range overrides { apiTokens = append(apiTokens, token) @@ -57,7 +57,7 @@ func InitTokens() { } } -func ParseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]SETTING_) { +func parseTokenConfigs(configs []TOKEN_CONFIG_) (map[string]SETTING_) { settings := map[string]SETTING_{} for _, config := range configs { From 77a3d579f16ffbeca03932b120d3c4c5e9cd5e74 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:12:17 +0200 Subject: [PATCH 76/97] fixing normilization --- utils/config/loader.go | 4 ++-- utils/config/token-config.go | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/utils/config/loader.go b/utils/config/loader.go index dc4e118..6f0a4e9 100644 --- a/utils/config/loader.go +++ b/utils/config/loader.go @@ -79,12 +79,12 @@ func Load() { config = mergeLayers() - InitTokens() - normalizeKeys(config) templateConfig(config) + InitTokens() + InitEnv() log.Info("Finished Loading Configuration") diff --git a/utils/config/token-config.go b/utils/config/token-config.go index a0e383c..85c4e1d 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -17,6 +17,10 @@ func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) + + normalizeKeys(tokensLayer) + + templateConfig(tokensLayer) } func InitTokens() { @@ -24,7 +28,7 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - transformChildrenUnderArray(config, "tokenConfigs", "override.variables", func(key string, value any) (string, any) { + transformChildrenUnderArray(tokensLayer, "tokenConfigs", "override.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) From 1a9b11cc89627224f40ca0cfe4783feb9a9cca5c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:15:48 +0200 Subject: [PATCH 77/97] further testing --- utils/config/token-config.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 85c4e1d..6f6a3b2 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -4,6 +4,7 @@ import ( "strconv" "strings" + "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -28,12 +29,16 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - transformChildrenUnderArray(tokensLayer, "tokenConfigs", "override.variables", func(key string, value any) (string, any) { + transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) + log.Dev(utils.ToJson(tokensLayer.All())) + tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) + log.Dev(utils.ToJson(tokenConfigs)) + overrides := parseTokenConfigs(tokenConfigs) for token, override := range overrides { From aaa48f8526f859afe575d28175366d3e6568b56c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:21:36 +0200 Subject: [PATCH 78/97] even more debugging --- utils/config/config.go | 2 ++ utils/config/token-config.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/utils/config/config.go b/utils/config/config.go index 08b654e..aaf1f12 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -173,6 +173,8 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin for i := range tokens { p := root + "." + strconv.Itoa(i) + "." + subPath + log.Dev(p) + if err := transformChildren(config, p, transform); err != nil { return err } diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 6f6a3b2..608be9f 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -19,9 +19,11 @@ func LoadTokens() { LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) + /* normalizeKeys(tokensLayer) templateConfig(tokensLayer) + */ } func InitTokens() { From 17407d360aaca9208f71a4f7e1ed3a085f90c092 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:27:29 +0200 Subject: [PATCH 79/97] of course: debugging --- utils/config/config.go | 10 ++++++---- utils/config/token-config.go | 6 +++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index aaf1f12..e78b2b7 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -162,20 +162,22 @@ func transformChildren(config *koanf.Koanf, path string, transform func(key stri // Does the same thing as transformChildren() but does it for each Array Item inside of root and transforms subPath func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath string, transform func(key string, value any) (string, any)) error { - var tokens []map[string]any + var items []map[string]any - err := config.Unmarshal(root, &tokens) + err := config.Unmarshal(root, &items) if err != nil { return err } - for i := range tokens { + for i := range items { p := root + "." + strconv.Itoa(i) + "." + subPath log.Dev(p) - if err := transformChildren(config, p, transform); err != nil { + err := transformChildren(config, p, transform) + + if err != nil { return err } } diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 608be9f..f8b54f1 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -31,10 +31,14 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { + log.Dev(utils.ToJson(tokensLayer.All())) + + err := transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) + log.Error(err.Error()) + log.Dev(utils.ToJson(tokensLayer.All())) tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) From 06ed1ec0cf33711eeea89446216b81f95dda9d71 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:29:07 +0200 Subject: [PATCH 80/97] stop exiting --- utils/config/token-config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index f8b54f1..1636f8b 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -37,7 +37,7 @@ func InitTokens() { return strings.ToUpper(key), value }) - log.Error(err.Error()) + log.Dev(err.Error()) log.Dev(utils.ToJson(tokensLayer.All())) From cada417a8a7c4f101d8b81567c7c1b713da327cc Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:39:02 +0200 Subject: [PATCH 81/97] revert err printing --- utils/config/token-config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/config/token-config.go b/utils/config/token-config.go index 1636f8b..c8434a6 100644 --- a/utils/config/token-config.go +++ b/utils/config/token-config.go @@ -33,12 +33,10 @@ func InitTokens() { log.Dev(utils.ToJson(tokensLayer.All())) - err := transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { + transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) - log.Dev(err.Error()) - log.Dev(utils.ToJson(tokensLayer.All())) tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) From cf6b036a227818dff844e386fce52a86097a56f6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:45:19 +0200 Subject: [PATCH 82/97] update transform logic --- utils/config/config.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index e78b2b7..4b9b3c6 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,6 +1,7 @@ package config import ( + "errors" "os" "path/filepath" "strconv" @@ -138,28 +139,41 @@ func normalizeKeys(config *koanf.Koanf) { // Transforms Children of path func transformChildren(config *koanf.Koanf, path string, transform func(key string, value any) (string, any)) error { var sub map[string]any + if err := config.Unmarshal(path, &sub); err != nil { return err } transformed := make(map[string]any) + for key, val := range sub { newKey, newVal := transform(key, val) transformed[newKey] = newVal } - - config.Load(confmap.Provider(map[string]any{ - path: map[string]any{}, - }, "."), nil) - config.Load(confmap.Provider(map[string]any{ - path: transformed, - }, "."), nil) + parts := strings.Split(path, ".") + if len(parts) < 1 { + return errors.New("invalid path: " + path) + } - return nil + parentPath := strings.Join(parts[:len(parts)-1], ".") + lastKey := parts[len(parts)-1] + + var parent map[string]any + + if err := config.Unmarshal(parentPath, &parent); err != nil { + return err + } + + parent[lastKey] = transformed + + return config.Load(confmap.Provider(map[string]any{ + parentPath: parent, + }, "."), nil) } + // Does the same thing as transformChildren() but does it for each Array Item inside of root and transforms subPath func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath string, transform func(key string, value any) (string, any)) error { var items []map[string]any From e5bbd778fe4ba473f72e771f1aec8dc405478b24 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:54:52 +0200 Subject: [PATCH 83/97] fix? --- utils/config/config.go | 56 +++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 4b9b3c6..ab757c8 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,10 +1,8 @@ package config import ( - "errors" "os" "path/filepath" - "strconv" "strings" "sync" @@ -139,63 +137,53 @@ func normalizeKeys(config *koanf.Koanf) { // Transforms Children of path func transformChildren(config *koanf.Koanf, path string, transform func(key string, value any) (string, any)) error { var sub map[string]any - if err := config.Unmarshal(path, &sub); err != nil { return err } transformed := make(map[string]any) - for key, val := range sub { newKey, newVal := transform(key, val) transformed[newKey] = newVal } + + config.Load(confmap.Provider(map[string]any{ + path: map[string]any{}, + }, "."), nil) - parts := strings.Split(path, ".") - if len(parts) < 1 { - return errors.New("invalid path: " + path) - } - - parentPath := strings.Join(parts[:len(parts)-1], ".") - lastKey := parts[len(parts)-1] - - var parent map[string]any - - if err := config.Unmarshal(parentPath, &parent); err != nil { - return err - } - - parent[lastKey] = transformed - - return config.Load(confmap.Provider(map[string]any{ - parentPath: parent, + config.Load(confmap.Provider(map[string]any{ + path: transformed, }, "."), nil) -} + return nil +} // Does the same thing as transformChildren() but does it for each Array Item inside of root and transforms subPath func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath string, transform func(key string, value any) (string, any)) error { - var items []map[string]any - - err := config.Unmarshal(root, &items) - - if err != nil { + var array []map[string]any + if err := config.Unmarshal(root, &array); err != nil { return err } - for i := range items { - p := root + "." + strconv.Itoa(i) + "." + subPath + for i := range array { + tmp := koanf.New(".") - log.Dev(p) + tmp.Load(confmap.Provider(map[string]any{ + "item": array[i], + }, "."), nil) - err := transformChildren(config, p, transform) - - if err != nil { + if err := transformChildren(tmp, "item." + subPath, transform); err != nil { return err } + + array[i] = tmp.All()["item"].(map[string]any) } + config.Load(confmap.Provider(map[string]any{ + root: array, + }, "."), nil) + return nil } From 830adb1222af5327f9cc013e3b879cb912b93174 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 22:01:17 +0200 Subject: [PATCH 84/97] check if path exists before continuing --- utils/config/config.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index ab757c8..ed4b99b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -173,11 +173,29 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin "item": array[i], }, "."), nil) - if err := transformChildren(tmp, "item." + subPath, transform); err != nil { - return err + path := "item." + subPath + + exists := tmp.Exists(path) + + if !exists { + tmp.Load(confmap.Provider(map[string]any{ + "item": map[string]any{ + subPath: map[string]any{}, + }, + }, "."), nil) } - array[i] = tmp.All()["item"].(map[string]any) + transformChildren(tmp, path, transform) + + item := tmp.All()["item"] + + itemMap, ok := item.(map[string]any) + + if ok { + array[i] = itemMap + } else { + array[i] = map[string]any{} + } } config.Load(confmap.Provider(map[string]any{ From 443a10d0aa93e25490350527ccc295baa635fde3 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 13 Sep 2025 22:06:51 +0200 Subject: [PATCH 85/97] simplified version --- utils/config/config.go | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index ed4b99b..a1789af 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strconv" "strings" "sync" @@ -167,41 +168,13 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin } for i := range array { - tmp := koanf.New(".") - - tmp.Load(confmap.Provider(map[string]any{ - "item": array[i], - }, "."), nil) - - path := "item." + subPath - - exists := tmp.Exists(path) - - if !exists { - tmp.Load(confmap.Provider(map[string]any{ - "item": map[string]any{ - subPath: map[string]any{}, - }, - }, "."), nil) - } + path := root + "." + strconv.Itoa(i) + "." + subPath - transformChildren(tmp, path, transform) - - item := tmp.All()["item"] - - itemMap, ok := item.(map[string]any) - - if ok { - array[i] = itemMap - } else { - array[i] = map[string]any{} + if config.Exists(path) { + transformChildren(config, path, transform) } } - config.Load(confmap.Provider(map[string]any{ - root: array, - }, "."), nil) - return nil } From ba96a3ffc6fb463e3bf50b77da8b057c6d2fd747 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:07:06 +0200 Subject: [PATCH 86/97] Day 2 of Debugging --- utils/config/config.go | 7 ++++++- utils/config/loader.go | 6 +++--- utils/config/{token-config.go => tokens.go} | 2 -- 3 files changed, 9 insertions(+), 6 deletions(-) rename utils/config/{token-config.go => tokens.go} (99%) diff --git a/utils/config/config.go b/utils/config/config.go index a1789af..b8453f1 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -163,14 +163,19 @@ func transformChildren(config *koanf.Koanf, path string, transform func(key stri // Does the same thing as transformChildren() but does it for each Array Item inside of root and transforms subPath func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath string, transform func(key string, value any) (string, any)) error { var array []map[string]any - if err := config.Unmarshal(root, &array); err != nil { + + err := config.Unmarshal(root, &array) + if err != nil { return err } for i := range array { path := root + "." + strconv.Itoa(i) + "." + subPath + log.Dev(path) + if config.Exists(path) { + log.Dev("Exists") transformChildren(config, path, transform) } } diff --git a/utils/config/loader.go b/utils/config/loader.go index 6f0a4e9..b088472 100644 --- a/utils/config/loader.go +++ b/utils/config/loader.go @@ -26,9 +26,9 @@ type ENV_ struct { } type SETTING_ struct { - BLOCKED_ENDPOINTS []string `koanf:"blockedendpoints"` - VARIABLES map[string]any `koanf:"variables"` - MESSAGE_ALIASES []middlewareTypes.MessageAlias `koanf:"messagealiases"` + BLOCKED_ENDPOINTS []string `koanf:"blockedendpoints"` + VARIABLES map[string]any `koanf:"variables"` + MESSAGE_ALIASES []middlewareTypes.MessageAlias `koanf:"messagealiases"` } var ENV *ENV_ = &ENV_{ diff --git a/utils/config/token-config.go b/utils/config/tokens.go similarity index 99% rename from utils/config/token-config.go rename to utils/config/tokens.go index c8434a6..bdb2b0d 100644 --- a/utils/config/token-config.go +++ b/utils/config/tokens.go @@ -19,11 +19,9 @@ func LoadTokens() { LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) - /* normalizeKeys(tokensLayer) templateConfig(tokensLayer) - */ } func InitTokens() { From 4ba4aa3d7fed64a558530eec252374a4338eff2b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:10:51 +0200 Subject: [PATCH 87/97] further dev --- utils/config/config.go | 5 +++++ utils/config/tokens.go | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index b8453f1..a83eed8 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/safestrings" @@ -169,6 +170,10 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin return err } + log.Dev("Len: " + strconv.Itoa(len(array))) + + log.Dev(utils.ToJson(array)) + for i := range array { path := root + "." + strconv.Itoa(i) + "." + subPath diff --git a/utils/config/tokens.go b/utils/config/tokens.go index bdb2b0d..ff101ff 100644 --- a/utils/config/tokens.go +++ b/utils/config/tokens.go @@ -31,10 +31,14 @@ func InitTokens() { log.Dev(utils.ToJson(tokensLayer.All())) - transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { + err := transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) + if err != nil { + log.Dev(err.Error()) + } + log.Dev(utils.ToJson(tokensLayer.All())) tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) From e326d56ed797b3778398feff2e49ca392eca1b8f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:13:10 +0200 Subject: [PATCH 88/97] applied normalization --- utils/config/tokens.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/config/tokens.go b/utils/config/tokens.go index ff101ff..dc977eb 100644 --- a/utils/config/tokens.go +++ b/utils/config/tokens.go @@ -31,7 +31,7 @@ func InitTokens() { log.Dev(utils.ToJson(tokensLayer.All())) - err := transformChildrenUnderArray(tokensLayer, "tokenConfigs", "overrides.variables", func(key string, value any) (string, any) { + err := transformChildrenUnderArray(tokensLayer, "tokenconfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) @@ -41,7 +41,7 @@ func InitTokens() { log.Dev(utils.ToJson(tokensLayer.All())) - tokensLayer.Unmarshal("tokenConfigs", &tokenConfigs) + tokensLayer.Unmarshal("tokenconfigs", &tokenConfigs) log.Dev(utils.ToJson(tokenConfigs)) From 6cb710ab1a8e63a68e909e43be2c71ec721d6d8e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:16:12 +0200 Subject: [PATCH 89/97] testing manual normalization on variables --- utils/config/tokens.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/config/tokens.go b/utils/config/tokens.go index dc977eb..dd33942 100644 --- a/utils/config/tokens.go +++ b/utils/config/tokens.go @@ -17,7 +17,7 @@ type TOKEN_CONFIG_ struct { func LoadTokens() { log.Debug("Loading Configs ", ENV.TOKENS_DIR) - LoadDir("tokenConfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) + LoadDir("tokenconfigs", ENV.TOKENS_DIR, tokensLayer, yaml.Parser()) normalizeKeys(tokensLayer) @@ -29,7 +29,9 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - log.Dev(utils.ToJson(tokensLayer.All())) + transformChildren(tokensLayer, "tokenconfigs.0.overrides.variables", func(key string, value any) (string, any) { + return strings.ToUpper(key), value + }) err := transformChildrenUnderArray(tokensLayer, "tokenconfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value @@ -43,8 +45,6 @@ func InitTokens() { tokensLayer.Unmarshal("tokenconfigs", &tokenConfigs) - log.Dev(utils.ToJson(tokenConfigs)) - overrides := parseTokenConfigs(tokenConfigs) for token, override := range overrides { From 5b904155eb413f1eceeba4b3a264891ad412ade2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:55:42 +0200 Subject: [PATCH 90/97] more debugging --- utils/config/config.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index a83eed8..d14e7d0 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -139,25 +139,39 @@ func normalizeKeys(config *koanf.Koanf) { // Transforms Children of path func transformChildren(config *koanf.Koanf, path string, transform func(key string, value any) (string, any)) error { var sub map[string]any - if err := config.Unmarshal(path, &sub); err != nil { + + err := config.Unmarshal(path, &sub) + + if err != nil { return err } + log.Dev(utils.ToJson(sub)) + transformed := make(map[string]any) + for key, val := range sub { newKey, newVal := transform(key, val) transformed[newKey] = newVal + + log.Dev(newKey) } + + log.Dev(utils.ToJson(transformed)) config.Load(confmap.Provider(map[string]any{ path: map[string]any{}, }, "."), nil) + log.Dev("1:\n" + utils.ToJson(config.All())) + config.Load(confmap.Provider(map[string]any{ path: transformed, }, "."), nil) + log.Dev("2:\n" + utils.ToJson(config.All())) + return nil } @@ -170,17 +184,10 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin return err } - log.Dev("Len: " + strconv.Itoa(len(array))) - - log.Dev(utils.ToJson(array)) - for i := range array { path := root + "." + strconv.Itoa(i) + "." + subPath - log.Dev(path) - if config.Exists(path) { - log.Dev("Exists") transformChildren(config, path, transform) } } From 73dfe29234533631a65a3d79036234e84e6d5f0e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:58:43 +0200 Subject: [PATCH 91/97] closing in on issue --- utils/config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/config/config.go b/utils/config/config.go index d14e7d0..7694f5b 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -139,6 +139,8 @@ func normalizeKeys(config *koanf.Koanf) { // Transforms Children of path func transformChildren(config *koanf.Koanf, path string, transform func(key string, value any) (string, any)) error { var sub map[string]any + + log.Dev(utils.ToJson(config.All()) + "\nPath:" + path) err := config.Unmarshal(path, &sub) From 30e28156c8bc0d284c2cb7a1be8574a5904cfd87 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 12:10:44 +0200 Subject: [PATCH 92/97] restructured transform logic --- utils/config/config.go | 46 ++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 7694f5b..238825d 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -1,9 +1,9 @@ package config import ( + "errors" "os" "path/filepath" - "strconv" "strings" "sync" @@ -139,41 +139,33 @@ func normalizeKeys(config *koanf.Koanf) { // Transforms Children of path func transformChildren(config *koanf.Koanf, path string, transform func(key string, value any) (string, any)) error { var sub map[string]any - - log.Dev(utils.ToJson(config.All()) + "\nPath:" + path) + if !config.Exists(path) { + return errors.New("invalid path") + } + err := config.Unmarshal(path, &sub) if err != nil { return err } - log.Dev(utils.ToJson(sub)) - transformed := make(map[string]any) for key, val := range sub { newKey, newVal := transform(key, val) transformed[newKey] = newVal - - log.Dev(newKey) } - - log.Dev(utils.ToJson(transformed)) config.Load(confmap.Provider(map[string]any{ path: map[string]any{}, }, "."), nil) - log.Dev("1:\n" + utils.ToJson(config.All())) - config.Load(confmap.Provider(map[string]any{ path: transformed, }, "."), nil) - log.Dev("2:\n" + utils.ToJson(config.All())) - return nil } @@ -186,14 +178,34 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin return err } - for i := range array { - path := root + "." + strconv.Itoa(i) + "." + subPath + transformed := []map[string]any{} + + for _, data := range array { + tmp := koanf.New(".") + + tmp.Load(confmap.Provider(data, "."), nil) + + log.Dev(utils.ToJson(tmp.All())) - if config.Exists(path) { - transformChildren(config, path, transform) + err := transformChildren(tmp, subPath, transform) + + if err != nil { + return err } + + log.Dev(utils.ToJson(tmp.All())) + + transformed = append(transformed, tmp.All()) } + config.Load(confmap.Provider(map[string]any{ + root: map[string]any{}, + }, "."), nil) + + config.Load(confmap.Provider(map[string]any{ + root: transformed, + }, "."), nil) + return nil } From e9474dbc1a6398608e8db3ff9342d5bd47e8b60a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 12:11:04 +0200 Subject: [PATCH 93/97] removed broken line --- utils/config/tokens.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/config/tokens.go b/utils/config/tokens.go index dd33942..da10195 100644 --- a/utils/config/tokens.go +++ b/utils/config/tokens.go @@ -29,10 +29,6 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - transformChildren(tokensLayer, "tokenconfigs.0.overrides.variables", func(key string, value any) (string, any) { - return strings.ToUpper(key), value - }) - err := transformChildrenUnderArray(tokensLayer, "tokenconfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) From fc0b9c51b2259eae5005b0752123302c5c493c2c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 12:16:34 +0200 Subject: [PATCH 94/97] fix? --- utils/config/config.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 238825d..82aec61 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -183,11 +183,13 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin for _, data := range array { tmp := koanf.New(".") - tmp.Load(confmap.Provider(data, "."), nil) + tmp.Load(confmap.Provider(map[string]any{ + "item": data, + }, "."), nil) log.Dev(utils.ToJson(tmp.All())) - err := transformChildren(tmp, subPath, transform) + err := transformChildren(tmp, "item." + subPath, transform) if err != nil { return err @@ -195,7 +197,7 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin log.Dev(utils.ToJson(tmp.All())) - transformed = append(transformed, tmp.All()) + transformed = append(transformed, tmp.All()["item"].(map[string]any)) } config.Load(confmap.Provider(map[string]any{ From 729c98b1de817687b2d5de27112f3af6fbb30e4f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 12:23:07 +0200 Subject: [PATCH 95/97] further type safety --- utils/config/config.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/config/config.go b/utils/config/config.go index 82aec61..10f1115 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -197,7 +197,15 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin log.Dev(utils.ToJson(tmp.All())) - transformed = append(transformed, tmp.All()["item"].(map[string]any)) + item := tmp.Get("item") + + if item != nil { + itemMap, ok := item.(map[string]any) + + if ok { + transformed = append(transformed, itemMap) + } + } } config.Load(confmap.Provider(map[string]any{ From 71867784d4bc4c705a30a6db36fd05e0245ff605 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 12:25:59 +0200 Subject: [PATCH 96/97] clear old keys --- utils/config/config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 10f1115..1ac3cd4 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -158,9 +158,7 @@ func transformChildren(config *koanf.Koanf, path string, transform func(key stri transformed[newKey] = newVal } - config.Load(confmap.Provider(map[string]any{ - path: map[string]any{}, - }, "."), nil) + config.Delete(path) config.Load(confmap.Provider(map[string]any{ path: transformed, From d19f75f3817bfb486b3dc295136788c3839c24fc Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sun, 14 Sep 2025 12:30:11 +0200 Subject: [PATCH 97/97] finished testing --- utils/config/config.go | 9 +-------- utils/config/loader.go | 4 ---- utils/config/tokens.go | 9 +-------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/utils/config/config.go b/utils/config/config.go index 1ac3cd4..8b0ed54 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -7,7 +7,6 @@ import ( "strings" "sync" - "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/codeshelldev/secured-signal-api/utils/safestrings" @@ -185,16 +184,12 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin "item": data, }, "."), nil) - log.Dev(utils.ToJson(tmp.All())) - err := transformChildren(tmp, "item." + subPath, transform) if err != nil { return err } - log.Dev(utils.ToJson(tmp.All())) - item := tmp.Get("item") if item != nil { @@ -206,9 +201,7 @@ func transformChildrenUnderArray(config *koanf.Koanf, root string, subPath strin } } - config.Load(confmap.Provider(map[string]any{ - root: map[string]any{}, - }, "."), nil) + config.Delete(root) config.Load(confmap.Provider(map[string]any{ root: transformed, diff --git a/utils/config/loader.go b/utils/config/loader.go index b088472..be45893 100644 --- a/utils/config/loader.go +++ b/utils/config/loader.go @@ -8,7 +8,6 @@ import ( "strings" middlewareTypes "github.com/codeshelldev/secured-signal-api/internals/proxy/middlewares/types" - "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -88,9 +87,6 @@ func Load() { InitEnv() log.Info("Finished Loading Configuration") - - log.Dev(utils.ToJson(config.All())) - log.Dev(utils.ToJson(ENV)) } func LoadDefaults() { diff --git a/utils/config/tokens.go b/utils/config/tokens.go index da10195..ce995a0 100644 --- a/utils/config/tokens.go +++ b/utils/config/tokens.go @@ -4,7 +4,6 @@ import ( "strconv" "strings" - "github.com/codeshelldev/secured-signal-api/utils" log "github.com/codeshelldev/secured-signal-api/utils/logger" "github.com/knadh/koanf/parsers/yaml" ) @@ -29,16 +28,10 @@ func InitTokens() { var tokenConfigs []TOKEN_CONFIG_ - err := transformChildrenUnderArray(tokensLayer, "tokenconfigs", "overrides.variables", func(key string, value any) (string, any) { + transformChildrenUnderArray(tokensLayer, "tokenconfigs", "overrides.variables", func(key string, value any) (string, any) { return strings.ToUpper(key), value }) - if err != nil { - log.Dev(err.Error()) - } - - log.Dev(utils.ToJson(tokensLayer.All())) - tokensLayer.Unmarshal("tokenconfigs", &tokenConfigs) overrides := parseTokenConfigs(tokenConfigs)