From f807c2759635a850a5a1fcb5b217feb99430c6f0 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Tue, 18 Feb 2025 17:42:51 +0200 Subject: [PATCH 1/6] Upgrade dependencies and Go version for compatibility Updated multiple dependencies in `go.mod` and `go.sum` to their latest versions, including a switch from YAML v2 to YAML v3. Also upgraded Go version to 1.23.0 to align with the updated toolchain and dependency requirements. --- installer/go.mod | 29 +++++++++++++---------------- installer/go.sum | 38 ++++++++++++++++++-------------------- installer/utils/os.go | 2 +- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/installer/go.mod b/installer/go.mod index 26c259366..0937fd02b 100644 --- a/installer/go.mod +++ b/installer/go.mod @@ -1,33 +1,30 @@ module github.com/utmstack/UTMStack/installer -go 1.21.0 -toolchain go1.22.5 +go 1.23.0 + +toolchain go1.23.5 require ( - github.com/cloudfoundry/gosigar v1.3.84 + github.com/cloudfoundry/gosigar v1.3.88 github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d github.com/shirou/gopsutil/v3 v3.24.5 - gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/go-ole/go-ole v1.3.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/lib/pq v1.10.9 + github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shoenig/test v1.8.0 // indirect - github.com/tklauser/numcpus v0.7.0 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.9.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - golang.org/x/net v0.34.0 // indirect - golang.org/x/sys v0.29.0 // indirect -) - -require ( - github.com/go-ole/go-ole v1.3.0 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/lib/pq v1.10.9 - github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect - github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/tklauser/go-sysconf v0.3.13 // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect ) diff --git a/installer/go.sum b/installer/go.sum index 53b330168..6ac129cef 100644 --- a/installer/go.sum +++ b/installer/go.sum @@ -1,5 +1,5 @@ -github.com/cloudfoundry/gosigar v1.3.84 h1:6WINPBQQLHcJwJTFaUpFcN9WwPRfuzQidxeq3xZiwoM= -github.com/cloudfoundry/gosigar v1.3.84/go.mod h1:EQ0zg38VQKzEYkmFgaYQ4ETQvI7TDgHhd/8Rk0XjhdI= +github.com/cloudfoundry/gosigar v1.3.88 h1:5DMK13C0wKBYBtDfFEXcgW9LpJK0gGdZRWH2kFeFfnk= +github.com/cloudfoundry/gosigar v1.3.88/go.mod h1:lC508fd/yfkk9XciYdbB/G0KlfaMm8nuhC9aGpZzxxY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,8 +15,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc= +github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -25,8 +25,8 @@ github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d h1:8fVmm2qScPn github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d/go.mod h1:dFu6nuJHC3u9kCDcyGrEL7LwhK2m6Mt+alyiiIjDrRY= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 h1:1KuuSOy4ZNgW0KA2oYIngXVFhQcXxhLqCVK7cBcldkk= -github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= @@ -49,28 +49,26 @@ github.com/shoenig/test v1.8.0 h1:8f4lrmjkoSykT+EfiTtJuWbV4eaNEBWsYXcl1n6C6BY= github.com/shoenig/test v1.8.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= -github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= -github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= -github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo= +github.com/tklauser/numcpus v0.9.0/go.mod h1:SN6Nq1O3VychhC1npsWostA+oW+VOQTxZrS604NSRyI= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= +golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/installer/utils/os.go b/installer/utils/os.go index bb5acb261..b17fd4460 100644 --- a/installer/utils/os.go +++ b/installer/utils/os.go @@ -6,7 +6,7 @@ import ( "os/exec" "path/filepath" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func RunEnvCmd(env []string, command string, arg ...string) error { From 3ac6aefe70d749363422d42eb9752ba1e35a02c2 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Feb 2025 11:06:41 +0200 Subject: [PATCH 2/6] Refactor error handling and rename function for clarity Improved error handling in several functions to make logic clearer and avoid potential issues. Renamed `expresion` to `expression` for correctness and updated related references. Consolidated redundant error paths and adjusted logic for better readability and maintainability. --- correlation/cache/operators.go | 67 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/correlation/cache/operators.go b/correlation/cache/operators.go index 5f51d3853..e1d3519d4 100644 --- a/correlation/cache/operators.go +++ b/correlation/cache/operators.go @@ -12,16 +12,20 @@ import ( func inCIDR(addr, network string) (bool, error) { _, subnet, err := net.ParseCIDR(network) - if err == nil { - ip := net.ParseIP(addr) - if ip != nil { - if subnet.Contains(ip) { - return true, nil - } - } + if err != nil { + return false, fmt.Errorf("invalid CIDR") + } + + ip := net.ParseIP(addr) + if ip == nil { return false, fmt.Errorf("invalid IP address") } - return false, err + + if subnet.Contains(ip) { + return true, nil + } + + return false, nil } func equal(val1, val2 string) bool { @@ -54,25 +58,30 @@ func endWith(str, suff string) bool { return strings.HasSuffix(str, suff) } -func expresion(exp, str string) (bool, error) { +func expression(exp, str string) (bool, error) { re, err := regexp.Compile(exp) - if err == nil { - if re.MatchString(str) { - return true, nil - } + if err != nil { + return false, err } - return false, err + + if re.MatchString(str) { + return true, nil + } + + return false, nil } func parseFloats(val1, val2 string) (float64, float64, error) { - f1, err1 := strconv.ParseFloat(val1, 64) - if err1 != nil { - return 0, 0, err1 + f1, err := strconv.ParseFloat(val1, 64) + if err != nil { + return 0, 0, err } - f2, err2 := strconv.ParseFloat(val2, 64) - if err2 != nil { - return 0, 0, err2 + + f2, err := strconv.ParseFloat(val2, 64) + if err != nil { + return 0, 0, err } + return f1, f2, nil } @@ -105,17 +114,17 @@ func compare(operator, val1, val2 string) bool { case "not end with": return !endWith(val1, val2) case "regexp": - matched, err := expresion(val2, val1) + matched, err := expression(val2, val1) if err != nil { return false } return matched case "not regexp": - matched, err := expresion(val2, val1) + matched, err := expression(val2, val1) if err != nil { return false } - return matched + return !matched case "<": f1, f2, err := parseFloats(val1, val2) if err != nil { @@ -144,16 +153,16 @@ func compare(operator, val1, val2 string) bool { return true case "in cidr": matched, err := inCIDR(val1, val2) - if err == nil { - return matched + if err != nil { + return false } - return false + return matched case "not in cidr": matched, err := inCIDR(val1, val2) - if err == nil { - return !matched + if err != nil { + return false } - return false + return !matched default: return false } From a9df5cf7efe7adf8c231189c71f568cae2d70d27 Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Feb 2025 11:09:54 +0200 Subject: [PATCH 3/6] Refactor geo updates and add new Threat Intelligence logic Reorganized the GeoIP update logic into a new `Load` function, improving code maintainability and clarity. Introduced Threat Intelligence feeds loading and blocklist handling via new methods in the `ti` package. Updated storage handling and various utility methods to streamline operations and improve consistency across modules. --- correlation/Dockerfile | 15 ++- correlation/api/newLogHandler.go | 2 + correlation/cache/cache.go | 42 +++---- correlation/cache/cache_test.go | 13 +-- correlation/correlation/analyzer.go | 74 ++---------- correlation/correlation/reporter.go | 54 +++++++-- correlation/docs/docs.go | 8 +- correlation/docs/swagger.json | 8 +- correlation/docs/swagger.yaml | 8 +- correlation/geo/{update.go => bases.go} | 99 ++++++---------- correlation/geo/geolocate.go | 53 ++------- correlation/go.mod | 58 +++++----- correlation/go.sum | 143 ++++++++++-------------- correlation/main.go | 30 ++--- correlation/rules/rules.go | 35 +++--- correlation/search/queue.go | 5 +- correlation/search/search.go | 6 +- correlation/sqldb/cnn.go | 19 ++-- correlation/sqldb/crud.go | 30 ----- correlation/ti/bases.go | 52 +++++++++ correlation/ti/ti.go | 121 ++++++++++++++++++++ correlation/utils/configuration.go | 3 +- correlation/utils/details.go | 62 ++++++++++ correlation/utils/httpClient.go | 30 +---- 24 files changed, 518 insertions(+), 452 deletions(-) rename correlation/geo/{update.go => bases.go} (60%) create mode 100644 correlation/ti/bases.go create mode 100644 correlation/ti/ti.go create mode 100644 correlation/utils/details.go diff --git a/correlation/Dockerfile b/correlation/Dockerfile index 0b1aa0041..5ee7bec42 100644 --- a/correlation/Dockerfile +++ b/correlation/Dockerfile @@ -1,6 +1,6 @@ -FROM ubuntu:22.04 -RUN apt update -RUN apt install -y ca-certificates git +FROM ubuntu:24.04 +RUN apt-get update +RUN apt-get install -y ca-certificates git wget COPY correlation /app/ COPY docs/swagger.json /app/docs/ COPY docs/swagger.yaml /app/docs/ @@ -9,4 +9,13 @@ COPY run.sh / RUN chmod +x /app/correlation RUN chmod +x /run.sh RUN update-ca-certificates +RUN wget -O /app/asn-blocks-v4.csv https://cdn.utmstack.com/geoip/asn-blocks-v4.csv +RUN wget -O /app/asn-blocks-v6.csv https://cdn.utmstack.com/geoip/asn-blocks-v6.csv +RUN wget -O /app/blocks-v4.csv https://cdn.utmstack.com/geoip/blocks-v4.csv +RUN wget -O /app/blocks-v6.csv https://cdn.utmstack.com/geoip/blocks-v6.csv +RUN wget -O /app/locations-en.csv https://cdn.utmstack.com/geoip/locations-en.csv +RUN wget -O /app/ip_blocklist.list https://intelligence.threatwinds.com/feeds/public/ip/cumulative.list +RUN wget -O /app/domain_blocklist.list https://intelligence.threatwinds.com/feeds/public/domain/cumulative.list +RUN wget -O /app/hostname_blocklist.list https://intelligence.threatwinds.com/feeds/public/hostname/cumulative.list + ENTRYPOINT [ "/run.sh" ] \ No newline at end of file diff --git a/correlation/api/newLogHandler.go b/correlation/api/newLogHandler.go index 98a96f359..c0c9fda64 100644 --- a/correlation/api/newLogHandler.go +++ b/correlation/api/newLogHandler.go @@ -3,6 +3,7 @@ package api import ( "encoding/json" "fmt" + "github.com/utmstack/UTMStack/correlation/ti" "io" "log" "net/http" @@ -74,6 +75,7 @@ func NewLog(c *gin.Context) { } cache.AddToCache(l) + ti.Enqueue(l) search.AddToQueue(l) response["status"] = "queued" c.JSON(http.StatusOK, response) diff --git a/correlation/cache/cache.go b/correlation/cache/cache.go index 8ce365d25..da4a80ea9 100644 --- a/correlation/cache/cache.go +++ b/correlation/cache/cache.go @@ -13,15 +13,15 @@ import ( const bufferSize int = 1000000 -var cacheStorageMutex = &sync.RWMutex{} +var storageMutex = &sync.RWMutex{} -var CacheStorage []string +var storage []string func Status() { for { - log.Printf("Logs in cache: %v", len(CacheStorage)) - if len(CacheStorage) != 0 { - est := gjson.Get(CacheStorage[0], "@timestamp").String() + log.Printf("Logs in cache: %v", len(storage)) + if len(storage) != 0 { + est := gjson.Get(storage[0], "@timestamp").String() log.Printf("Old document in cache: %s", est) } time.Sleep(60 * time.Second) @@ -31,8 +31,8 @@ func Status() { func Search(allOf []rules.AllOf, oneOf []rules.OneOf, seconds int64) []string { var elements []string - cacheStorageMutex.RLock() - defer cacheStorageMutex.RUnlock() + storageMutex.RLock() + defer storageMutex.RUnlock() cToBreak := 0 ait := time.Now().UTC().Unix() - func() int64 { @@ -43,8 +43,8 @@ func Search(allOf []rules.AllOf, oneOf []rules.OneOf, seconds int64) []string { return seconds } }() - for i := len(CacheStorage) - 1; i >= 0; i-- { - est := gjson.Get(CacheStorage[i], "@timestamp").String() + for i := len(storage) - 1; i >= 0; i-- { + est := gjson.Get(storage[i], "@timestamp").String() eit, err := time.Parse(time.RFC3339Nano, est) if err != nil { log.Printf("Could not parse @timestamp: %v", err) @@ -61,23 +61,23 @@ func Search(allOf []rules.AllOf, oneOf []rules.OneOf, seconds int64) []string { var allCatch bool var oneCatch bool for _, of := range oneOf { - oneCatch = evalElement(CacheStorage[i], of.Field, of.Operator, of.Value) + oneCatch = evalElement(storage[i], of.Field, of.Operator, of.Value) if oneCatch { break } } for _, af := range allOf { - allCatch = evalElement(CacheStorage[i], af.Field, af.Operator, af.Value) + allCatch = evalElement(storage[i], af.Field, af.Operator, af.Value) if !allCatch { break } } if (len(allOf) == 0 || allCatch) && (len(oneOf) == 0 || oneCatch) { - elements = append(elements, CacheStorage[i]) + elements = append(elements, storage[i]) } } } - + return elements } @@ -97,9 +97,9 @@ func ProcessQueue() { go func() { for { l := <-logs - cacheStorageMutex.Lock() - CacheStorage = append(CacheStorage, l) - cacheStorageMutex.Unlock() + storageMutex.Lock() + storage = append(storage, l) + storageMutex.Unlock() } }() } @@ -109,11 +109,11 @@ func Clean() { for { var clean bool - if len(CacheStorage) > 1 { + if len(storage) > 1 { if utils.AssignedMemory >= 80 { clean = true } else { - old := gjson.Get(CacheStorage[0], "@timestamp").String() + old := gjson.Get(storage[0], "@timestamp").String() oldTime, err := time.Parse(time.RFC3339Nano, old) if err != nil { log.Printf("Could not parse old log timestamp. Cleaning up") @@ -129,9 +129,9 @@ func Clean() { } if clean { - cacheStorageMutex.Lock() - CacheStorage = CacheStorage[1:] - cacheStorageMutex.Unlock() + storageMutex.Lock() + storage = storage[1:] + storageMutex.Unlock() } else { time.Sleep(5 * time.Second) } diff --git a/correlation/cache/cache_test.go b/correlation/cache/cache_test.go index f5941c607..bded36df1 100644 --- a/correlation/cache/cache_test.go +++ b/correlation/cache/cache_test.go @@ -1,11 +1,8 @@ -package cache_test +package cache import ( - "testing" - - - "github.com/utmstack/UTMStack/correlation/cache" "github.com/utmstack/UTMStack/correlation/rules" + "testing" ) func TestSearch(t *testing.T) { @@ -16,7 +13,7 @@ func TestSearch(t *testing.T) { `{"@timestamp":"2022-01-01T00:00:03.000Z","field1":"value1","field2":"value2"}`, `{"@timestamp":"2022-01-01T00:00:04.000Z","field1":"value1","field2":"value2"}`, } - cache.CacheStorage = cacheStorage + storage = cacheStorage allOf := []rules.AllOf{ {Field: "field1", Operator: "==", Value: "value1"}, } @@ -31,7 +28,7 @@ func TestSearch(t *testing.T) { `{"@timestamp":"2022-01-01T00:00:01.000Z","field1":"value1","field2":"value2"}`, `{"@timestamp":"2022-01-01T00:00:00.000Z","field1":"value1","field2":"value2"}`, } - result := cache.Search(allOf, oneOf, int64(seconds)) + result := Search(allOf, oneOf, int64(seconds)) if len(result) != len(expected) { t.Errorf("Expected %d elements, but got %d", len(expected), len(result)) } @@ -40,4 +37,4 @@ func TestSearch(t *testing.T) { t.Errorf("Expected %s, but got %s", expected[i], r) } } -} \ No newline at end of file +} diff --git a/correlation/correlation/analyzer.go b/correlation/correlation/analyzer.go index 8b14addde..f17b0418c 100644 --- a/correlation/correlation/analyzer.go +++ b/correlation/correlation/analyzer.go @@ -1,17 +1,15 @@ package correlation import ( - "net" - - "github.com/utmstack/UTMStack/correlation/geo" - "github.com/utmstack/UTMStack/correlation/rules" "github.com/tidwall/gjson" + "github.com/utmstack/UTMStack/correlation/rules" + "github.com/utmstack/UTMStack/correlation/utils" ) -func processResponse(logs []string, rule rules.Rule, save []rules.SavedField, tmpLogs *[20][]map[string]string, +func processResponse(logs []string, rule rules.Rule, save []utils.SavedField, tmpLogs *[20][]map[string]string, steps, step, minCount int) { - if len(logs) >= func()int{ - switch minCount{ + if len(logs) >= func() int { + switch minCount { case 0: return 1 default: @@ -19,67 +17,9 @@ func processResponse(logs []string, rule rules.Rule, save []rules.SavedField, tm } }() { for _, l := range logs { - var fields = map[string]string{ - "id": gjson.Get(l, "id").String(), - } - //User saved fields - for _, save := range save { - fields[save.Alias] = gjson.Get(l, save.Field).String() - } - // Try to resolve SourceHost if SourceIP exists but not SourceHost - if fields["SourceHost"] == "" && fields["SourceIP"] != "" { - host, _ := net.LookupHost(fields["SourceIP"]) - fields["SourceHost"] = host[0] - } - // Try to resolve DestinationHost if DestinationIP exists but not DestinationHost - if fields["DestinationHost"] == "" && fields["DestinationIP"] != "" { - host, _ := net.LookupHost(fields["DestinationIP"]) - fields["DestinationHost"] = host[0] - } - // Try to resolve SourceIP if SourceHost exists but not SourceIP - if fields["SourceHost"] != "" && fields["SourceIP"] == "" { - ip, _ := net.LookupIP(fields["SourceHost"]) - if len(ip) != 0 && ip[0].String() != "" { - fields["SourceIP"] = ip[0].String() - } - } - // Try to resolve DestinationIP if DestinationHost exists but not DestinationIP - if fields["DestinationHost"] != "" && fields["DestinationIP"] == "" { - ip, _ := net.LookupIP(fields["DestinationHost"]) - if len(ip) != 0 && ip[0].String() != "" { - fields["DestinationIP"] = ip[0].String() - } - } - // Try to geolocate SourceIP if exists - if fields["SourceIP"] != "" { - location := geo.Geolocate(fields["SourceIP"]) - fields["SourceCountry"] = location["country"] - fields["SourceCountryCode"] = location["countryCode"] - fields["SourceCity"] = location["city"] - fields["SourceLat"] = location["latitude"] - fields["SourceLon"] = location["longitude"] - fields["SourceAccuracyRadius"] = location["accuracyRadius"] - fields["SourceASN"] = location["asn"] - fields["SourceASO"] = location["aso"] - fields["SourceIsSatelliteProvider"] = location["isSatelliteProvider"] - fields["SourceIsAnonymousProxy"] = location["isAnonymousProxy"] - } - // Try to geolocate DetinationIP if exists - if fields["DestinationIP"] != "" { - location := geo.Geolocate(fields["DestinationIP"]) - fields["DestinationCountry"] = location["country"] - fields["DestinationCountryCode"] = location["countryCode"] - fields["DestinationCity"] = location["city"] - fields["DestinationLat"] = location["latitude"] - fields["DestinationLon"] = location["longitude"] - fields["DestinationAccuracyRadius"] = location["accuracyRadius"] - fields["DestinationASN"] = location["asn"] - fields["DestinationASO"] = location["aso"] - fields["DestinationIsSatelliteProvider"] = location["isSatelliteProvider"] - fields["DestinationIsAnonymousProxy"] = location["isAnonymousProxy"] - } + fields := utils.ExtractDetails(save, l) - // Alert in the last step or save data to next cicle + // Alert in the last step or save data to the next iteration if steps-1 == step { // Use content of AlertName as Name if exists var alertName string diff --git a/correlation/correlation/reporter.go b/correlation/correlation/reporter.go index bccdee6fe..4b0a43249 100644 --- a/correlation/correlation/reporter.go +++ b/correlation/correlation/reporter.go @@ -2,16 +2,15 @@ package correlation import ( "encoding/json" - "strconv" - "strings" - "time" - - "log" - "github.com/google/uuid" "github.com/levigross/grequests" + "github.com/utmstack/UTMStack/correlation/geo" "github.com/utmstack/UTMStack/correlation/search" "github.com/utmstack/UTMStack/correlation/utils" + "log" + "strconv" + "strings" + "time" ) type Host struct { @@ -65,13 +64,47 @@ type AlertFields struct { } func Alert(name, severity, description, solution, category, tactic string, reference []string, dataType, dataSource string, - details map[string]string) { + fields map[string]string) { + + // Try to geolocate SourceIP if exists + if fields["SourceIP"] != "" { + location := geo.Geolocate(fields["SourceIP"]) + if len(location) != 0 { + fields["SourceCountry"] = location["country"] + fields["SourceCountryCode"] = location["countryCode"] + fields["SourceCity"] = location["city"] + fields["SourceLat"] = location["latitude"] + fields["SourceLon"] = location["longitude"] + fields["SourceAccuracyRadius"] = location["accuracyRadius"] + fields["SourceASN"] = location["asn"] + fields["SourceASO"] = location["aso"] + fields["SourceIsSatelliteProvider"] = location["isSatelliteProvider"] + fields["SourceIsAnonymousProxy"] = location["isAnonymousProxy"] + } + } + + // Try to geolocate DestinationIP if exists + if fields["DestinationIP"] != "" { + location := geo.Geolocate(fields["DestinationIP"]) + if len(location) != 0 { + fields["DestinationCountry"] = location["country"] + fields["DestinationCountryCode"] = location["countryCode"] + fields["DestinationCity"] = location["city"] + fields["DestinationLat"] = location["latitude"] + fields["DestinationLon"] = location["longitude"] + fields["DestinationAccuracyRadius"] = location["accuracyRadius"] + fields["DestinationASN"] = location["asn"] + fields["DestinationASO"] = location["aso"] + fields["DestinationIsSatelliteProvider"] = location["isSatelliteProvider"] + fields["DestinationIsAnonymousProxy"] = location["isAnonymousProxy"] + } + } log.Printf("Reporting alert: %s", name) - if !UpdateAlert(name, severity, details) { + if !UpdateAlert(name, severity, fields) { NewAlert(name, severity, description, solution, category, tactic, reference, dataType, dataSource, - details) + fields) } } @@ -208,11 +241,12 @@ func UpdateAlert(name, severity string, details map[string]string) bool { }, }, }) - _ = r.Close() if err != nil { log.Printf("Could not update existent alert: %v", err) return false } + + _ = r.Close() } } } diff --git a/correlation/docs/docs.go b/correlation/docs/docs.go index 564f347d8..a705cddcf 100644 --- a/correlation/docs/docs.go +++ b/correlation/docs/docs.go @@ -10,11 +10,11 @@ const docTemplate = `{ "description": "{{escape .Description}}", "title": "{{.Title}}", "contact": { - "name": "Osmany Montero", - "email": "osmany@quantfall.com" + "name": "UTMStack LLC", + "email": "contact@utmstack.com" }, "license": { - "name": "Private" + "name": "AGPLv3" }, "version": "{{.Version}}" }, @@ -47,7 +47,7 @@ var SwaggerInfo = &swag.Spec{ BasePath: "/v1", Schemes: []string{}, Title: "UTMStack's Correlation Engine", - Description: "Rules based correlation engine for UTMStack.", + Description: "Rules-based correlation engine for UTMStack.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, LeftDelim: "{{", diff --git a/correlation/docs/swagger.json b/correlation/docs/swagger.json index 9e984815e..8a5456388 100644 --- a/correlation/docs/swagger.json +++ b/correlation/docs/swagger.json @@ -1,14 +1,14 @@ { "swagger": "2.0", "info": { - "description": "Rules based correlation engine for UTMStack.", + "description": "Rules-based correlation engine for UTMStack.", "title": "UTMStack's Correlation Engine", "contact": { - "name": "Osmany Montero", - "email": "osmany@quantfall.com" + "name": "UTMStack LLC", + "email": "contact@utmstack.com" }, "license": { - "name": "Private" + "name": "AGPLv3" }, "version": "1.0" }, diff --git a/correlation/docs/swagger.yaml b/correlation/docs/swagger.yaml index a76fbbf35..088abb17c 100644 --- a/correlation/docs/swagger.yaml +++ b/correlation/docs/swagger.yaml @@ -2,11 +2,11 @@ basePath: /v1 host: localhost:8080 info: contact: - email: osmany@quantfall.com - name: Osmany Montero - description: Rules based correlation engine for UTMStack. + email: contact@utmstack.com + name: UTMStack LLC + description: Rules-based correlation engine for UTMStack. license: - name: Private + name: AGPLv3 title: UTMStack's Correlation Engine version: "1.0" paths: diff --git a/correlation/geo/update.go b/correlation/geo/bases.go similarity index 60% rename from correlation/geo/update.go rename to correlation/geo/bases.go index 4a67b30be..093ed243d 100644 --- a/correlation/geo/update.go +++ b/correlation/geo/bases.go @@ -1,76 +1,47 @@ package geo import ( + "github.com/utmstack/UTMStack/correlation/utils" "log" "net" - "os" "path/filepath" "strconv" - "time" - - "github.com/utmstack/UTMStack/correlation/utils" ) -// Goroutine for update geolocalization databases -func Update(updateReady chan bool) { - first := true - for { - log.Printf("Downloading GeoIP databases") - cnf := utils.GetConfig() - var files = map[string]string{ - filepath.Join(cnf.GeoIPFolder, "asn-blocks-v4.csv"): "https://cdn.utmstack.com/geoip/asn-blocks-v4.csv", - filepath.Join(cnf.GeoIPFolder, "asn-blocks-v6.csv"): "https://cdn.utmstack.com/geoip/asn-blocks-v6.csv", - filepath.Join(cnf.GeoIPFolder, "blocks-v4.csv"): "https://cdn.utmstack.com/geoip/blocks-v4.csv", - filepath.Join(cnf.GeoIPFolder, "blocks-v6.csv"): "https://cdn.utmstack.com/geoip/blocks-v6.csv", - filepath.Join(cnf.GeoIPFolder, "locations-en.csv"): "https://cdn.utmstack.com/geoip/locations-en.csv", - } - - if _, err := os.Stat(cnf.GeoIPFolder); os.IsNotExist(err) { - os.MkdirAll(cnf.GeoIPFolder, os.ModeDir) - } - - if _, err := os.Stat(filepath.Join(cnf.GeoIPFolder, "locations-en.csv")); os.IsNotExist(err) || !first { - for file, url := range files { - if err := utils.Download(url, file); err != nil { - log.Printf("Could not download file: %v", err) - continue - } - } - } - - for file := range files { - csv := utils.ReadCSV(file) - mu.Lock() - switch file { - case filepath.Join(cnf.GeoIPFolder, "asn-blocks-v4.csv"): - asnBlocks = nil - populateASNBlocks(csv) - case filepath.Join(cnf.GeoIPFolder, "asn-blocks-v6.csv"): - populateASNBlocks(csv) - case filepath.Join(cnf.GeoIPFolder, "blocks-v4.csv"): - cityBlocks = nil - populateCityBlocks(csv) - case filepath.Join(cnf.GeoIPFolder, "blocks-v6.csv"): - populateCityBlocks(csv) - case filepath.Join(cnf.GeoIPFolder, "locations-en.csv"): - cityLocations = nil - populateCityLocations(csv) - } - mu.Unlock() - } - - log.Printf("asnBlocks rows: %v", len(asnBlocks)) - log.Printf("cityBlocks rows: %v", len(cityBlocks)) - log.Printf("cityLocations rows: %v", len(cityLocations)) - log.Printf("GeoIP databases updated") - - if first { - first = false - updateReady <- true - } - - time.Sleep(168 * time.Hour) +func Load() { + log.Printf("Loading GeoIP databases") + + var files = []string{ + "asn-blocks-v4.csv", + "asn-blocks-v6.csv", + "blocks-v4.csv", + "blocks-v6.csv", + "locations-en.csv", + } + + for _, file := range files { + csv := utils.ReadCSV(filepath.Join("/app", file)) + switch file { + case "asn-blocks-v4.csv": + asnBlocks = nil + populateASNBlocks(csv) + case "asn-blocks-v6.csv": + populateASNBlocks(csv) + case "blocks-v4.csv": + cityBlocks = nil + populateCityBlocks(csv) + case "blocks-v6.csv": + populateCityBlocks(csv) + case "locations-en.csv": + cityLocations = nil + populateCityLocations(csv) + } } + + log.Printf("asnBlocks rows: %v", len(asnBlocks)) + log.Printf("cityBlocks rows: %v", len(cityBlocks)) + log.Printf("cityLocations rows: %v", len(cityLocations)) + log.Printf("GeoIP databases loaded") } func populateASNBlocks(csv [][]string) { @@ -114,6 +85,7 @@ func populateCityBlocks(csv [][]string) { if line[1] == "" { continue } + geonameID, err := strconv.Atoi(line[1]) if err != nil { log.Printf("Could not parse geonameID in populateCityBlocks: %v", err) @@ -171,7 +143,6 @@ func populateCityBlocks(csv [][]string) { } cityBlocks = append(cityBlocks, t) - } } diff --git a/correlation/geo/geolocate.go b/correlation/geo/geolocate.go index 4ef67c923..55cdd55bc 100644 --- a/correlation/geo/geolocate.go +++ b/correlation/geo/geolocate.go @@ -3,17 +3,8 @@ package geo import ( "fmt" "net" - "strings" - "sync" - - "github.com/levigross/grequests" ) -var mu = &sync.Mutex{} - -var myExternalIP net.IP -var myExternalIPOnce sync.Once - type asnBlock struct { network *net.IPNet asn int @@ -63,28 +54,7 @@ func IsLocal(a net.IP) bool { return false } -func GetExternal() net.IP { - if resp, err := grequests.Get("http://myexternalip.com/raw", nil); err == nil { - ip := net.ParseIP(strings.TrimSpace(resp.String())) - return ip - } - return nil -} - -func GetExternalOnce() net.IP { - myExternalIPOnce.Do(func() { - myExternalIP = GetExternal() - }) - return myExternalIP -} - -func getCity(a string) cityBlock { - ip := net.ParseIP(a) - - if IsLocal(ip) { - ip = GetExternalOnce() - } - +func getCity(ip net.IP) cityBlock { var city cityBlock for _, e := range cityBlocks { if e.network.Contains(ip) { @@ -94,13 +64,7 @@ func getCity(a string) cityBlock { return city } -func getASN(a string) asnBlock { - ip := net.ParseIP(a) - - if IsLocal(ip) { - ip = GetExternalOnce() - } - +func getASN(ip net.IP) asnBlock { var asn asnBlock for _, e := range asnBlocks { if e.network.Contains(ip) { @@ -121,11 +85,16 @@ func getLocation(geonameID int) cityLocation { } func Geolocate(ip string) map[string]string { - mu.Lock() - asn := getASN(ip) - city := getCity(ip) + parsedIP := net.ParseIP(ip) + + if IsLocal(parsedIP) { + return map[string]string{} + } + + asn := getASN(parsedIP) + city := getCity(parsedIP) location := getLocation(city.geonameID) - mu.Unlock() + return map[string]string{ "country": location.countryName, "countryCode": location.countryISOCode, diff --git a/correlation/go.mod b/correlation/go.mod index 0cb4f204b..569b9f679 100644 --- a/correlation/go.mod +++ b/correlation/go.mod @@ -1,12 +1,14 @@ module github.com/utmstack/UTMStack/correlation -go 1.21 +go 1.22.0 + +toolchain go1.23.5 require ( github.com/fsnotify/fsnotify v1.8.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 - github.com/levigross/grequests v0.0.0-20221222020224-9eee758d18d5 + github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d github.com/lib/pq v1.10.9 github.com/shirou/gopsutil/v3 v3.24.5 github.com/swaggo/files v1.0.1 @@ -18,46 +20,44 @@ require ( require ( github.com/KyleBanks/depth v1.2.1 // indirect - github.com/bytedance/sonic v1.11.6 // indirect - github.com/bytedance/sonic/loader v0.1.1 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/bytedance/sonic v1.12.8 // indirect + github.com/bytedance/sonic/loader v0.2.3 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.20.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/go-playground/validator/v10 v10.24.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.9.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/arch v0.14.0 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/tools v0.30.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/correlation/go.sum b/correlation/go.sum index cfbe9ee96..e8b5d2cc4 100644 --- a/correlation/go.sum +++ b/correlation/go.sum @@ -1,55 +1,47 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= -github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= -github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic v1.12.8 h1:4xYRVRlXIgvSZ4e8iVTlMF5szgpXd4AfvuWgA8I8lgs= +github.com/bytedance/sonic v1.12.8/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0= +github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= -github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= -github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg= +github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -63,30 +55,23 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/levigross/grequests v0.0.0-20221222020224-9eee758d18d5 h1:AsF9Q1mQoyLv0HzvHFW7O+19dHilOcKU74k7E5ufI1A= -github.com/levigross/grequests v0.0.0-20221222020224-9eee758d18d5/go.mod h1:XfzeIE2WC7CGDhlZJY/rUdqUPy0IPcyI6hoIjhAMNbQ= +github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d h1:8fVmm2qScPn4JAF/YdTtqrPP3n58FgZ4GbKTNfaPuRs= +github.com/levigross/grequests v0.0.0-20231203190023-9c307ef1f48d/go.mod h1:dFu6nuJHC3u9kCDcyGrEL7LwhK2m6Mt+alyiiIjDrRY= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= -github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -94,13 +79,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= -github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= @@ -110,14 +94,13 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= @@ -131,10 +114,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo= +github.com/tklauser/numcpus v0.9.0/go.mod h1:SN6Nq1O3VychhC1npsWostA+oW+VOQTxZrS604NSRyI= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= @@ -142,26 +125,25 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= -golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= +golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -172,10 +154,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -183,28 +163,21 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= +golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/correlation/main.go b/correlation/main.go index a3a5544c0..a4f049064 100644 --- a/correlation/main.go +++ b/correlation/main.go @@ -1,10 +1,14 @@ package main import ( + "github.com/utmstack/UTMStack/correlation/ti" "os" "os/signal" "syscall" + "github.com/gin-gonic/gin" + swaggerFiles "github.com/swaggo/files" + ginSwagger "github.com/swaggo/gin-swagger" "github.com/utmstack/UTMStack/correlation/api" "github.com/utmstack/UTMStack/correlation/cache" "github.com/utmstack/UTMStack/correlation/correlation" @@ -15,28 +19,23 @@ import ( "github.com/utmstack/UTMStack/correlation/sqldb" "github.com/utmstack/UTMStack/correlation/statistics" "github.com/utmstack/UTMStack/correlation/utils" - "github.com/gin-gonic/gin" - swaggerFiles "github.com/swaggo/files" - ginSwagger "github.com/swaggo/gin-swagger" ) // @title UTMStack's Correlation Engine // @version 1.0 -// @description Rules based correlation engine for UTMStack. -// @contact.name Osmany Montero -// @contact.email osmany@quantfall.com -// @license.name Private +// @description Rules-based correlation engine for UTMStack. +// @contact.name UTMStack LLC +// @contact.email contact@utmstack.com +// @license.name AGPLv3 // @host localhost:8080 // @BasePath /v1 func main() { sqldb.Connect() + geo.Load() + ti.Load() ready := make(chan bool, 1) - - go geo.Update(ready) - <-ready - go rules.Update(ready) <-ready @@ -51,11 +50,11 @@ func main() { go cache.ProcessQueue() go search.ProcessQueue() go statistics.Update() + go ti.IsBlocklisted() go func() { gin.SetMode(gin.ReleaseMode) - //r := gin.Default() r := gin.New() r.Use(gin.Recovery()) r.Use(gin.ErrorLogger()) @@ -66,11 +65,14 @@ func main() { docURL := ginSwagger.URL("/swagger/doc.json") r.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, docURL)) - r.Run() + err := r.Run() + if err != nil { + panic(err) + } }() signals := make(chan os.Signal, 1) - go rules.RulesChanges(signals) + go rules.Changes(signals) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) <-signals } diff --git a/correlation/rules/rules.go b/correlation/rules/rules.go index 42cf19ad7..8ed39a382 100644 --- a/correlation/rules/rules.go +++ b/correlation/rules/rules.go @@ -31,11 +31,6 @@ func ListRulesFiles() []string { return files } -type SavedField struct { - Field string `yaml:"field"` - Alias string `yaml:"alias"` -} - type AllOf struct { Field string `yaml:"field"` Operator string `yaml:"operator"` @@ -49,17 +44,17 @@ type OneOf struct { } type Cache struct { - AllOf []AllOf `yaml:"allOf"` - OneOf []OneOf `yaml:"oneOf"` - TimeLapse int64 `yaml:"timeLapse"` - MinCount int `yaml:"minCount"` - Save []SavedField `yaml:"save"` + AllOf []AllOf `yaml:"allOf"` + OneOf []OneOf `yaml:"oneOf"` + TimeLapse int64 `yaml:"timeLapse"` + MinCount int `yaml:"minCount"` + Save []utils.SavedField `yaml:"save"` } type Search struct { - Query string `yaml:"query"` - MinCount int `yaml:"minCount"` - Save []SavedField `yaml:"save"` + Query string `yaml:"query"` + MinCount int `yaml:"minCount"` + Save []utils.SavedField `yaml:"save"` } type Rule struct { @@ -85,15 +80,15 @@ func GetRules() []Rule { utils.ReadYaml(file, &tmpRules) log.Printf("%v rule/s found", len(tmpRules)) for _, tr := range tmpRules { - new := true + n := true for _, r := range rules { if r.Name == tr.Name { - new = false + n = false log.Printf("Ignoring rule: '%s' from: %s", r.Name, file) break } } - if new { + if n { rules = append(rules, tr) } } @@ -102,7 +97,7 @@ func GetRules() []Rule { return rules } -func RulesChanges(signals chan os.Signal) { +func Changes(signals chan os.Signal) { cnf := utils.GetConfig() watcher, err := fsnotify.NewWatcher() if err != nil { @@ -141,15 +136,15 @@ func RulesChanges(signals chan os.Signal) { if err != nil { log.Printf("Could not list rules folders: %v", err) } - new := true + n := true if info.IsDir() { for _, folder := range folders { if path == folder { - new = false + n = false break } } - if new { + if n { folders = append(folders, path) if err := watcher.Add(path); err != nil { log.Printf("Could not start watcher for a rules folder: %v", err) diff --git a/correlation/search/queue.go b/correlation/search/queue.go index f2fd3f01d..e7a952e09 100644 --- a/correlation/search/queue.go +++ b/correlation/search/queue.go @@ -36,7 +36,7 @@ func ProcessQueue() { tmp := nd nd = "" ndMutex.Unlock() - + body, err := utils.DoPost(url, "application/x-ndjson", strings.NewReader(tmp)) if err != nil { log.Fatalf("Could not send logs to Elasticsearch: %v. %s", err, body) @@ -48,7 +48,7 @@ func ProcessQueue() { for { l := <-logs - var cl *bytes.Buffer = new(bytes.Buffer) + var cl = new(bytes.Buffer) dataType := gjson.Get(l, "dataType").String() dataSource := gjson.Get(l, "dataSource").String() @@ -72,4 +72,3 @@ func ProcessQueue() { }() } } - diff --git a/correlation/search/search.go b/correlation/search/search.go index 322073c30..a179561aa 100644 --- a/correlation/search/search.go +++ b/correlation/search/search.go @@ -5,8 +5,8 @@ import ( "log" "strings" - "github.com/utmstack/UTMStack/correlation/utils" "github.com/tidwall/gjson" + "github.com/utmstack/UTMStack/correlation/utils" ) func Search(query string) []string { @@ -19,8 +19,8 @@ func Search(query string) []string { } else { hits := gjson.Get(string(cnn), "hits.hits").Array() for _, hit := range hits { - log := gjson.Get(hit.String(), "_source") - result = append(result, log.String()) + l := gjson.Get(hit.String(), "_source") + result = append(result, l.String()) } } return result diff --git a/correlation/sqldb/cnn.go b/correlation/sqldb/cnn.go index cf4d91ac9..365569b2c 100644 --- a/correlation/sqldb/cnn.go +++ b/correlation/sqldb/cnn.go @@ -5,8 +5,8 @@ import ( "fmt" "log" - "github.com/utmstack/UTMStack/correlation/utils" _ "github.com/lib/pq" + "github.com/utmstack/UTMStack/correlation/utils" ) var db *sql.DB @@ -14,18 +14,18 @@ var err error func Connect() { cnf := utils.GetConfig() - log.Printf("Connecting to PostgreSQL server: %s using port: %v", cnf.PostgreSQL.Server, cnf.PostgreSQL.Port) + log.Printf("Connecting to Postgres server: %s using port: %v", cnf.Postgres.Server, cnf.Postgres.Port) dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%v sslmode=disable", - cnf.PostgreSQL.Server, - cnf.PostgreSQL.User, - cnf.PostgreSQL.Password, - cnf.PostgreSQL.Database, - cnf.PostgreSQL.Port, + cnf.Postgres.Server, + cnf.Postgres.User, + cnf.Postgres.Password, + cnf.Postgres.Database, + cnf.Postgres.Port, ) db, err = sql.Open("postgres", dsn) if err != nil { - log.Fatalf("Could not connect to PostgreSQL: %v", err) + log.Fatalf("Could not connect to Postgres: %v", err) } ping() @@ -33,7 +33,6 @@ func Connect() { func ping() { if err := db.Ping(); err != nil { - log.Fatalf("Could not reconnect to PostgreSQL: %v", err) + log.Fatalf("Could not reconnect to Postgres: %v", err) } } - diff --git a/correlation/sqldb/crud.go b/correlation/sqldb/crud.go index 7b79bc553..c8001b3c6 100644 --- a/correlation/sqldb/crud.go +++ b/correlation/sqldb/crud.go @@ -5,36 +5,6 @@ import ( "time" ) -func GetStatus() ([]map[string]interface{}, error) { - var dataSourceStatus []map[string]interface{} - var err error - - ping() - - rows, err := db.Query(`SELECT source, data_type, timestamp, median FROM utm_data_input_status`) - - if err != nil { - log.Printf("Error getting status from utm_data_input_status: %v", err) - } else { - for rows.Next() { - var ( - source, dataType string - timestamp, median int64 - ) - rows.Scan(&source, &dataType, ×tamp, &median) - dataSourceStatus = append(dataSourceStatus, - map[string]interface{}{ - "dataSource": source, - "dataType": dataType, - "timestamp": timestamp, - "median": median, - }) - } - } - - return dataSourceStatus, err -} - func UpdateStatistics(i, s, t string, c int64) { ping() diff --git a/correlation/ti/bases.go b/correlation/ti/bases.go new file mode 100644 index 000000000..297ef3ef9 --- /dev/null +++ b/correlation/ti/bases.go @@ -0,0 +1,52 @@ +package ti + +import ( + "bufio" + "log" + "os" + "path/filepath" +) + +func Load() { + log.Printf("Loading Threat Intelligence Feeds") + + var files = []string{ + "ip_blocklist.list", + "domain_blocklist.list", + "hostname_blocklist.list", + } + + for _, file := range files { + var t string + + switch file { + case "ip_blocklist.list": + t = "IP" + case "domain_blocklist.list": + t = "domain" + case "hostname_blocklist.list": + t = "hostname" + } + + f, err := os.Open(filepath.Join("/app", file)) + if err != nil { + log.Printf("Could not open file: %v", err) + continue + } + + scanner := bufio.NewScanner(f) + + for scanner.Scan() { + element := scanner.Text() + if element == "" { + continue + } + + blockList[element] = t + } + + _ = f.Close() + } + + log.Printf("Threat Intelligence feeds loaded") +} diff --git a/correlation/ti/ti.go b/correlation/ti/ti.go new file mode 100644 index 000000000..6b82caa00 --- /dev/null +++ b/correlation/ti/ti.go @@ -0,0 +1,121 @@ +package ti + +import ( + "fmt" + "github.com/tidwall/gjson" + "github.com/utmstack/UTMStack/correlation/correlation" + "github.com/utmstack/UTMStack/correlation/utils" + "runtime" + "strings" +) + +var blockList map[string]string +var channel chan string + +func init() { + blockList = make(map[string]string, 10000) + channel = make(chan string, 10000) +} + +func IsBlocklisted() { + saveFields := []utils.SavedField{ + { + Field: "logx.*.proto", + Alias: "Protocol", + }, + { + Field: "logx.*.src_ip", + Alias: "SourceIP", + }, + { + Field: "logx.*.dest_ip", + Alias: "DestinationIP", + }, + { + Field: "logx.*.src_port", + Alias: "SourcePort", + }, + { + Field: "logx.*.dest_port", + Alias: "DestinationPort", + }, + } + + numCPU := runtime.NumCPU() * 2 + for i := 0; i < numCPU; i++ { + go func() { + for { + log := <-channel + + for key, value := range blockList { + var stop bool + + switch value { + case "IP": + sourceIp := gjson.Get(log, "logx.*.src_ip") + destinationIp := gjson.Get(log, "logx.*.dest_ip") + + if sourceIp.String() == key { + correlation.Alert( + "Connection attempt from a malicious IP", + "Low", + "A blocklisted element has been identified in the logs. Further investigation is recommended.", + "", + "Threat Intelligence", + "", + []string{"https://threatwinds.com"}, + gjson.Get(log, "dataType").String(), + gjson.Get(log, "dataSource").String(), + utils.ExtractDetails(saveFields, log), + ) + + stop = true + } + + if destinationIp.String() == key { + correlation.Alert( + "Connection attempt to a malicious IP", + "High", + "A blocklisted element has been identified in the logs. Further investigation is recommended.", + "", + "Threat Intelligence", + "", + []string{"https://threatwinds.com"}, + gjson.Get(log, "dataType").String(), + gjson.Get(log, "dataSource").String(), + utils.ExtractDetails(saveFields, log), + ) + + stop = true + } + } + + if stop { + break + } + + if strings.Contains(log, key) { + correlation.Alert( + fmt.Sprintf("Maliciuos %s found in log", value), + "Low", + "A blocklisted element has been identified in the logs. Further investigation is recommended.", + "", + "Threat Intelligence", + "", + []string{"https://threatwinds.com"}, + gjson.Get(log, "dataType").String(), + gjson.Get(log, "dataSource").String(), + utils.ExtractDetails(saveFields, log), + ) + + break + } + } + } + }() + } +} + +func Enqueue(log string) { + channel <- log +} diff --git a/correlation/utils/configuration.go b/correlation/utils/configuration.go index 17f5593b6..01967734e 100644 --- a/correlation/utils/configuration.go +++ b/correlation/utils/configuration.go @@ -6,9 +6,8 @@ import ( type Config struct { RulesFolder string `yaml:"rulesFolder"` - GeoIPFolder string `yaml:"geoipFolder"` Elasticsearch string `yaml:"elasticsearch"` - PostgreSQL struct { + Postgres struct { Server string `yaml:"server"` Port string `yaml:"port"` User string `yaml:"user"` diff --git a/correlation/utils/details.go b/correlation/utils/details.go new file mode 100644 index 000000000..8735fa64c --- /dev/null +++ b/correlation/utils/details.go @@ -0,0 +1,62 @@ +package utils + +import ( + "github.com/tidwall/gjson" + "net" +) + +type SavedField struct { + Field string `yaml:"field"` + Alias string `yaml:"alias"` +} + +func ExtractDetails(save []SavedField, l string) map[string]string { + var fields = map[string]string{ + "id": gjson.Get(l, "id").String(), + } + + //User saved fields + for _, save := range save { + saveValue := gjson.Get(l, save.Field).String() + if saveValue == "" { + continue + } + + fields[save.Alias] = saveValue + } + + _, srcHostOk := fields["SourceHost"] + _, dstHostOk := fields["DestinationHost"] + _, srcIPOk := fields["SourceIP"] + _, dstIPOk := fields["DestinationIP"] + + // Try to resolve SourceHost if SourceIP exists but not SourceHost + if !srcHostOk && srcIPOk { + host, _ := net.LookupHost(fields["SourceIP"]) + fields["SourceHost"] = host[0] + } + + // Try to resolve DestinationHost if DestinationIP exists but not DestinationHost + if !dstHostOk && dstIPOk { + host, _ := net.LookupHost(fields["DestinationIP"]) + fields["DestinationHost"] = host[0] + } + + // Try to resolve SourceIP if SourceHost exists but not SourceIP + if srcHostOk && !srcIPOk { + ip, _ := net.LookupIP(fields["SourceHost"]) + if len(ip) != 0 && ip[0].String() != "" { + fields["SourceIP"] = ip[0].String() + } + } + + // Try to resolve DestinationIP if DestinationHost exists but not DestinationIP + if dstHostOk && !dstIPOk { + ip, _ := net.LookupIP(fields["DestinationHost"]) + if len(ip) != 0 && ip[0].String() != "" { + fields["DestinationIP"] = ip[0].String() + } + } + + return fields +} diff --git a/correlation/utils/httpClient.go b/correlation/utils/httpClient.go index 31a84c6ba..2ca460e3e 100644 --- a/correlation/utils/httpClient.go +++ b/correlation/utils/httpClient.go @@ -2,38 +2,10 @@ package utils import ( "io" - "io/ioutil" "log" "net/http" - "os" ) -func Download(url, file string) error { - out, err := os.Create(file) - if err != nil { - log.Printf("Could not create file: %v", err) - return err - } - - defer out.Close() - - resp, err := http.Get(url) - if err != nil { - log.Printf("Could not do request to the URL: %v", err) - return err - } - - defer resp.Body.Close() - - n, err := io.Copy(out, resp.Body) - if err != nil { - log.Printf("Could not save data to file: %v", err) - return err - } - log.Printf("Downloaded %d bytes from %s", n, url) - return nil -} - func DoPost(url, contentType string, body io.Reader) ([]byte, error) { res, err := http.Post(url, contentType, body) if err != nil { @@ -43,7 +15,7 @@ func DoPost(url, contentType string, body io.Reader) ([]byte, error) { defer res.Body.Close() - response, err := ioutil.ReadAll(res.Body) + response, err := io.ReadAll(res.Body) if err != nil { log.Printf("Could not read response: %v", err) return []byte{}, err From 26626bbdeb4958ea342addf6e42a20018ff7bb4e Mon Sep 17 00:00:00 2001 From: Osmany Montero Date: Wed, 19 Feb 2025 11:13:14 +0200 Subject: [PATCH 4/6] Replace PrepareSystem with PrepareKernel and remove GeoIP persistent storage. Updated function calls in cloud.go and master.go to use PrepareKernel for better clarity and consistency. Removed GeoIP-related configuration and volumes as they are no longer required in the application setup. Cleaned up related references in config and compose configurations. --- correlation/config.yml.prod | 1 - installer/cloud.go | 2 +- installer/master.go | 2 +- installer/types/compose.go | 5 ----- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/correlation/config.yml.prod b/correlation/config.yml.prod index e79a3c6f4..c90520031 100644 --- a/correlation/config.yml.prod +++ b/correlation/config.yml.prod @@ -1,5 +1,4 @@ rulesFolder: /app/rulesets/ -geoipFolder: /app/geosets/ elasticsearch: "http://ELASTICSEARCH_HOST:ELASTICSEARCH_PORT" postgresql: server: POSTGRESQL_HOST diff --git a/installer/cloud.go b/installer/cloud.go index 6deb2c80d..d357b6763 100644 --- a/installer/cloud.go +++ b/installer/cloud.go @@ -55,7 +55,7 @@ func Cloud(c *types.Config, update bool) error { if utils.GetLock(202402081552, stack.LocksDir) { fmt.Println("Preparing kernel to run UTMStack") - if err := PrepareSystem(); err != nil { + if err := PrepareKernel(); err != nil { return err } diff --git a/installer/master.go b/installer/master.go index 13b0827f4..5b08f5ee8 100644 --- a/installer/master.go +++ b/installer/master.go @@ -55,7 +55,7 @@ func Master(c *types.Config) error { if utils.GetLock(202402081552, stack.LocksDir) { fmt.Println("Preparing kernel to run UTMStack") - if err := PrepareSystem(); err != nil { + if err := PrepareKernel(); err != nil { return err } diff --git a/installer/types/compose.go b/installer/types/compose.go index 788bae85c..a425f2cfd 100644 --- a/installer/types/compose.go +++ b/installer/types/compose.go @@ -431,7 +431,6 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { }, Volumes: []string{ stack.Rules + ":/app/rulesets", - "geoip_data:/app/geosets", }, Environment: []string{ "SERVER_NAME=" + conf.ServerName, @@ -615,10 +614,6 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { "external": false, } - c.Volumes["geoip_data"] = Volume{ - "external": false, - } - c.Volumes["agent_manager"] = Volume{ "external": false, } From ab9e6e6dcfd93ae66e1c2956a40f04d62e28f662 Mon Sep 17 00:00:00 2001 From: Jose Angel Sanchez Velazquez Date: Wed, 19 Feb 2025 10:14:28 +0100 Subject: [PATCH 5/6] fixed operator evaluation not passing tests --- correlation/cache/operators.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/correlation/cache/operators.go b/correlation/cache/operators.go index 5f51d3853..b8f0c17f5 100644 --- a/correlation/cache/operators.go +++ b/correlation/cache/operators.go @@ -12,16 +12,14 @@ import ( func inCIDR(addr, network string) (bool, error) { _, subnet, err := net.ParseCIDR(network) - if err == nil { - ip := net.ParseIP(addr) - if ip != nil { - if subnet.Contains(ip) { - return true, nil - } - } + if err != nil { + return false, err + } + ip := net.ParseIP(addr) + if ip == nil { return false, fmt.Errorf("invalid IP address") } - return false, err + return subnet.Contains(ip), nil } func equal(val1, val2 string) bool { @@ -56,12 +54,10 @@ func endWith(str, suff string) bool { func expresion(exp, str string) (bool, error) { re, err := regexp.Compile(exp) - if err == nil { - if re.MatchString(str) { - return true, nil - } + if err != nil { + return false, err } - return false, err + return re.MatchString(str), nil } func parseFloats(val1, val2 string) (float64, float64, error) { @@ -115,7 +111,7 @@ func compare(operator, val1, val2 string) bool { if err != nil { return false } - return matched + return !matched case "<": f1, f2, err := parseFloats(val1, val2) if err != nil { @@ -160,8 +156,10 @@ func compare(operator, val1, val2 string) bool { } func evalElement(elem, field, operator, value string) bool { - if gjson.Get(elem, field).Exists() { - return compare(operator, gjson.Get(elem, field).String(), value) + if elem := gjson.Get(elem, field); elem.Exists() { + raw := elem + fmt.Printf("raw: %s\n", raw) + return compare(operator, raw.String(), value) } else if operator == "not exist" { return true } From 4f4db77dc2fae45287ea169634b39be25d7539ae Mon Sep 17 00:00:00 2001 From: Jose Angel Sanchez Velazquez Date: Wed, 19 Feb 2025 10:19:25 +0100 Subject: [PATCH 6/6] remove debug print --- correlation/cache/operators.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/correlation/cache/operators.go b/correlation/cache/operators.go index 46bb92c1a..c0322a14c 100644 --- a/correlation/cache/operators.go +++ b/correlation/cache/operators.go @@ -159,9 +159,7 @@ func compare(operator, val1, val2 string) bool { func evalElement(elem, field, operator, value string) bool { if elem := gjson.Get(elem, field); elem.Exists() { - raw := elem - fmt.Printf("raw: %s\n", raw) - return compare(operator, raw.String(), value) + return compare(operator, elem.String(), value) } else if operator == "not exist" { return true }