diff --git a/agent/agent/configuration/config.go b/agent/agent/configuration/config.go index 97e819895..38ed72fbf 100644 --- a/agent/agent/configuration/config.go +++ b/agent/agent/configuration/config.go @@ -7,9 +7,14 @@ import ( "sync" aesCrypt "github.com/AtlasInsideCorp/AtlasInsideAES" + "github.com/google/uuid" "github.com/utmstack/UTMStack/agent/agent/utils" ) +type InstallationUUID struct { + UUID string `yaml:"uuid"` +} + type Config struct { Server string `yaml:"server"` AgentID uint `yaml:"agent-id"` @@ -31,12 +36,13 @@ func GetInitialConfig() (*Config, string) { } var ( - cnf Config - confOnce sync.Once + cnf = Config{} + confOnce sync.Once + instuuid = "" + instuuidOnce sync.Once ) func GetCurrentConfig() (*Config, error) { - cnf = Config{} var errR error confOnce.Do(func() { path, err := utils.GetMyPath() @@ -44,6 +50,9 @@ func GetCurrentConfig() (*Config, error) { errR = fmt.Errorf("failed to get current path: %v", err) return } + + uuidExists := utils.CheckIfPathExist(filepath.Join(path, UUIDFileName)) + var encryptConfig Config if err = utils.ReadYAML(filepath.Join(path, "config.yml"), &encryptConfig); err != nil { errR = fmt.Errorf("error reading config file: %v", err) @@ -51,10 +60,25 @@ func GetCurrentConfig() (*Config, error) { } // Get key - key, err := utils.GenerateKey(REPLACE_KEY) - if err != nil { - errR = fmt.Errorf("error geneating key: %v", err) - return + var key []byte + if uuidExists { + uuid, err := GetUUID() + if err != nil { + errR = fmt.Errorf("failed to get uuid: %v", err) + return + } + + key, err = utils.GenerateKeyByUUID(REPLACE_KEY, uuid) + if err != nil { + errR = fmt.Errorf("error geneating key: %v", err) + return + } + } else { + key, err = utils.GenerateKey(REPLACE_KEY) + if err != nil { + errR = fmt.Errorf("error geneating key: %v", err) + return + } } // Decrypt config @@ -69,6 +93,12 @@ func GetCurrentConfig() (*Config, error) { cnf.AgentKey = agentKey cnf.SkipCertValidation = encryptConfig.SkipCertValidation + if !uuidExists { + if err := SaveConfig(&cnf); err != nil { + errR = fmt.Errorf("error writing config file: %v", err) + return + } + } }) if errR != nil { return nil, errR @@ -83,8 +113,13 @@ func SaveConfig(cnf *Config) error { return fmt.Errorf("failed to get current path: %v", err) } + uuid, err := GenerateNewUUID() + if err != nil { + return fmt.Errorf("failed to generate uuid: %v", err) + } + // Get key - key, err := utils.GenerateKey(REPLACE_KEY) + key, err := utils.GenerateKeyByUUID(REPLACE_KEY, uuid) if err != nil { return fmt.Errorf("error geneating key: %v", err) } @@ -108,3 +143,50 @@ func SaveConfig(cnf *Config) error { } return nil } + +func GenerateNewUUID() (string, error) { + uuid, err := uuid.NewRandom() + if err != nil { + return "", fmt.Errorf("failed to generate uuid: %v", err) + } + + InstallationUUID := InstallationUUID{ + UUID: uuid.String(), + } + + path, err := utils.GetMyPath() + if err != nil { + return "", fmt.Errorf("failed to get current path: %v", err) + } + + if err = utils.WriteYAML(filepath.Join(path, UUIDFileName), InstallationUUID); err != nil { + return "", fmt.Errorf("error writing uuid file: %v", err) + } + + return InstallationUUID.UUID, nil +} + +func GetUUID() (string, error) { + var errR error + instuuidOnce.Do(func() { + path, err := utils.GetMyPath() + if err != nil { + errR = fmt.Errorf("failed to get current path: %v", err) + return + } + + var uuid = InstallationUUID{} + if err = utils.ReadYAML(filepath.Join(path, UUIDFileName), &uuid); err != nil { + errR = fmt.Errorf("error reading uuid file: %v", err) + return + } + + instuuid = uuid.UUID + }) + + if errR != nil { + return "", errR + } + + return instuuid, nil +} diff --git a/agent/agent/configuration/const.go b/agent/agent/configuration/const.go index 4a8b34044..5037b372b 100644 --- a/agent/agent/configuration/const.go +++ b/agent/agent/configuration/const.go @@ -26,6 +26,7 @@ const ( WinLockName = "utmstack_windows_collector.lock" RedlineLockName = "utmstack_redline.lock" RedlineServName = "UTMStackRedline" + UUIDFileName = "uuid.yml" ) type LogType string diff --git a/agent/agent/go.mod b/agent/agent/go.mod index 704b1028e..348ad7700 100644 --- a/agent/agent/go.mod +++ b/agent/agent/go.mod @@ -5,6 +5,7 @@ go 1.21.1 require ( github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 github.com/elastic/go-sysinfo v1.11.1 + github.com/google/uuid v1.3.1 github.com/kardianos/service v1.2.2 github.com/quantfall/holmes v1.3.0 google.golang.org/grpc v1.59.0 diff --git a/agent/agent/go.sum b/agent/agent/go.sum index 918546360..91aa57b40 100644 --- a/agent/agent/go.sum +++ b/agent/agent/go.sum @@ -25,6 +25,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= diff --git a/agent/agent/logservice/processor.go b/agent/agent/logservice/processor.go index f6831620c..13135ec1c 100644 --- a/agent/agent/logservice/processor.go +++ b/agent/agent/logservice/processor.go @@ -1,6 +1,7 @@ package logservice import ( + "bufio" context "context" "fmt" "os" @@ -24,9 +25,11 @@ type LogPipe struct { } var ( - processor LogProcessor - processorOnce sync.Once - LogQueue = make(chan LogPipe, 1000) + processor LogProcessor + processorOnce sync.Once + LogQueue = make(chan LogPipe, 1000) + MinutesForCleanLog = 10080 // 7 days in minutes(7*24*60) + MinutesForReportLogsCounted = time.Duration(5 * time.Minute) ) func GetLogProcessor() LogProcessor { @@ -41,17 +44,12 @@ func (l *LogProcessor) ProcessLogs(client LogServiceClient, ctx context.Context, reconnectDelay := configuration.InitialReconnectDelay invalidKeyCounter := 0 - path, err := utils.GetMyPath() - if err != nil { - h.FatalError("Failed to get current path: %v", err) - } - - filePath := filepath.Join(path, "logs_process") - utils.CreatePathIfNotExist(filePath) - fileNames := map[string]*os.File{} - defer func() { - for _, file := range fileNames { - file.Close() + logsProcessCounter := map[string]int{} + go func() { + for { + time.Sleep(MinutesForReportLogsCounted) + SaveCountedLogs(h, logsProcessCounter) + logsProcessCounter = map[string]int{} } }() @@ -104,38 +102,8 @@ func (l *LogProcessor) ProcessLogs(client LogServiceClient, ctx context.Context, continue } + logsProcessCounter[newLog.Src] += len(newLog.Logs) invalidKeyCounter = 0 - - fileIsOpen := false - for name := range fileNames { - if name == filepath.Join(filePath, string(newLog.Src)+".txt") { - fileIsOpen = true - } - } - - newFileName := filepath.Join(filePath, string(newLog.Src)+".txt") - if !fileIsOpen { - file, err := os.OpenFile(newFileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - h.Error("error opening file: %s", err) - time.Sleep(reconnectDelay) - connectionTime = utils.IncrementReconnectTime(connectionTime, reconnectDelay, configuration.MaxConnectionTime) - reconnectDelay = utils.IncrementReconnectDelay(reconnectDelay, configuration.MaxReconnectDelay) - continue - } - fileNames[newFileName] = file - } - - for _, mylog := range newLog.Logs { - _, err = fileNames[newFileName].WriteString(fmt.Sprintf("%s\n", mylog)) - if err != nil { - h.Info("error writing to file: %s\n", err) - time.Sleep(reconnectDelay) - connectionTime = utils.IncrementReconnectTime(connectionTime, reconnectDelay, configuration.MaxConnectionTime) - reconnectDelay = utils.IncrementReconnectDelay(reconnectDelay, configuration.MaxReconnectDelay) - continue - } - } } } @@ -154,3 +122,61 @@ func (l *LogProcessor) ProcessLogsWithHighPriority(msg string, client LogService } return nil } + +func SaveCountedLogs(h *holmes.Logger, logsProcessCounter map[string]int) { + path, err := utils.GetMyPath() + if err != nil { + h.FatalError("Failed to get current path: %v", err) + } + + filePath := filepath.Join(path, "logs_process") + logFile := filepath.Join(filePath, "processed_logs.txt") + utils.CreatePathIfNotExist(filePath) + + file, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + h.Error("error opening processed_logs.txt file: %s", err) + return + } + defer file.Close() + + var firstLogTime time.Time + var firstLine string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + firstLine = scanner.Text() + break + } + + if firstLine != "" { + firstLogTime, err = time.Parse("2006/01/02 15:04:05.9999999 -0700 MST", strings.Split(firstLine, " - ")[0]) + if err != nil { + h.Error("error parsing first log time: %s", err) + return + } + + if !firstLogTime.IsZero() && time.Since(firstLogTime).Minutes() >= float64(MinutesForCleanLog) { + file.Close() + if err := os.Remove(logFile); err != nil { + h.Error("error removing processed_logs.txt file: %s", err) + return + } + file, err = os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + h.Error("error opening processed_logs.txt file: %s", err) + return + } + } + } + + for name, counter := range logsProcessCounter { + if counter > 0 { + _, err = file.WriteString(fmt.Sprintf("%v - %d logs from %s have been processed\n", time.Now().Format("2006/01/02 15:04:05.9999999 -0700 MST"), counter, name)) + if err != nil { + h.Error("error writing to processed_logs.txt file: %s", err) + continue + } + } + } + +} diff --git a/agent/agent/redline/redline.go b/agent/agent/redline/redline.go index 6b9ddb1ae..029a3cc2f 100644 --- a/agent/agent/redline/redline.go +++ b/agent/agent/redline/redline.go @@ -26,7 +26,7 @@ func CheckRedlineService(h *holmes.Logger) { if attempts >= 3 { h.Info("Redline service has been stopped") if err := utils.Execute(filepath.Join(path, bin), path, "send-log", fmt.Sprintf("%s service has been stopped", configuration.RedlineServName)); err != nil { - h.Error("error checking %s: error sending log : %v", err) + h.Error("error checking %s: error sending log : %v", configuration.RedlineServName, err) time.Sleep(time.Second * 5) continue } diff --git a/agent/agent/utils/crypt.go b/agent/agent/utils/crypt.go index dcbace7b2..6abb8a712 100644 --- a/agent/agent/utils/crypt.go +++ b/agent/agent/utils/crypt.go @@ -15,3 +15,9 @@ func GenerateKey(REPALCE_KEY string) ([]byte, error) { base64Key := base64.StdEncoding.EncodeToString(data) return []byte(REPALCE_KEY + base64Key), nil } + +func GenerateKeyByUUID(REPLACE_KEY string, uuid string) ([]byte, error) { + data := []byte(REPLACE_KEY + uuid) + base64Key := base64.StdEncoding.EncodeToString(data) + return []byte(base64Key), nil +} diff --git a/agent/redline/protector/protector.go b/agent/redline/protector/protector.go index 16e2f886d..1148b2db1 100644 --- a/agent/redline/protector/protector.go +++ b/agent/redline/protector/protector.go @@ -26,7 +26,7 @@ func ProtectService(servName, lockName string, h *holmes.Logger) { if attempts >= 3 { h.Info("%s service has been stopped", servName) if err := utils.Execute(filepath.Join(path, bin), path, "send-log", fmt.Sprintf("%s service has been stopped", servName)); err != nil { - h.Error("error checking %s: error sending log : %v", err) + h.Error("error checking %s: error sending log : %v", servName, err) time.Sleep(time.Second * 5) continue } diff --git a/agent/versions.json b/agent/versions.json index 4da226325..f09b479c7 100644 --- a/agent/versions.json +++ b/agent/versions.json @@ -5,6 +5,12 @@ "agent_version": "10.1.1", "updater_version": "10.1.2", "redline_version": "10.1.1" + }, + { + "master_version": "10.2.0", + "agent_version": "10.2.0", + "updater_version": "10.1.2", + "redline_version": "10.1.1" } ] } diff --git a/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.ts b/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.ts index c0fc9aa2d..91c73584b 100644 --- a/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.ts +++ b/frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.ts @@ -49,8 +49,13 @@ export class GuideLinuxAgentComponent implements OnInit { `/opt/utmstack-linux-agent/utmstack_agent_installer && sudo /opt/utmstack-linux-agent/utmstack_agent_installer install ` + ip + ` ` + this.token + ` yes`; } getUninstallCommand(): string { - return `sudo /opt/utmstack-linux-agent/utmstack_agent_installer uninstall && echo "Removing UTMStack Agent dependencies..." ` + - `&& sleep 10 && sudo rm -rf "/opt/utmstack-linux-agent" && echo "UTMStack Agent dependencies removed successfully."`; + return `sudo /opt/utmstack-linux-agent/utmstack_agent_installer uninstall || true; sudo systemctl stop UTMStackAgent || true; ` + + `sudo systemctl disable UTMStackAgent || true; sudo rm /etc/systemd/system/UTMStackAgent.service || true; sudo systemctl stop UTMStackRedline || true; ` + + `sudo systemctl disable UTMStackRedline || true; sudo rm /etc/systemd/system/UTMStackRedline.service || true; ` + + `sudo systemctl stop UTMStackUpdater || true; sudo systemctl disable UTMStackUpdater || true; ` + + `sudo rm /etc/systemd/system/UTMStackUpdater.service || true; sudo systemctl stop UTMStackModulesLogsCollector || true; ` + + `sudo systemctl disable UTMStackModulesLogsCollector || true; sudo rm /etc/systemd/system/UTMStackModulesLogsCollector.service || true; ` + + `sudo systemctl daemon-reload || true; echo "Removing UTMStack Agent dependencies..." ` + + `&& sleep 10 && sudo rm -rf "/opt/utmstack-linux-agent" && echo "UTMStack Agent dependencies removed successfully."`; } - }