Bugfix/agent/non printable chars

agent/agent/configuration/config.go

@@ -7,9 +7,14 @@ import (
 	"sync"
 
 	aesCrypt "github.com/AtlasInsideCorp/AtlasInsideAES"
+	"github.com/google/uuid"
 	"github.com/utmstack/UTMStack/agent/agent/utils"
 )
 
+type InstallationUUID struct {
+	UUID string `yaml:"uuid"`
+}
+
 type Config struct {
 	Server             string `yaml:"server"`
 	AgentID            uint   `yaml:"agent-id"`
@@ -31,30 +36,49 @@ func GetInitialConfig() (*Config, string) {
 }
 
 var (
-	cnf      Config
-	confOnce sync.Once
+	cnf          = Config{}
+	confOnce     sync.Once
+	instuuid     = ""
+	instuuidOnce sync.Once
 )
 
 func GetCurrentConfig() (*Config, error) {
-	cnf = Config{}
 	var errR error
 	confOnce.Do(func() {
 		path, err := utils.GetMyPath()
 		if err != nil {
 			errR = fmt.Errorf("failed to get current path: %v", err)
 			return
 		}
+
+		uuidExists := utils.CheckIfPathExist(filepath.Join(path, UUIDFileName))
+
 		var encryptConfig Config
 		if err = utils.ReadYAML(filepath.Join(path, "config.yml"), &encryptConfig); err != nil {
 			errR = fmt.Errorf("error reading config file: %v", err)
 			return
 		}
 
 		// Get key
-		key, err := utils.GenerateKey(REPLACE_KEY)
-		if err != nil {
-			errR = fmt.Errorf("error geneating key: %v", err)
-			return
+		var key []byte
+		if uuidExists {
+			uuid, err := GetUUID()
+			if err != nil {
+				errR = fmt.Errorf("failed to get uuid: %v", err)
+				return
+			}
+
+			key, err = utils.GenerateKeyByUUID(REPLACE_KEY, uuid)
+			if err != nil {
+				errR = fmt.Errorf("error geneating key: %v", err)
+				return
+			}
+		} else {
+			key, err = utils.GenerateKey(REPLACE_KEY)
+			if err != nil {
+				errR = fmt.Errorf("error geneating key: %v", err)
+				return
+			}
 		}
 
 		// Decrypt config
@@ -69,6 +93,12 @@ func GetCurrentConfig() (*Config, error) {
 		cnf.AgentKey = agentKey
 		cnf.SkipCertValidation = encryptConfig.SkipCertValidation
 
+		if !uuidExists {
+			if err := SaveConfig(&cnf); err != nil {
+				errR = fmt.Errorf("error writing config file: %v", err)
+				return
+			}
+		}
 	})
 	if errR != nil {
 		return nil, errR
@@ -83,8 +113,13 @@ func SaveConfig(cnf *Config) error {
 		return fmt.Errorf("failed to get current path: %v", err)
 	}
 
+	uuid, err := GenerateNewUUID()
+	if err != nil {
+		return fmt.Errorf("failed to generate uuid: %v", err)
+	}
+
 	// Get key
-	key, err := utils.GenerateKey(REPLACE_KEY)
+	key, err := utils.GenerateKeyByUUID(REPLACE_KEY, uuid)
 	if err != nil {
 		return fmt.Errorf("error geneating key: %v", err)
 	}
@@ -108,3 +143,50 @@ func SaveConfig(cnf *Config) error {
 	}
 	return nil
 }
+
+func GenerateNewUUID() (string, error) {
+	uuid, err := uuid.NewRandom()
+	if err != nil {
+		return "", fmt.Errorf("failed to generate uuid: %v", err)
+	}
+
+	InstallationUUID := InstallationUUID{
+		UUID: uuid.String(),
+	}
+
+	path, err := utils.GetMyPath()
+	if err != nil {
+		return "", fmt.Errorf("failed to get current path: %v", err)
+	}
+
+	if err = utils.WriteYAML(filepath.Join(path, UUIDFileName), InstallationUUID); err != nil {
+		return "", fmt.Errorf("error writing uuid file: %v", err)
+	}
+
+	return InstallationUUID.UUID, nil
+}
+
+func GetUUID() (string, error) {
+	var errR error
+	instuuidOnce.Do(func() {
+		path, err := utils.GetMyPath()
+		if err != nil {
+			errR = fmt.Errorf("failed to get current path: %v", err)
+			return
+		}
+
+		var uuid = InstallationUUID{}
+		if err = utils.ReadYAML(filepath.Join(path, UUIDFileName), &uuid); err != nil {
+			errR = fmt.Errorf("error reading uuid file: %v", err)
+			return
+		}
+
+		instuuid = uuid.UUID
+	})
+
+	if errR != nil {
+		return "", errR
+	}
+
+	return instuuid, nil
+}

agent/agent/configuration/const.go

@@ -26,6 +26,7 @@ const (
 	WinLockName     = "utmstack_windows_collector.lock"
 	RedlineLockName = "utmstack_redline.lock"
 	RedlineServName = "UTMStackRedline"
+	UUIDFileName    = "uuid.yml"
 )
 
 type LogType string

agent/agent/go.mod

@@ -5,6 +5,7 @@ go 1.21.1
 require (
 	github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0
 	github.com/elastic/go-sysinfo v1.11.1
+	github.com/google/uuid v1.3.1
 	github.com/kardianos/service v1.2.2
 	github.com/quantfall/holmes v1.3.0
 	google.golang.org/grpc v1.59.0

agent/agent/go.sum

@@ -25,6 +25,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=

agent/agent/logservice/processor.go

@@ -1,6 +1,7 @@
 package logservice
 
 import (
+	"bufio"
 	context "context"
 	"fmt"
 	"os"
@@ -24,9 +25,11 @@ type LogPipe struct {
 }
 
 var (
-	processor     LogProcessor
-	processorOnce sync.Once
-	LogQueue      = make(chan LogPipe, 1000)
+	processor                   LogProcessor
+	processorOnce               sync.Once
+	LogQueue                    = make(chan LogPipe, 1000)
+	MinutesForCleanLog          = 10080 // 7 days in minutes(7*24*60)
+	MinutesForReportLogsCounted = time.Duration(5 * time.Minute)
 )
 
 func GetLogProcessor() LogProcessor {
@@ -41,17 +44,12 @@ func (l *LogProcessor) ProcessLogs(client LogServiceClient, ctx context.Context,
 	reconnectDelay := configuration.InitialReconnectDelay
 	invalidKeyCounter := 0
 
-	path, err := utils.GetMyPath()
-	if err != nil {
-		h.FatalError("Failed to get current path: %v", err)
-	}
-
-	filePath := filepath.Join(path, "logs_process")
-	utils.CreatePathIfNotExist(filePath)
-	fileNames := map[string]*os.File{}
-	defer func() {
-		for _, file := range fileNames {
-			file.Close()
+	logsProcessCounter := map[string]int{}
+	go func() {
+		for {
+			time.Sleep(MinutesForReportLogsCounted)
+			SaveCountedLogs(h, logsProcessCounter)
+			logsProcessCounter = map[string]int{}
 		}
 	}()
 
@@ -104,38 +102,8 @@ func (l *LogProcessor) ProcessLogs(client LogServiceClient, ctx context.Context,
 			continue
 		}
 
+		logsProcessCounter[newLog.Src] += len(newLog.Logs)
 		invalidKeyCounter = 0
-
-		fileIsOpen := false
-		for name := range fileNames {
-			if name == filepath.Join(filePath, string(newLog.Src)+".txt") {
-				fileIsOpen = true
-			}
-		}
-
-		newFileName := filepath.Join(filePath, string(newLog.Src)+".txt")
-		if !fileIsOpen {
-			file, err := os.OpenFile(newFileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-			if err != nil {
-				h.Error("error opening file: %s", err)
-				time.Sleep(reconnectDelay)
-				connectionTime = utils.IncrementReconnectTime(connectionTime, reconnectDelay, configuration.MaxConnectionTime)
-				reconnectDelay = utils.IncrementReconnectDelay(reconnectDelay, configuration.MaxReconnectDelay)
-				continue
-			}
-			fileNames[newFileName] = file
-		}
-
-		for _, mylog := range newLog.Logs {
-			_, err = fileNames[newFileName].WriteString(fmt.Sprintf("%s\n", mylog))
-			if err != nil {
-				h.Info("error writing to file: %s\n", err)
-				time.Sleep(reconnectDelay)
-				connectionTime = utils.IncrementReconnectTime(connectionTime, reconnectDelay, configuration.MaxConnectionTime)
-				reconnectDelay = utils.IncrementReconnectDelay(reconnectDelay, configuration.MaxReconnectDelay)
-				continue
-			}
-		}
 	}
 }
 
@@ -154,3 +122,61 @@ func (l *LogProcessor) ProcessLogsWithHighPriority(msg string, client LogService
 	}
 	return nil
 }
+
+func SaveCountedLogs(h *holmes.Logger, logsProcessCounter map[string]int) {
+	path, err := utils.GetMyPath()
+	if err != nil {
+		h.FatalError("Failed to get current path: %v", err)
+	}
+
+	filePath := filepath.Join(path, "logs_process")
+	logFile := filepath.Join(filePath, "processed_logs.txt")
+	utils.CreatePathIfNotExist(filePath)
+
+	file, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		h.Error("error opening processed_logs.txt file: %s", err)
+		return
+	}
+	defer file.Close()
+
+	var firstLogTime time.Time
+	var firstLine string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		firstLine = scanner.Text()
+		break
+	}
+
+	if firstLine != "" {
+		firstLogTime, err = time.Parse("2006/01/02 15:04:05.9999999 -0700 MST", strings.Split(firstLine, " - ")[0])
+		if err != nil {
+			h.Error("error parsing first log time: %s", err)
+			return
+		}
+
+		if !firstLogTime.IsZero() && time.Since(firstLogTime).Minutes() >= float64(MinutesForCleanLog) {
+			file.Close()
+			if err := os.Remove(logFile); err != nil {
+				h.Error("error removing processed_logs.txt file: %s", err)
+				return
+			}
+			file, err = os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+			if err != nil {
+				h.Error("error opening processed_logs.txt file: %s", err)
+				return
+			}
+		}
+	}
+
+	for name, counter := range logsProcessCounter {
+		if counter > 0 {
+			_, err = file.WriteString(fmt.Sprintf("%v - %d logs from %s have been processed\n", time.Now().Format("2006/01/02 15:04:05.9999999 -0700 MST"), counter, name))
+			if err != nil {
+				h.Error("error writing to processed_logs.txt file: %s", err)
+				continue
+			}
+		}
+	}
+
+}

agent/agent/redline/redline.go

@@ -26,7 +26,7 @@ func CheckRedlineService(h *holmes.Logger) {
 		if attempts >= 3 {
 			h.Info("Redline service has been stopped")
 			if err := utils.Execute(filepath.Join(path, bin), path, "send-log", fmt.Sprintf("%s service has been stopped", configuration.RedlineServName)); err != nil {
-				h.Error("error checking %s: error sending log : %v", err)
+				h.Error("error checking %s: error sending log : %v", configuration.RedlineServName, err)
 				time.Sleep(time.Second * 5)
 				continue
 			}

agent/agent/utils/crypt.go

@@ -15,3 +15,9 @@ func GenerateKey(REPALCE_KEY string) ([]byte, error) {
 	base64Key := base64.StdEncoding.EncodeToString(data)
 	return []byte(REPALCE_KEY + base64Key), nil
 }
+
+func GenerateKeyByUUID(REPLACE_KEY string, uuid string) ([]byte, error) {
+	data := []byte(REPLACE_KEY + uuid)
+	base64Key := base64.StdEncoding.EncodeToString(data)
+	return []byte(base64Key), nil
+}

agent/redline/protector/protector.go

@@ -26,7 +26,7 @@ func ProtectService(servName, lockName string, h *holmes.Logger) {
 		if attempts >= 3 {
 			h.Info("%s service has been stopped", servName)
 			if err := utils.Execute(filepath.Join(path, bin), path, "send-log", fmt.Sprintf("%s service has been stopped", servName)); err != nil {
-				h.Error("error checking %s: error sending log : %v", err)
+				h.Error("error checking %s: error sending log : %v", servName, err)
 				time.Sleep(time.Second * 5)
 				continue
 			}

agent/versions.json

@@ -5,6 +5,12 @@
             "agent_version": "10.1.1",
             "updater_version": "10.1.2",
             "redline_version": "10.1.1"
+        },
+        {
+            "master_version": "10.2.0",
+            "agent_version": "10.2.0",
+            "updater_version": "10.1.2",
+            "redline_version": "10.1.1"
         }
     ]
 }

frontend/src/app/app-module/guides/guide-linux-agent/guide-linux-agent.component.ts

@@ -49,8 +49,13 @@ export class GuideLinuxAgentComponent implements OnInit {
       `/opt/utmstack-linux-agent/utmstack_agent_installer && sudo /opt/utmstack-linux-agent/utmstack_agent_installer install ` + ip  + ` <secret>` + this.token + `</secret> yes`;
   }
   getUninstallCommand(): string {
-    return `sudo /opt/utmstack-linux-agent/utmstack_agent_installer uninstall && echo "Removing UTMStack Agent dependencies..." ` +
-      `&& sleep 10 && sudo rm -rf "/opt/utmstack-linux-agent" && echo "UTMStack Agent dependencies removed successfully."`;
+    return `sudo /opt/utmstack-linux-agent/utmstack_agent_installer uninstall || true; sudo systemctl stop UTMStackAgent || true; ` +
+    `sudo systemctl disable UTMStackAgent || true; sudo rm /etc/systemd/system/UTMStackAgent.service || true; sudo systemctl stop UTMStackRedline || true; ` +
+    `sudo systemctl disable UTMStackRedline || true; sudo rm /etc/systemd/system/UTMStackRedline.service || true; ` +
+    `sudo systemctl stop UTMStackUpdater || true; sudo systemctl disable UTMStackUpdater || true; ` +
+    `sudo rm /etc/systemd/system/UTMStackUpdater.service || true; sudo systemctl stop UTMStackModulesLogsCollector || true; ` +
+    `sudo systemctl disable UTMStackModulesLogsCollector || true; sudo rm /etc/systemd/system/UTMStackModulesLogsCollector.service || true; ` +
+    `sudo systemctl daemon-reload || true; echo "Removing UTMStack Agent dependencies..." ` +
+    `&& sleep 10 && sudo rm -rf "/opt/utmstack-linux-agent" && echo "UTMStack Agent dependencies removed successfully."`;
   }
-
 }