From 46a3b4ef53aa33660c1814b58fc9b4788e2253f7 Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Thu, 17 Oct 2024 23:18:18 +0200 Subject: [PATCH 1/4] Update username and other metadata from decorators --- logging/process.go | 39 +++++++++++++++++++-------------------- logging/utils.go | 32 +++++++++++++++++++++++++++++++- nodes/ipaddress.go | 4 ---- nodes/names.go | 15 +++++++++++++++ 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/logging/process.go b/logging/process.go index 6ee54392..6f2d50db 100644 --- a/logging/process.go +++ b/logging/process.go @@ -20,34 +20,33 @@ func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipad log.Debug().Msgf("parsing logs for metadata in %s:%s", logType, environment) } // Iterate through received messages to extract metadata - var uuids, hosts, names, users, osqueryusers, hashes, dhashes, osqueryversions []string + var uuid, hostname, localname, username, osqueryuser, confighash, daemonhash, osqueryversion string for _, l := range logs { - uuids = append(uuids, l.HostIdentifier) - hosts = append(hosts, l.Decorations.Hostname) - names = append(names, l.Decorations.LocalHostname) - users = append(users, l.Decorations.Username) - osqueryusers = append(osqueryusers, l.Decorations.OsqueryUser) - hashes = append(hashes, l.Decorations.ConfigHash) - dhashes = append(dhashes, l.Decorations.DaemonHash) - osqueryversions = append(osqueryversions, l.Version) + uuid = metadataVerification(uuid, l.HostIdentifier) + hostname = metadataVerification(hostname, l.Decorations.Hostname) + localname = metadataVerification(localname, l.Decorations.LocalHostname) + username = metadataVerification(username, l.Decorations.Username) + osqueryuser = metadataVerification(osqueryuser, l.Decorations.OsqueryUser) + confighash = metadataVerification(confighash, l.Decorations.ConfigHash) + daemonhash = metadataVerification(daemonhash, l.Decorations.DaemonHash) + osqueryversion = metadataVerification(osqueryversion, l.Decorations.OsqueryVersion) } if debug { - log.Debug().Msgf("metadata and dispatch for %s", uniq(uuids)[0]) + log.Debug().Msgf("metadata and dispatch for %s", uuid) } - // FIXME it only uses the first element from the []string that uniq returns metadata := nodes.NodeMetadata{ - IPAddress: ipaddress, - Username: uniq(users)[0], - OsqueryUser: uniq(osqueryusers)[0], - Hostname: uniq(hosts)[0], - Localname: uniq(names)[0], - ConfigHash: uniq(hashes)[0], - DaemonHash: uniq(dhashes)[0], - OsqueryVersion: uniq(osqueryversions)[0], + IPAddress: metadataNotEmpty(ipaddress), + Username: metadataNotEmpty(username), + OsqueryUser: metadataNotEmpty(osqueryuser), + Hostname: metadataNotEmpty(hostname), + Localname: metadataNotEmpty(localname), + ConfigHash: metadataNotEmpty(confighash), + DaemonHash: metadataNotEmpty(daemonhash), + OsqueryVersion: metadataNotEmpty(osqueryversion), BytesReceived: dataLen, } // Dispatch logs and update metadata - l.DispatchLogs(data, uniq(uuids)[0], logType, environment, metadata, debug) + l.DispatchLogs(data, uuid, logType, environment, metadata, debug) } // ProcessLogQueryResult - Helper to process on-demand query result logs diff --git a/logging/utils.go b/logging/utils.go index a54079ba..9bb5e93e 100644 --- a/logging/utils.go +++ b/logging/utils.go @@ -1,6 +1,16 @@ package logging -import "github.com/jmpsec/osctrl/backend" +import ( + "github.com/jmpsec/osctrl/backend" + "github.com/rs/zerolog/log" +) + +const ( + // NotReturned - Value not returned from agent + NotReturned = "not returned" + // Mismatched - Value mismatched in log entries + Mismatched = "mismatched" +) // Helper to remove duplicates from array of strings func uniq(duplicated []string) []string { @@ -19,3 +29,23 @@ func uniq(duplicated []string) []string { func sameConfigDB(loggerOne, loggerTwo backend.JSONConfigurationDB) bool { return (loggerOne.Host == loggerTwo.Host) && (loggerOne.Port == loggerTwo.Port) && (loggerOne.Name == loggerTwo.Name) } + +// Helper to be used preparing metadata for each decorator +func metadataVerification(dst, src string) string { + if src != dst { + if dst == "" { + return src + } + log.Warn().Msgf("mismatched metadata: %s != %s", dst, src) + return Mismatched + } + return src +} + +// Helper to make sure all metadata values are not empty +func metadataNotEmpty(value string) string { + if value == "" { + return NotReturned + } + return value +} diff --git a/nodes/ipaddress.go b/nodes/ipaddress.go index 645d076e..4aa2be9f 100644 --- a/nodes/ipaddress.go +++ b/nodes/ipaddress.go @@ -2,7 +2,6 @@ package nodes import ( "fmt" - "time" "gorm.io/gorm" ) @@ -37,9 +36,6 @@ func (n *NodeManager) UpdateIPAddress(ipaddress string, node OsqueryNode) error if err := n.IncHistoryIPAddress(node.UUID, ipaddress); err != nil { return fmt.Errorf("incNodeHistoryIPAddress %v", err) } - if err := n.DB.Model(&node).Update("updated_at", time.Now()).Error; err != nil { - return fmt.Errorf("Update %v", err) - } } return nil } diff --git a/nodes/names.go b/nodes/names.go index d124c4c0..ace646f8 100644 --- a/nodes/names.go +++ b/nodes/names.go @@ -94,6 +94,11 @@ func (n *NodeManager) RecordLocalname(localname string, node OsqueryNode) error return fmt.Errorf("newNodeHistoryLocalname %v", err) } } + if localname != node.Localname { + if err := n.DB.Model(&node).Update("localname", localname).Error; err != nil { + return fmt.Errorf("Update node %v", err) + } + } return nil } @@ -116,6 +121,11 @@ func (n *NodeManager) RecordHostname(hostname string, node OsqueryNode) error { return fmt.Errorf("newNodeHistoryHostname %v", err) } } + if hostname != node.Hostname { + if err := n.DB.Model(&node).Update("hostname", hostname).Error; err != nil { + return fmt.Errorf("Update node %v", err) + } + } return nil } @@ -138,6 +148,11 @@ func (n *NodeManager) RecordUsername(username string, node OsqueryNode) error { return fmt.Errorf("newNodeHistoryUsername %v", err) } } + if username != node.Username { + if err := n.DB.Model(&node).Update("username", username).Error; err != nil { + return fmt.Errorf("Update node %v", err) + } + } return nil } From a971a5a5b2952aba1381fc57034d4405c3cf5e36 Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Fri, 18 Oct 2024 00:47:47 +0200 Subject: [PATCH 2/4] Process metadata only on result logs --- logging/process.go | 54 +++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/logging/process.go b/logging/process.go index 6f2d50db..48f11725 100644 --- a/logging/process.go +++ b/logging/process.go @@ -19,31 +19,35 @@ func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipad if debug { log.Debug().Msgf("parsing logs for metadata in %s:%s", logType, environment) } - // Iterate through received messages to extract metadata - var uuid, hostname, localname, username, osqueryuser, confighash, daemonhash, osqueryversion string - for _, l := range logs { - uuid = metadataVerification(uuid, l.HostIdentifier) - hostname = metadataVerification(hostname, l.Decorations.Hostname) - localname = metadataVerification(localname, l.Decorations.LocalHostname) - username = metadataVerification(username, l.Decorations.Username) - osqueryuser = metadataVerification(osqueryuser, l.Decorations.OsqueryUser) - confighash = metadataVerification(confighash, l.Decorations.ConfigHash) - daemonhash = metadataVerification(daemonhash, l.Decorations.DaemonHash) - osqueryversion = metadataVerification(osqueryversion, l.Decorations.OsqueryVersion) - } - if debug { - log.Debug().Msgf("metadata and dispatch for %s", uuid) - } - metadata := nodes.NodeMetadata{ - IPAddress: metadataNotEmpty(ipaddress), - Username: metadataNotEmpty(username), - OsqueryUser: metadataNotEmpty(osqueryuser), - Hostname: metadataNotEmpty(hostname), - Localname: metadataNotEmpty(localname), - ConfigHash: metadataNotEmpty(confighash), - DaemonHash: metadataNotEmpty(daemonhash), - OsqueryVersion: metadataNotEmpty(osqueryversion), - BytesReceived: dataLen, + // Iterate through received messages to extract metadata, if the logs are results + var metadata nodes.NodeMetadata + var uuid string + if logType == types.ResultLog { + var hostname, localname, username, osqueryuser, confighash, daemonhash, osqueryversion string + for _, l := range logs { + uuid = metadataVerification(uuid, l.HostIdentifier) + hostname = metadataVerification(hostname, l.Decorations.Hostname) + localname = metadataVerification(localname, l.Decorations.LocalHostname) + username = metadataVerification(username, l.Decorations.Username) + osqueryuser = metadataVerification(osqueryuser, l.Decorations.OsqueryUser) + confighash = metadataVerification(confighash, l.Decorations.ConfigHash) + daemonhash = metadataVerification(daemonhash, l.Decorations.DaemonHash) + osqueryversion = metadataVerification(osqueryversion, l.Decorations.OsqueryVersion) + } + if debug { + log.Debug().Msgf("metadata and dispatch for %s", uuid) + } + metadata = nodes.NodeMetadata{ + IPAddress: metadataNotEmpty(ipaddress), + Username: metadataNotEmpty(username), + OsqueryUser: metadataNotEmpty(osqueryuser), + Hostname: metadataNotEmpty(hostname), + Localname: metadataNotEmpty(localname), + ConfigHash: metadataNotEmpty(confighash), + DaemonHash: metadataNotEmpty(daemonhash), + OsqueryVersion: metadataNotEmpty(osqueryversion), + BytesReceived: dataLen, + } } // Dispatch logs and update metadata l.DispatchLogs(data, uuid, logType, environment, metadata, debug) From cff277fac9343afab513d199941c50891a5707aa Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Fri, 18 Oct 2024 02:30:31 +0200 Subject: [PATCH 3/4] Optimize the number of db writes to update metadata --- logging/process.go | 54 +++++++++++++++++++++++----------------------- nodes/names.go | 15 ------------- nodes/nodes.go | 46 ++++++++++++++++++++++++++------------- 3 files changed, 58 insertions(+), 57 deletions(-) diff --git a/logging/process.go b/logging/process.go index 48f11725..7a327c59 100644 --- a/logging/process.go +++ b/logging/process.go @@ -19,35 +19,35 @@ func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipad if debug { log.Debug().Msgf("parsing logs for metadata in %s:%s", logType, environment) } - // Iterate through received messages to extract metadata, if the logs are results - var metadata nodes.NodeMetadata - var uuid string - if logType == types.ResultLog { - var hostname, localname, username, osqueryuser, confighash, daemonhash, osqueryversion string - for _, l := range logs { - uuid = metadataVerification(uuid, l.HostIdentifier) - hostname = metadataVerification(hostname, l.Decorations.Hostname) - localname = metadataVerification(localname, l.Decorations.LocalHostname) - username = metadataVerification(username, l.Decorations.Username) - osqueryuser = metadataVerification(osqueryuser, l.Decorations.OsqueryUser) - confighash = metadataVerification(confighash, l.Decorations.ConfigHash) - daemonhash = metadataVerification(daemonhash, l.Decorations.DaemonHash) + // Iterate through received messages to extract metadata + var uuid, hostname, localname, username, osqueryuser, confighash, daemonhash, osqueryversion string + for _, l := range logs { + uuid = metadataVerification(uuid, l.HostIdentifier) + hostname = metadataVerification(hostname, l.Decorations.Hostname) + localname = metadataVerification(localname, l.Decorations.LocalHostname) + username = metadataVerification(username, l.Decorations.Username) + osqueryuser = metadataVerification(osqueryuser, l.Decorations.OsqueryUser) + confighash = metadataVerification(confighash, l.Decorations.ConfigHash) + daemonhash = metadataVerification(daemonhash, l.Decorations.DaemonHash) + if l.Decorations.OsqueryVersion != "" && l.Version != l.Decorations.OsqueryVersion { + log.Warn().Msgf("mismatched osquery version: %s != %s", l.Version, l.Decorations.OsqueryVersion) + } else { osqueryversion = metadataVerification(osqueryversion, l.Decorations.OsqueryVersion) } - if debug { - log.Debug().Msgf("metadata and dispatch for %s", uuid) - } - metadata = nodes.NodeMetadata{ - IPAddress: metadataNotEmpty(ipaddress), - Username: metadataNotEmpty(username), - OsqueryUser: metadataNotEmpty(osqueryuser), - Hostname: metadataNotEmpty(hostname), - Localname: metadataNotEmpty(localname), - ConfigHash: metadataNotEmpty(confighash), - DaemonHash: metadataNotEmpty(daemonhash), - OsqueryVersion: metadataNotEmpty(osqueryversion), - BytesReceived: dataLen, - } + } + if debug { + log.Debug().Msgf("metadata and dispatch for %s", uuid) + } + metadata := nodes.NodeMetadata{ + IPAddress: ipaddress, + Username: username, + OsqueryUser: osqueryuser, + Hostname: hostname, + Localname: localname, + ConfigHash: confighash, + DaemonHash: daemonhash, + OsqueryVersion: osqueryversion, + BytesReceived: dataLen, } // Dispatch logs and update metadata l.DispatchLogs(data, uuid, logType, environment, metadata, debug) diff --git a/nodes/names.go b/nodes/names.go index ace646f8..d124c4c0 100644 --- a/nodes/names.go +++ b/nodes/names.go @@ -94,11 +94,6 @@ func (n *NodeManager) RecordLocalname(localname string, node OsqueryNode) error return fmt.Errorf("newNodeHistoryLocalname %v", err) } } - if localname != node.Localname { - if err := n.DB.Model(&node).Update("localname", localname).Error; err != nil { - return fmt.Errorf("Update node %v", err) - } - } return nil } @@ -121,11 +116,6 @@ func (n *NodeManager) RecordHostname(hostname string, node OsqueryNode) error { return fmt.Errorf("newNodeHistoryHostname %v", err) } } - if hostname != node.Hostname { - if err := n.DB.Model(&node).Update("hostname", hostname).Error; err != nil { - return fmt.Errorf("Update node %v", err) - } - } return nil } @@ -148,11 +138,6 @@ func (n *NodeManager) RecordUsername(username string, node OsqueryNode) error { return fmt.Errorf("newNodeHistoryUsername %v", err) } } - if username != node.Username { - if err := n.DB.Model(&node).Update("username", username).Error; err != nil { - return fmt.Errorf("Update node %v", err) - } - } return nil } diff --git a/nodes/nodes.go b/nodes/nodes.go index 327cc883..7deccf8c 100644 --- a/nodes/nodes.go +++ b/nodes/nodes.go @@ -338,27 +338,53 @@ func (n *NodeManager) UpdateMetadataByUUID(uuid string, metadata NodeMetadata) e if err != nil { return fmt.Errorf("getNodeByUUID %v", err) } + // Prepare metadata updates + updates := map[string]interface{}{ + "bytes_received": node.BytesReceived + metadata.BytesReceived, + } // Record username if err := n.RecordUsername(metadata.Username, node); err != nil { return fmt.Errorf("RecordUsername %v", err) } + if metadata.Username != node.Username { + updates["username"] =metadata.Username + } // Record hostname if err := n.RecordHostname(metadata.Hostname, node); err != nil { return fmt.Errorf("RecordHostname %v", err) } + if metadata.Hostname != node.Hostname { + updates["hostname"] = metadata.Hostname + } // Record localname if err := n.RecordLocalname(metadata.Localname, node); err != nil { return fmt.Errorf("RecordLocalname %v", err) } + if metadata.Localname != node.Localname { + updates["localname"] = metadata.Localname + } // Record IP address if err := n.RecordIPAddress(metadata.IPAddress, node); err != nil { return fmt.Errorf("RecordIPAddress %v", err) } + if metadata.IPAddress != node.IPAddress { + updates["ip_address"] = metadata.IPAddress + } // Configuration and daemon hash and osquery version update, if different - if (metadata.ConfigHash != node.ConfigHash) || (metadata.DaemonHash != node.DaemonHash) || (metadata.OsqueryVersion != node.OsqueryVersion) || (metadata.OsqueryUser != node.OsqueryUser) { - if err := n.MetadataRefresh(node, metadata); err != nil { - return fmt.Errorf("MetadataRefresh %v", err) - } + if metadata.ConfigHash != node.ConfigHash { + updates["config_hash"] = metadata.ConfigHash + } + if metadata.DaemonHash != node.DaemonHash { + updates["daemon_hash"] = metadata.DaemonHash + } + if metadata.OsqueryVersion != node.OsqueryVersion { + updates["osquery_version"] = metadata.OsqueryVersion + } + if metadata.OsqueryUser != node.OsqueryUser { + updates["osquery_user"] = metadata.OsqueryUser + } + if err := n.MetadataRefresh(node, updates); err != nil { + return fmt.Errorf("MetadataRefresh %v", err) } return nil } @@ -575,17 +601,7 @@ func (n *NodeManager) ConfigRefresh(node OsqueryNode, lastIp string, incBytes in } // MetadataRefresh to perform all needed update operations per node to keep metadata refreshed -func (n *NodeManager) MetadataRefresh(node OsqueryNode, metadata NodeMetadata) error { - updates := map[string]interface{}{ - "config_hash": metadata.ConfigHash, - "daemon_hash": metadata.DaemonHash, - "osquery_version": metadata.OsqueryVersion, - "osquery_user": metadata.OsqueryUser, - "bytes_received": node.BytesReceived + metadata.BytesReceived, - } - if metadata.IPAddress != "" { - updates["ip_address"] = metadata.IPAddress - } +func (n *NodeManager) MetadataRefresh(node OsqueryNode, updates map[string]interface{}) error { if err := n.DB.Model(&node).Updates(updates).Error; err != nil { return fmt.Errorf("Updates %v", err) } From 712ad0a74776936cc1f6c007ee7c88e03567eff4 Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:18:35 +0200 Subject: [PATCH 4/4] Skip update metadata if values are empty' --- nodes/nodes.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nodes/nodes.go b/nodes/nodes.go index 7deccf8c..4bfe99ac 100644 --- a/nodes/nodes.go +++ b/nodes/nodes.go @@ -346,41 +346,41 @@ func (n *NodeManager) UpdateMetadataByUUID(uuid string, metadata NodeMetadata) e if err := n.RecordUsername(metadata.Username, node); err != nil { return fmt.Errorf("RecordUsername %v", err) } - if metadata.Username != node.Username { + if metadata.Username != node.Username && metadata.Username != "" { updates["username"] =metadata.Username } // Record hostname if err := n.RecordHostname(metadata.Hostname, node); err != nil { return fmt.Errorf("RecordHostname %v", err) } - if metadata.Hostname != node.Hostname { + if metadata.Hostname != node.Hostname && metadata.Hostname != "" { updates["hostname"] = metadata.Hostname } // Record localname if err := n.RecordLocalname(metadata.Localname, node); err != nil { return fmt.Errorf("RecordLocalname %v", err) } - if metadata.Localname != node.Localname { + if metadata.Localname != node.Localname && metadata.Localname != "" { updates["localname"] = metadata.Localname } // Record IP address if err := n.RecordIPAddress(metadata.IPAddress, node); err != nil { return fmt.Errorf("RecordIPAddress %v", err) } - if metadata.IPAddress != node.IPAddress { + if metadata.IPAddress != node.IPAddress && metadata.IPAddress != "" { updates["ip_address"] = metadata.IPAddress } // Configuration and daemon hash and osquery version update, if different - if metadata.ConfigHash != node.ConfigHash { + if metadata.ConfigHash != node.ConfigHash && metadata.ConfigHash != "" { updates["config_hash"] = metadata.ConfigHash } - if metadata.DaemonHash != node.DaemonHash { + if metadata.DaemonHash != node.DaemonHash && metadata.DaemonHash != "" { updates["daemon_hash"] = metadata.DaemonHash } - if metadata.OsqueryVersion != node.OsqueryVersion { + if metadata.OsqueryVersion != node.OsqueryVersion && metadata.OsqueryVersion != "" { updates["osquery_version"] = metadata.OsqueryVersion } - if metadata.OsqueryUser != node.OsqueryUser { + if metadata.OsqueryUser != node.OsqueryUser && metadata.OsqueryUser != "" { updates["osquery_user"] = metadata.OsqueryUser } if err := n.MetadataRefresh(node, updates); err != nil {