diff --git a/logging/process.go b/logging/process.go index 6ee54392..7a327c59 100644 --- a/logging/process.go +++ b/logging/process.go @@ -20,34 +20,37 @@ func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipad log.Debug().Msgf("parsing logs for metadata in %s:%s", logType, environment) } // Iterate through received messages to extract metadata - var uuids, hosts, names, users, osqueryusers, hashes, dhashes, osqueryversions []string + var uuid, hostname, localname, username, osqueryuser, confighash, daemonhash, osqueryversion string for _, l := range logs { - uuids = append(uuids, l.HostIdentifier) - hosts = append(hosts, l.Decorations.Hostname) - names = append(names, l.Decorations.LocalHostname) - users = append(users, l.Decorations.Username) - osqueryusers = append(osqueryusers, l.Decorations.OsqueryUser) - hashes = append(hashes, l.Decorations.ConfigHash) - dhashes = append(dhashes, l.Decorations.DaemonHash) - osqueryversions = append(osqueryversions, l.Version) + uuid = metadataVerification(uuid, l.HostIdentifier) + hostname = metadataVerification(hostname, l.Decorations.Hostname) + localname = metadataVerification(localname, l.Decorations.LocalHostname) + username = metadataVerification(username, l.Decorations.Username) + osqueryuser = metadataVerification(osqueryuser, l.Decorations.OsqueryUser) + confighash = metadataVerification(confighash, l.Decorations.ConfigHash) + daemonhash = metadataVerification(daemonhash, l.Decorations.DaemonHash) + if l.Decorations.OsqueryVersion != "" && l.Version != l.Decorations.OsqueryVersion { + log.Warn().Msgf("mismatched osquery version: %s != %s", l.Version, l.Decorations.OsqueryVersion) + } else { + osqueryversion = metadataVerification(osqueryversion, l.Decorations.OsqueryVersion) + } } if debug { - log.Debug().Msgf("metadata and dispatch for %s", uniq(uuids)[0]) + log.Debug().Msgf("metadata and dispatch for %s", uuid) } - // FIXME it only uses the first element from the []string that uniq returns metadata := nodes.NodeMetadata{ IPAddress: ipaddress, - Username: uniq(users)[0], - OsqueryUser: uniq(osqueryusers)[0], - Hostname: uniq(hosts)[0], - Localname: uniq(names)[0], - ConfigHash: uniq(hashes)[0], - DaemonHash: uniq(dhashes)[0], - OsqueryVersion: uniq(osqueryversions)[0], + Username: username, + OsqueryUser: osqueryuser, + Hostname: hostname, + Localname: localname, + ConfigHash: confighash, + DaemonHash: daemonhash, + OsqueryVersion: osqueryversion, BytesReceived: dataLen, } // Dispatch logs and update metadata - l.DispatchLogs(data, uniq(uuids)[0], logType, environment, metadata, debug) + l.DispatchLogs(data, uuid, logType, environment, metadata, debug) } // ProcessLogQueryResult - Helper to process on-demand query result logs diff --git a/logging/utils.go b/logging/utils.go index a54079ba..9bb5e93e 100644 --- a/logging/utils.go +++ b/logging/utils.go @@ -1,6 +1,16 @@ package logging -import "github.com/jmpsec/osctrl/backend" +import ( + "github.com/jmpsec/osctrl/backend" + "github.com/rs/zerolog/log" +) + +const ( + // NotReturned - Value not returned from agent + NotReturned = "not returned" + // Mismatched - Value mismatched in log entries + Mismatched = "mismatched" +) // Helper to remove duplicates from array of strings func uniq(duplicated []string) []string { @@ -19,3 +29,23 @@ func uniq(duplicated []string) []string { func sameConfigDB(loggerOne, loggerTwo backend.JSONConfigurationDB) bool { return (loggerOne.Host == loggerTwo.Host) && (loggerOne.Port == loggerTwo.Port) && (loggerOne.Name == loggerTwo.Name) } + +// Helper to be used preparing metadata for each decorator +func metadataVerification(dst, src string) string { + if src != dst { + if dst == "" { + return src + } + log.Warn().Msgf("mismatched metadata: %s != %s", dst, src) + return Mismatched + } + return src +} + +// Helper to make sure all metadata values are not empty +func metadataNotEmpty(value string) string { + if value == "" { + return NotReturned + } + return value +} diff --git a/nodes/ipaddress.go b/nodes/ipaddress.go index 645d076e..4aa2be9f 100644 --- a/nodes/ipaddress.go +++ b/nodes/ipaddress.go @@ -2,7 +2,6 @@ package nodes import ( "fmt" - "time" "gorm.io/gorm" ) @@ -37,9 +36,6 @@ func (n *NodeManager) UpdateIPAddress(ipaddress string, node OsqueryNode) error if err := n.IncHistoryIPAddress(node.UUID, ipaddress); err != nil { return fmt.Errorf("incNodeHistoryIPAddress %v", err) } - if err := n.DB.Model(&node).Update("updated_at", time.Now()).Error; err != nil { - return fmt.Errorf("Update %v", err) - } } return nil } diff --git a/nodes/nodes.go b/nodes/nodes.go index 327cc883..4bfe99ac 100644 --- a/nodes/nodes.go +++ b/nodes/nodes.go @@ -338,27 +338,53 @@ func (n *NodeManager) UpdateMetadataByUUID(uuid string, metadata NodeMetadata) e if err != nil { return fmt.Errorf("getNodeByUUID %v", err) } + // Prepare metadata updates + updates := map[string]interface{}{ + "bytes_received": node.BytesReceived + metadata.BytesReceived, + } // Record username if err := n.RecordUsername(metadata.Username, node); err != nil { return fmt.Errorf("RecordUsername %v", err) } + if metadata.Username != node.Username && metadata.Username != "" { + updates["username"] =metadata.Username + } // Record hostname if err := n.RecordHostname(metadata.Hostname, node); err != nil { return fmt.Errorf("RecordHostname %v", err) } + if metadata.Hostname != node.Hostname && metadata.Hostname != "" { + updates["hostname"] = metadata.Hostname + } // Record localname if err := n.RecordLocalname(metadata.Localname, node); err != nil { return fmt.Errorf("RecordLocalname %v", err) } + if metadata.Localname != node.Localname && metadata.Localname != "" { + updates["localname"] = metadata.Localname + } // Record IP address if err := n.RecordIPAddress(metadata.IPAddress, node); err != nil { return fmt.Errorf("RecordIPAddress %v", err) } + if metadata.IPAddress != node.IPAddress && metadata.IPAddress != "" { + updates["ip_address"] = metadata.IPAddress + } // Configuration and daemon hash and osquery version update, if different - if (metadata.ConfigHash != node.ConfigHash) || (metadata.DaemonHash != node.DaemonHash) || (metadata.OsqueryVersion != node.OsqueryVersion) || (metadata.OsqueryUser != node.OsqueryUser) { - if err := n.MetadataRefresh(node, metadata); err != nil { - return fmt.Errorf("MetadataRefresh %v", err) - } + if metadata.ConfigHash != node.ConfigHash && metadata.ConfigHash != "" { + updates["config_hash"] = metadata.ConfigHash + } + if metadata.DaemonHash != node.DaemonHash && metadata.DaemonHash != "" { + updates["daemon_hash"] = metadata.DaemonHash + } + if metadata.OsqueryVersion != node.OsqueryVersion && metadata.OsqueryVersion != "" { + updates["osquery_version"] = metadata.OsqueryVersion + } + if metadata.OsqueryUser != node.OsqueryUser && metadata.OsqueryUser != "" { + updates["osquery_user"] = metadata.OsqueryUser + } + if err := n.MetadataRefresh(node, updates); err != nil { + return fmt.Errorf("MetadataRefresh %v", err) } return nil } @@ -575,17 +601,7 @@ func (n *NodeManager) ConfigRefresh(node OsqueryNode, lastIp string, incBytes in } // MetadataRefresh to perform all needed update operations per node to keep metadata refreshed -func (n *NodeManager) MetadataRefresh(node OsqueryNode, metadata NodeMetadata) error { - updates := map[string]interface{}{ - "config_hash": metadata.ConfigHash, - "daemon_hash": metadata.DaemonHash, - "osquery_version": metadata.OsqueryVersion, - "osquery_user": metadata.OsqueryUser, - "bytes_received": node.BytesReceived + metadata.BytesReceived, - } - if metadata.IPAddress != "" { - updates["ip_address"] = metadata.IPAddress - } +func (n *NodeManager) MetadataRefresh(node OsqueryNode, updates map[string]interface{}) error { if err := n.DB.Model(&node).Updates(updates).Error; err != nil { return fmt.Errorf("Updates %v", err) }