From 23d31ac17bea06a4a0794dd8047165eb6e3565fe Mon Sep 17 00:00:00 2001 From: lisaSW Date: Sun, 9 Dec 2018 02:50:02 -0700 Subject: [PATCH 1/2] added new fields to hosts --- analysis/beacon/writer.go | 1 + analysis/blacklist/ips.go | 9 +++++++++ analysis/structure/hosts.go | 27 +++++++++++++++++++++++++++ commands/analyze.go | 13 +++++++++---- datatypes/structure/structure.go | 11 +++++++++-- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/analysis/beacon/writer.go b/analysis/beacon/writer.go index 0b5bf6b5..2a1c2bcf 100644 --- a/analysis/beacon/writer.go +++ b/analysis/beacon/writer.go @@ -50,6 +50,7 @@ func (w *writer) start() { //TODO: Implement bulk writes for data := range w.writeChannel { ssn.DB(w.db.GetSelectedDB()).C(w.conf.T.Beacon.BeaconTable).Insert(data) + } w.writeWg.Done() }() diff --git a/analysis/blacklist/ips.go b/analysis/blacklist/ips.go index de10b515..3ee7e505 100644 --- a/analysis/blacklist/ips.go +++ b/analysis/blacklist/ips.go @@ -95,6 +95,7 @@ func buildBlacklistedIPs(ips *mgo.Iter, res *resources.Resources, continue } outputCollection.Insert(&blIP) + } } } @@ -153,6 +154,14 @@ func fillBlacklistedIP(blIP *data.BlacklistedIP, db, uconnCollection string, totalBytes += uconn.TotalBytes totalConnections += uconn.ConnectionCount uniqueConnCount++ + + if source { + ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Dst}, bson.M{"$inc": bson.M{"bl_recv_count": 1}}) + ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Dst}, bson.M{"$inc": bson.M{"bl_recv_avg_bytes": uconn.AverageBytes}}) + } else { + ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Src}, bson.M{"$inc": bson.M{"bl_send_count": 1}}) + ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Src}, bson.M{"$inc": bson.M{"bl_send_avg_bytes": uconn.AverageBytes}}) + } } blIP.Connections = totalConnections blIP.UniqueConnections = uniqueConnCount diff --git a/analysis/structure/hosts.go b/analysis/structure/hosts.go index aa58a5f7..f1d14fdd 100644 --- a/analysis/structure/hosts.go +++ b/analysis/structure/hosts.go @@ -165,6 +165,33 @@ func getHosts(res *resources.Resources, conf *config.Config, sourceCollection st // entry.IPv6Binary = ipv6ToBinary(ip) // } + if queryRes.Local { + + // add highest beacon conncount and score + var beaconRes struct { + ConnectionCount int `bson:"connection_count"` + Score float64 `bson:"score"` + } + err1 := session.DB(res.DB.GetSelectedDB()).C(conf.T.Beacon.BeaconTable).Find(bson.M{"src": queryRes.IP}).Sort("-score").Limit(1).One(&beaconRes) + if err1 == nil { + entry.MaxBeaconScore = beaconRes.Score + entry.MaxBeaconConnCount = beaconRes.ConnectionCount + } + + // Count how many times the host made a TXT query + txtCount, err2 := session.DB(res.DB.GetSelectedDB()).C(conf.T.Structure.DNSTable). + Find(bson.M{ + "$and": []bson.M{ + bson.M{"id_orig_h": queryRes.IP}, + bson.M{"qtype_name": "TXT"}, + }}).Count() + + if err2 == nil { + entry.TxtQueryCount = txtCount + } + + } + output = append(output, entry) } diff --git a/commands/analyze.go b/commands/analyze.go index c1638190..cbde2001 100644 --- a/commands/analyze.go +++ b/commands/analyze.go @@ -109,7 +109,11 @@ func analyze(inDb string, configFile string) error { logAnalysisFunc("Unique Connections", td, res, structure.BuildUniqueConnectionsCollection, ) - + // must go after uconns + logAnalysisFunc("Beaconing", td, res, + beacon.BuildBeaconCollection, + ) + // must go after beaconing logAnalysisFunc("Unique Hosts", td, res, func(innerRes *resources.Resources) { structure.BuildHostsCollection(innerRes) @@ -118,18 +122,19 @@ func analyze(inDb string, configFile string) error { logAnalysisFunc("Unique Hostnames", td, res, dns.BuildHostnamesCollection, ) + logAnalysisFunc("Exploded DNS", td, res, dns.BuildExplodedDNSCollection, ) + logAnalysisFunc("User Agent", td, res, useragent.BuildUserAgentCollection, ) + logAnalysisFunc("Blacklisted", td, res, blacklist.BuildBlacklistedCollections, ) - logAnalysisFunc("Beaconing", td, res, - beacon.BuildBeaconCollection, - ) + logAnalysisFunc("Cross Reference", td, res, crossref.BuildXRefCollection, ) diff --git a/datatypes/structure/structure.go b/datatypes/structure/structure.go index 9b9eb2e0..9324a668 100644 --- a/datatypes/structure/structure.go +++ b/datatypes/structure/structure.go @@ -28,7 +28,14 @@ type ( CountDst int32 `bson:"count_dst"` IPv4Binary int64 `bson:"ipv4_binary"` // IPv6Binary IPv6Integers `bson:"ipv6_binary"` // for future ipv6 support - MaxDuration float32 `bson:"max_duration"` + MaxDuration float32 `bson:"max_duration"` + MaxBeaconScore float64 `bson:"max_beacon_score"` + MaxBeaconConnCount int `bson:"max_beacon_conn_count"` + BLSendCount int32 `bson:"bl_send_count"` + BLRecvCount int32 `bson:"bl_recv_count"` + BLSendAvgBytes int32 `bson:"bl_send_avg_bytes"` + BLRecvAvgBytes int32 `bson:"bl_recv_avg_bytes"` + TxtQueryCount int `bson:"txt_query_count"` } //UniqueConnection describes a pair of IP addresses which contacted @@ -41,7 +48,7 @@ type ( LocalSrc bool `bson:"local_src"` LocalDst bool `bson:"local_dst"` TotalBytes int `bson:"total_bytes"` - AverageBytes float32 `bson:"average_bytes"` + AverageBytes float32 `bson:"avg_bytes"` TsList []int64 `bson:"ts_list"` // Connection timestamps for this src, dst pair OrigIPBytes []int64 `bson:"orig_bytes_list"` // Src to dst connection sizes for each connection MaxDuration float32 `bson:"max_duration"` From 58ba1798ba458af992ed1cf3b629b88ddb6e455e Mon Sep 17 00:00:00 2001 From: Ethan Robish Date: Sun, 9 Dec 2018 14:20:51 -0600 Subject: [PATCH 2/2] Combine queries, add comments, and add additional field --- analysis/blacklist/ips.go | 44 ++++++++++++++++++++++++++++---- datatypes/structure/structure.go | 8 +++--- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/analysis/blacklist/ips.go b/analysis/blacklist/ips.go index 3ee7e505..995afbae 100644 --- a/analysis/blacklist/ips.go +++ b/analysis/blacklist/ips.go @@ -83,6 +83,7 @@ func buildBlacklistedIPs(ips *mgo.Iter, res *resources.Resources, &blIP, res.DB.GetSelectedDB(), res.Config.T.Structure.UniqueConnTable, + res.Config.T.Structure.HostTable, ssn, source, ) @@ -136,8 +137,12 @@ func checkRitaBlacklistIPs(ips *mgo.Iter, blHandle *bl.Blacklist, close(resultsChannel) } +// fillBlacklistedIP tallies the total number of bytes and connections +// made to each blacklisted IP. It stores this information in the blIP +// parameter. The source parameter is true if the blacklisted IP initiated +// the connections or false if the blacklisted IP received the connections. func fillBlacklistedIP(blIP *data.BlacklistedIP, db, uconnCollection string, - ssn *mgo.Session, source bool) error { + hostCollection string, ssn *mgo.Session, source bool) error { var connQuery bson.M if source { connQuery = bson.M{"src": blIP.IP} @@ -150,19 +155,48 @@ func fillBlacklistedIP(blIP *data.BlacklistedIP, db, uconnCollection string, var uniqueConnCount int uniqueConnections := ssn.DB(db).C(uconnCollection).Find(connQuery).Iter() var uconn structure.UniqueConnection + + // Loop through uconn to add up the total number of bytes and connections + // Also update the non-blacklist side of the connection's host collection entry for uniqueConnections.Next(&uconn) { totalBytes += uconn.TotalBytes totalConnections += uconn.ConnectionCount uniqueConnCount++ + // For every set of connections made to a blacklisted IP, we want to + // keep track of how much data (# of conns and # of bytes) was sent + // or received by the internal IP. if source { - ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Dst}, bson.M{"$inc": bson.M{"bl_recv_count": 1}}) - ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Dst}, bson.M{"$inc": bson.M{"bl_recv_avg_bytes": uconn.AverageBytes}}) + // If the blacklisted IP initiated the connection, then bl_in_count + // holds the number of unique blacklisted IPs connected to the given + // host. + // bl_sum_avg_bytes adds the average number of bytes over all + // individual connections between these two systems. This is an + // indication of how much data was transferred overall but not take + // into account the number of connections. + // bl_total_bytes adds the total number of bytes sent over all + // individual connections between the two systems. + ssn.DB(db).C(hostCollection).Update( + bson.M{"ip": uconn.Dst}, + bson.D{ + {"$inc", bson.M{"bl_in_count": 1}}, + {"$inc", bson.M{"bl_sum_avg_bytes": uconn.AverageBytes}}, + {"$inc", bson.M{"bl_total_bytes": uconn.TotalBytes}}, + }) } else { - ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Src}, bson.M{"$inc": bson.M{"bl_send_count": 1}}) - ssn.DB(db).C("host").Update(bson.M{"ip": uconn.Src}, bson.M{"$inc": bson.M{"bl_send_avg_bytes": uconn.AverageBytes}}) + // If the internal system initiated the connection, then bl_out_count + // holds the number of unique blacklisted IPs the given host contacted. + // bl_sum_avg_bytes and bl_total_bytes are the same as above. + ssn.DB(db).C(hostCollection).Update( + bson.M{"ip": uconn.Src}, + bson.D{ + {"$inc", bson.M{"bl_out_count": 1}}, + {"$inc", bson.M{"bl_sum_avg_bytes": uconn.AverageBytes}}, + {"$inc", bson.M{"bl_total_bytes": uconn.TotalBytes}}, + }) } } + blIP.Connections = totalConnections blIP.UniqueConnections = uniqueConnCount blIP.TotalBytes = totalBytes diff --git a/datatypes/structure/structure.go b/datatypes/structure/structure.go index 9324a668..e48e6852 100644 --- a/datatypes/structure/structure.go +++ b/datatypes/structure/structure.go @@ -31,10 +31,10 @@ type ( MaxDuration float32 `bson:"max_duration"` MaxBeaconScore float64 `bson:"max_beacon_score"` MaxBeaconConnCount int `bson:"max_beacon_conn_count"` - BLSendCount int32 `bson:"bl_send_count"` - BLRecvCount int32 `bson:"bl_recv_count"` - BLSendAvgBytes int32 `bson:"bl_send_avg_bytes"` - BLRecvAvgBytes int32 `bson:"bl_recv_avg_bytes"` + BlOutCount int32 `bson:"bl_out_count"` + BlInCount int32 `bson:"bl_in_count"` + BlSumAvgBytes int32 `bson:"bl_sum_avg_bytes"` + BlTotalBytes int32 `bson:"bl_total_bytes"` TxtQueryCount int `bson:"txt_query_count"` }