add fields to hosts

analysis/beacon/writer.go

@@ -50,6 +50,7 @@ func (w *writer) start() {
 		//TODO: Implement bulk writes
 		for data := range w.writeChannel {
 			ssn.DB(w.db.GetSelectedDB()).C(w.conf.T.Beacon.BeaconTable).Insert(data)
+
 		}
 		w.writeWg.Done()
 	}()

analysis/blacklist/ips.go

@@ -83,6 +83,7 @@ func buildBlacklistedIPs(ips *mgo.Iter, res *resources.Resources,
 					&blIP,
 					res.DB.GetSelectedDB(),
 					res.Config.T.Structure.UniqueConnTable,
+					res.Config.T.Structure.HostTable,
 					ssn,
 					source,
 				)
@@ -95,6 +96,7 @@ func buildBlacklistedIPs(ips *mgo.Iter, res *resources.Resources,
 					continue
 				}
 				outputCollection.Insert(&blIP)
+
 			}
 		}
 	}
@@ -135,8 +137,12 @@ func checkRitaBlacklistIPs(ips *mgo.Iter, blHandle *bl.Blacklist,
 	close(resultsChannel)
 }
 
+// fillBlacklistedIP tallies the total number of bytes and connections
+// made to each blacklisted IP. It stores this information in the blIP
+// parameter. The source parameter is true if the blacklisted IP initiated
+// the connections or false if the blacklisted IP received the connections.
 func fillBlacklistedIP(blIP *data.BlacklistedIP, db, uconnCollection string,
-	ssn *mgo.Session, source bool) error {
+	hostCollection string, ssn *mgo.Session, source bool) error {
 	var connQuery bson.M
 	if source {
 		connQuery = bson.M{"src": blIP.IP}
@@ -149,11 +155,48 @@ func fillBlacklistedIP(blIP *data.BlacklistedIP, db, uconnCollection string,
 	var uniqueConnCount int
 	uniqueConnections := ssn.DB(db).C(uconnCollection).Find(connQuery).Iter()
 	var uconn structure.UniqueConnection
+
+	// Loop through uconn to add up the total number of bytes and connections
+	// Also update the non-blacklist side of the connection's host collection entry
 	for uniqueConnections.Next(&uconn) {
 		totalBytes += uconn.TotalBytes
 		totalConnections += uconn.ConnectionCount
 		uniqueConnCount++
+
+		// For every set of connections made to a blacklisted IP, we want to
+		// keep track of how much data (# of conns and # of bytes) was sent
+		// or received by the internal IP.
+		if source {
+			// If the blacklisted IP initiated the connection, then bl_in_count
+			// holds the number of unique blacklisted IPs connected to the given
+			// host.
+			// bl_sum_avg_bytes adds the average number of bytes over all
+			// individual connections between these two systems. This is an
+			// indication of how much data was transferred overall but not take
+			// into account the number of connections.
+			// bl_total_bytes adds the total number of bytes sent over all
+			// individual connections between the two systems.
+			ssn.DB(db).C(hostCollection).Update(
+				bson.M{"ip": uconn.Dst},
+				bson.D{
+					{"$inc", bson.M{"bl_in_count": 1}},
+					{"$inc", bson.M{"bl_sum_avg_bytes": uconn.AverageBytes}},
+					{"$inc", bson.M{"bl_total_bytes": uconn.TotalBytes}},
+				})
+		} else {
+			// If the internal system initiated the connection, then bl_out_count
+			// holds the number of unique blacklisted IPs the given host contacted.
+			// bl_sum_avg_bytes and bl_total_bytes are the same as above.
+			ssn.DB(db).C(hostCollection).Update(
+				bson.M{"ip": uconn.Src},
+				bson.D{
+					{"$inc", bson.M{"bl_out_count": 1}},
+					{"$inc", bson.M{"bl_sum_avg_bytes": uconn.AverageBytes}},
+					{"$inc", bson.M{"bl_total_bytes": uconn.TotalBytes}},
+				})
+		}
 	}
+
 	blIP.Connections = totalConnections
 	blIP.UniqueConnections = uniqueConnCount
 	blIP.TotalBytes = totalBytes

analysis/structure/hosts.go

@@ -165,6 +165,33 @@ func getHosts(res *resources.Resources, conf *config.Config, sourceCollection st
 		// 	entry.IPv6Binary = ipv6ToBinary(ip)
 		// }
 
+		if queryRes.Local {
+
+			// add highest beacon conncount and score
+			var beaconRes struct {
+				ConnectionCount int     `bson:"connection_count"`
+				Score           float64 `bson:"score"`
+			}
+			err1 := session.DB(res.DB.GetSelectedDB()).C(conf.T.Beacon.BeaconTable).Find(bson.M{"src": queryRes.IP}).Sort("-score").Limit(1).One(&beaconRes)
+			if err1 == nil {
+				entry.MaxBeaconScore = beaconRes.Score
+				entry.MaxBeaconConnCount = beaconRes.ConnectionCount
+			}
+
+			// Count how many times the host made a TXT query
+			txtCount, err2 := session.DB(res.DB.GetSelectedDB()).C(conf.T.Structure.DNSTable).
+				Find(bson.M{
+					"$and": []bson.M{
+						bson.M{"id_orig_h": queryRes.IP},
+						bson.M{"qtype_name": "TXT"},
+					}}).Count()
+
+			if err2 == nil {
+				entry.TxtQueryCount = txtCount
+			}
+
+		}
+
 		output = append(output, entry)
 
 	}

commands/analyze.go

@@ -109,7 +109,11 @@ func analyze(inDb string, configFile string) error {
 		logAnalysisFunc("Unique Connections", td, res,
 			structure.BuildUniqueConnectionsCollection,
 		)
-
+		// must go after uconns
+		logAnalysisFunc("Beaconing", td, res,
+			beacon.BuildBeaconCollection,
+		)
+		// must go after beaconing
 		logAnalysisFunc("Unique Hosts", td, res,
 			func(innerRes *resources.Resources) {
 				structure.BuildHostsCollection(innerRes)
@@ -118,18 +122,19 @@ func analyze(inDb string, configFile string) error {
 		logAnalysisFunc("Unique Hostnames", td, res,
 			dns.BuildHostnamesCollection,
 		)
+
 		logAnalysisFunc("Exploded DNS", td, res,
 			dns.BuildExplodedDNSCollection,
 		)
+
 		logAnalysisFunc("User Agent", td, res,
 			useragent.BuildUserAgentCollection,
 		)
+
 		logAnalysisFunc("Blacklisted", td, res,
 			blacklist.BuildBlacklistedCollections,
 		)
-		logAnalysisFunc("Beaconing", td, res,
-			beacon.BuildBeaconCollection,
-		)
+
 		logAnalysisFunc("Cross Reference", td, res,
 			crossref.BuildXRefCollection,
 		)

datatypes/structure/structure.go

@@ -28,7 +28,14 @@ type (
 		CountDst   int32         `bson:"count_dst"`
 		IPv4Binary int64         `bson:"ipv4_binary"`
 		// IPv6Binary IPv6Integers  `bson:"ipv6_binary"` // for future ipv6 support
-		MaxDuration float32 `bson:"max_duration"`
+		MaxDuration        float32 `bson:"max_duration"`
+		MaxBeaconScore     float64 `bson:"max_beacon_score"`
+		MaxBeaconConnCount int     `bson:"max_beacon_conn_count"`
+		BlOutCount         int32   `bson:"bl_out_count"`
+		BlInCount          int32   `bson:"bl_in_count"`
+		BlSumAvgBytes      int32   `bson:"bl_sum_avg_bytes"`
+		BlTotalBytes       int32   `bson:"bl_total_bytes"`
+		TxtQueryCount      int     `bson:"txt_query_count"`
 	}
 
 	//UniqueConnection describes a pair of IP addresses which contacted
@@ -41,7 +48,7 @@ type (
 		LocalSrc        bool          `bson:"local_src"`
 		LocalDst        bool          `bson:"local_dst"`
 		TotalBytes      int           `bson:"total_bytes"`
-		AverageBytes    float32       `bson:"average_bytes"`
+		AverageBytes    float32       `bson:"avg_bytes"`
 		TsList          []int64       `bson:"ts_list"`         // Connection timestamps for this src, dst pair
 		OrigIPBytes     []int64       `bson:"orig_bytes_list"` // Src to dst connection sizes for each connection
 		MaxDuration     float32       `bson:"max_duration"`