Update test scripts

scripts/install_grpcurl.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+go get github.com/fullstorydev/grpcurl/...
+go install github.com/fullstorydev/grpcurl/cmd/grpcurl
+

scripts/startService.sh

@@ -24,8 +24,8 @@ export HUBBLE_PORT=4245
 export OPERATION_MODE=2
 export CRON_JOB_TIME_INTERVAL="@every 0h0m5s"
 
-# network log source: hubble | db
-export NETWORK_LOG_FROM=db
+# network log source: hubble | db | file (for testing functionalities)
+export NETWORK_LOG_FROM=file
 
 # discovered policy output: db or db|file
 export DISCOVERED_POLICY_TO="db|file"
@@ -51,4 +51,4 @@ export DISCOVERY_RULE_TYPES=511
 # skip namepsace info
 export IGNORING_NAMESPACES="kube-system|knox-auto-policy|cilium|hipster"
 
-$KNOX_AUTO_HOME/src/knoxAutoPolicy
+$KNOX_AUTO_HOME/src/knoxAutoPolicy

src/core/configManager.go

@@ -106,8 +106,11 @@ func LoadDefaultConfig() {
 	Cfg.CronJobTimeInterval = libs.GetEnv("CRON_JOB_TIME_INTERVAL", "@every 0h0m5s")
 	Cfg.OneTimeJobTimeSelection = "" // e.g., 2021-01-20 07:00:23|2021-01-20 07:00:25
 
-	// input & output
+	// input
 	Cfg.NetworkLogFrom = libs.GetEnv("NETWORK_LOG_FROM", "db")
+	Cfg.NetworkLogFile = libs.GetEnv("NETWORK_LOG_FILE", "./flows.json")
+
+	// output
 	Cfg.DiscoveredPolicyTo = libs.GetEnv("DISCOVERED_POLICY_TO", "db")
 	Cfg.PolicyDir = libs.GetEnv("POLICY_DIR", "./")
 
@@ -134,6 +137,11 @@ func LoadDefaultConfig() {
 	}
 }
 
+// SetLogFile for testing
+func SetLogFile(file string) {
+	Cfg.NetworkLogFile = file
+}
+
 // AddConfiguration function
 func AddConfiguration(newConfig types.Configuration) error {
 	return libs.AddConfiguration(Cfg.ConfigDB, newConfig)

src/core/helperFunctions.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/accuknox/knoxAutoPolicy/src/libs"
 	types "github.com/accuknox/knoxAutoPolicy/src/types"
+	"github.com/cilium/cilium/api/v1/flow"
 )
 
 // =========== //
@@ -200,8 +201,8 @@ func updateDstLabels(dsts []MergedPortDst, pods []types.Pod) []MergedPortDst {
 // == Flow ID Tracking == //
 // ====================== //
 
-// resetTrackFlowID function
-func resetTrackFlowID() {
+// clearTrackFlowID function
+func clearTrackFlowID() {
 	FlowIDTracker = map[FlowIDTracking][]int{}
 	FlowIDTracker2 = map[FlowIDTracking2][]int{}
 }
@@ -541,3 +542,123 @@ func updateServiceEndpoint(services []types.Service, endpoints []types.Endpoint,
 		}
 	}
 }
+
+// =============== //
+// == Clearance == //
+// =============== //
+
+func clearDomainToIPs() {
+	DomainToIPs = map[string][]string{}
+}
+
+func cleargLabeledSrcsPerDst() {
+	gLabeledSrcsPerDst = map[string]labeledSrcsPerDstMap{}
+}
+
+func clearHTTPAggregator() {
+	WildPaths = []string{WildPathDigit, WildPathChar}
+	MergedSrcPerMergedDstForHTTP = map[string][]*HTTPDst{}
+}
+
+func clearGlobalVariabels() {
+	clearDomainToIPs()
+	cleargLabeledSrcsPerDst()
+	clearHTTPAggregator()
+	clearTrackFlowID()
+}
+
+// ============= //
+// == Testing == //
+// ============= //
+
+// ReplaceMultiubuntuPodName ...
+func ReplaceMultiubuntuPodName(flows []*flow.Flow, pods []types.Pod) {
+	var pod1Name, pod2Name, pod3Name, pod4Name, pod5Name string
+	var kubeDNS string
+
+	for _, pod := range pods {
+		if strings.Contains(pod.PodName, "ubuntu-1-deployment") {
+			pod1Name = pod.PodName
+		}
+
+		if strings.Contains(pod.PodName, "ubuntu-2-deployment") {
+			pod2Name = pod.PodName
+		}
+
+		if strings.Contains(pod.PodName, "ubuntu-3-deployment") {
+			pod3Name = pod.PodName
+		}
+
+		if strings.Contains(pod.PodName, "ubuntu-4-deployment") {
+			pod4Name = pod.PodName
+		}
+
+		if strings.Contains(pod.PodName, "ubuntu-5-deployment") {
+			pod5Name = pod.PodName
+		}
+
+		if strings.Contains(pod.PodName, "kube-dns") && !strings.Contains(pod.PodName, "kube-dns-autoscaler") {
+			kubeDNS = pod.PodName
+		}
+	}
+
+	for i, flow := range flows {
+		if strings.Contains(flow.GetSource().GetPodName(), "ubuntu-1-deployment") {
+			flows[i].Source.PodName = pod1Name
+		}
+
+		if strings.Contains(flow.GetDestination().GetPodName(), "ubuntu-1-deployment") {
+			flows[i].Destination.PodName = pod1Name
+		}
+
+		///
+
+		if strings.Contains(flow.GetSource().GetPodName(), "ubuntu-2-deployment") {
+			flows[i].Source.PodName = pod2Name
+		}
+
+		if strings.Contains(flow.GetDestination().GetPodName(), "ubuntu-2-deployment") {
+			flows[i].Destination.PodName = pod2Name
+		}
+
+		///
+
+		if strings.Contains(flow.GetSource().GetPodName(), "ubuntu-3-deployment") {
+			flows[i].Source.PodName = pod3Name
+		}
+
+		if strings.Contains(flow.GetDestination().GetPodName(), "ubuntu-3-deployment") {
+			flows[i].Destination.PodName = pod3Name
+		}
+
+		///
+
+		if strings.Contains(flow.GetSource().GetPodName(), "ubuntu-4-deployment") {
+			flows[i].Source.PodName = pod4Name
+		}
+
+		if strings.Contains(flow.GetDestination().GetPodName(), "ubuntu-4-deployment") {
+			flows[i].Destination.PodName = pod4Name
+		}
+
+		///
+
+		if strings.Contains(flow.GetSource().GetPodName(), "ubuntu-5-deployment") {
+			flows[i].Source.PodName = pod5Name
+		}
+
+		if strings.Contains(flow.GetDestination().GetPodName(), "ubuntu-5-deployment") {
+			flows[i].Destination.PodName = pod5Name
+		}
+
+		///
+
+		if strings.Contains(flow.GetSource().GetPodName(), "kube-dns") && !strings.Contains(flow.GetSource().GetPodName(), "kube-dns-autoscaler") {
+			flows[i].Source.PodName = kubeDNS
+		}
+
+		if strings.Contains(flow.GetDestination().GetPodName(), "kube-dns") && !strings.Contains(flow.GetSource().GetPodName(), "kube-dns-autoscaler") {
+			flows[i].Destination.PodName = kubeDNS
+		}
+	}
+}

src/core/networkPolicy.go

@@ -1,7 +1,10 @@
 package core
 
 import (
+	"encoding/json"
+	"io/ioutil"
 	"net"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -11,6 +14,7 @@ import (
 	"github.com/accuknox/knoxAutoPolicy/src/libs"
 	"github.com/accuknox/knoxAutoPolicy/src/plugin"
 	"github.com/accuknox/knoxAutoPolicy/src/types"
+	"github.com/cilium/cilium/api/v1/flow"
 
 	"github.com/google/go-cmp/cmp"
 
@@ -1506,6 +1510,31 @@ func getNetworkLogs() []types.KnoxNetworkLog {
 				networkLogs = append(networkLogs, log)
 			}
 		}
+	} else if Cfg.NetworkLogFrom == "file" { // from json file for testing
+		log.Info().Msg("Get network flow from the json file : " + Cfg.NetworkLogFile)
+		flows := []*flow.Flow{}
+
+		// Open our jsonFile
+		logFile, err := os.Open(Cfg.NetworkLogFile)
+		if err != nil {
+			log.Error().Msg(err.Error())
+			return nil
+		}
+		defer logFile.Close()
+
+		byteValue, _ := ioutil.ReadAll(logFile)
+		json.Unmarshal(byteValue, &flows)
+
+		// replace the pod names in prepared-flows with the working pod names
+		pods := libs.GetPods()
+		ReplaceMultiubuntuPodName(flows, pods)
+
+		// convert file flows -> network logs (but, in this case, no flow id..)
+		for _, flow := range flows {
+			if log, valid := plugin.ConvertCiliumFlowToKnoxNetworkLog(flow); valid {
+				networkLogs = append(networkLogs, log)
+			}
+		}
 	} else {
 		log.Error().Msgf("Network log source not correct: %s", Cfg.NetworkLogFrom)
 		return nil
@@ -1531,6 +1560,10 @@ func StartToDiscoveryWorker() {
 	// get network logs
 	networkLogs := getNetworkLogs()
 	if networkLogs == nil {
+		if Cfg.OperationMode == 2 && Status == StatusRunning {
+			Status = StatusIdle
+		}
+
 		return
 	}
 
@@ -1571,7 +1604,7 @@ func StartToDiscoveryWorker() {
 		log.Info().Msgf("Policy discovery started for namespace: [%s]", namespace)
 
 		// reset flow id track every target namespace
-		resetTrackFlowID()
+		clearTrackFlowID()
 
 		// discover network policies based on the network logs
 		discoveredPolicies := DiscoverNetworkPolicy(namespace, nsFilteredLogs, services, endpoints, pods)
@@ -1677,7 +1710,5 @@ func StopWorker() {
 
 	if Cfg.OperationMode == 1 { // every time intervals
 		StopCronJob()
-	} else { // one-time generation
-
 	}
 }

src/libs/dbHandler.go

@@ -176,3 +176,14 @@ func InsertDiscoveredPolicies(cfg types.ConfigDB, policies []types.KnoxNetworkPo
 		}
 	}
 }
+
+// ClearDBTables function
+func ClearDBTables(cfg types.ConfigDB) {
+	if cfg.DBDriver == "mysql" {
+		if err := ClearDBTablesMySQL(cfg); err != nil {
+			log.Error().Msg(err.Error())
+		}
+	} else if cfg.DBDriver == "mongodb" {
+
+	}
+}

src/libs/mysqlHandler.go

@@ -303,6 +303,24 @@ func InsertDiscoveredPoliciesToMySQL(cfg types.ConfigDB, policies []types.KnoxNe
 	return nil
 }
 
+// ClearDBTablesMySQL function
+func ClearDBTablesMySQL(cfg types.ConfigDB) error {
+	db := ConnectMySQL(cfg)
+	defer db.Close()
+
+	query := "DELETE FROM " + cfg.TableDiscoveredPolicy
+	if _, err := db.Query(query); err != nil {
+		return err
+	}
+
+	query = "DELETE FROM " + cfg.TableNetworkFlow
+	if _, err := db.Query(query); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // =================== //
 // == Configuration == //
 // =================== //