Merge branch 'dev'

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "deployments"]
-	path = deployments
-	url = https://github.com/accuknox/knoxAutoPolicy-deployment

getting-started/development_guide.md

@@ -58,3 +58,43 @@ tests/
   multi-ubuntu  - Example microservices for testing
   unit-tests    - Automated unit test framework for knoxAutoPolicy
 ```
+
+## Setting up dev env
+
+Assuming you have the knoxAutoPolicy repo checked out.
+Setup Cilium env in minikube or in k8s-VMs. Ensure following:
+
+#### Ensure Hubble relay service is enabled and port-forwarding is enabled on the host
+```
+$ kubectl -n kube-system port-forward service/hubble-relay --address 0.0.0.0 --address :: 4245:80
+```
+
+#### Setup mysql
+```
+$ cd tests/mysql
+$ docker-compose -f docker-compose.yml up
+```
+
+#### Compile knoxAutoPolicy
+```
+$ cd src
+$ make
+```
+This should generate the binary `knoxAutoPolicy` in the src folder.
+
+#### Update configuration
+Edit `src/conf/local.yaml` to ensure that:
+* `cilium-hubble: url` address is set to the localhost and port is set to 4245
+* `network-log-from: "hubble"` is set
+* `cluster-info-from: "k8sclient"` is set
+Note that this must already be set to these values by default.
+
+#### Execute knoxAutoPolicy
+```
+$ ./scripts/start_service.sh
+```
+
+#### Trigger policy discovery
+```
+$ ./scripts/start_net_worker.sh
+```

helm/values-dev.yaml

@@ -101,7 +101,8 @@ config:
     name: knoxautopolicy
     network:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
-      cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      cron-job-time-interval: "0h0m30s"         # format: XhYmZs 
+      operation-trigger: 1000
       network-log-from: "db"                    # db|hubble
       network-log-file: "./flow.json"           # file path
       network-policy-to: "db|file"              # db, file
@@ -110,7 +111,8 @@ config:
       network-policy-rule-types: 511
     system:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
-      cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      cron-job-time-interval: "0h0m30s"         # format: XhYmZs 
+      operation-trigger: 100
       system-log-from: "db"                     # db|kubearmor
       system-log-file: "./log.json"             # file path
       system-policy-to: "db|file"               # db, file
@@ -138,7 +140,7 @@ config:
       number-of-consumers: 3
       broker-address-family: v4
       session-timeout-ms: 6000
-      auto-offset-reset: "earliest"
+      auto-offset-reset: "latest" # earliest | latest
       bootstrap-servers: "dev-kafka-kafka-external-bootstrap.accuknox-dev-kafka.svc.cluster.local:9095"
       group-id: knoxautopolicy
       topics: 

helm/values-prod.yaml

@@ -97,6 +97,7 @@ config:
     network:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
       cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      operation-trigger: 100000
       network-log-from: "db"                    # db|hubble
       network-log-file: "./flow.json"           # file path
       network-policy-to: "db|file"              # db, file
@@ -106,6 +107,7 @@ config:
     system:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
       cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      operation-trigger: 1000
       system-log-from: "db"                     # db|kubearmor
       system-log-file: "./log.json"             # file path
       system-policy-to: "db|file"               # db, file

helm/values-verify.yaml

@@ -98,6 +98,7 @@ config:
     network:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
       cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      operation-trigger: 100000
       network-log-from: "db"                    # db|hubble
       network-log-file: "./flow.json"           # file path
       network-policy-to: "db|file"              # db, file
@@ -107,6 +108,7 @@ config:
     system:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
       cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      operation-trigger: 1000
       system-log-from: "db"                     # db|kubearmor
       system-log-file: "./log.json"             # file path
       system-policy-to: "db|file"               # db, file

helm/values.yaml

@@ -102,6 +102,7 @@ config:
     network:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
       cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      operation-trigger: 100000
       network-log-from: "db"                    # db|hubble
       network-log-file: "./flow.json"           # file path
       network-policy-to: "db|file"              # db, file
@@ -111,6 +112,7 @@ config:
     system:
       operation-mode: 1                         # 1: cronjob | 2: one-time-job
       cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
+      operation-trigger: 1000
       system-log-from: "db"                     # db|kubearmor
       system-log-file: "./log.json"             # file path
       system-policy-to: "db|file"               # db, file

src/conf/local.yaml

@@ -3,13 +3,15 @@ application:
   network:
     operation-mode: 1                         # 1: cronjob | 2: one-time-job
     cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
-    network-log-from: "db"                    # db|hubble
+    operation-trigger: 100
+    network-log-from: "hubble"                # db|hubble
     network-log-file: "./flow.json"           # file path
     network-policy-to: "db|file"              # db, file
     network-policy-dir: "./"
   system:
-    operation-mode: 2                         # 1: cronjob | 2: one-time-job
-    cron-job-time-interval: "0h0m5s"          # format: XhYmZs 
+    operation-mode: 1                         # 1: cronjob | 2: one-time-job
+    cron-job-time-interval: "0h0m10s"         # format: XhYmZs
+    operation-trigger: 100 
     system-log-from: "db"                     # db|kubearmor
     system-log-file: "./log.json"             # file path
     system-policy-to: "db|file"               # db, file
@@ -44,6 +46,7 @@ feed-consumer:
     group-id: knoxautopolicy
     topics: 
       - cilium-telemetry-new
+      - cilium-telemetry
       - kubearmor-syslogs
     security:
       protocol: SSL
@@ -59,7 +62,8 @@ feed-consumer:
 
 logging:
   level: "INFO"
-
+
+# kubectl -n kube-system port-forward service/hubble-relay --address 0.0.0.0 --address :: 4245:80
 cilium-hubble:
-  url: 10.4.41.240
-  port: 80
+  url: localhost
+  port: 4245

src/config/configManager.go

@@ -108,6 +108,7 @@ func LoadDefaultConfig() {
 		OperationMode:           viper.GetInt("application.network.operation-mode"),
 		CronJobTimeInterval:     "@every " + viper.GetString("application.network.cron-job-time-interval"),
 		OneTimeJobTimeSelection: "", // e.g., 2021-01-20 07:00:23|2021-01-20 07:00:25
+		OperationTrigger:        viper.GetInt("application.network.operation-trigger"),
 
 		NetworkLogFrom:   viper.GetString("application.network.network-log-from"),
 		NetworkLogFile:   viper.GetString("application.network.network-log-file"),
@@ -130,6 +131,7 @@ func LoadDefaultConfig() {
 		OperationMode:           viper.GetInt("application.system.operation-mode"),
 		CronJobTimeInterval:     "@every " + viper.GetString("application.system.cron-job-time-interval"),
 		OneTimeJobTimeSelection: "", // e.g., 2021-01-20 07:00:23|2021-01-20 07:00:25
+		OperationTrigger:        viper.GetInt("application.system.operation-trigger"),
 
 		SysPolicyTypes: 7,
 
@@ -239,6 +241,10 @@ func GetCfgNetOneTime() string {
 	return CurrentCfg.ConfigNetPolicy.OneTimeJobTimeSelection
 }
 
+func GetCfgNetOperationTrigger() int {
+	return CurrentCfg.ConfigNetPolicy.OperationTrigger
+}
+
 // == //
 
 func GetCfgNetworkLogFrom() string {
@@ -301,6 +307,10 @@ func GetCfgSysOperationMode() int {
 	return CurrentCfg.ConfigSysPolicy.OperationMode
 }
 
+func GetCfgSysOperationTrigger() int {
+	return CurrentCfg.ConfigSysPolicy.OperationTrigger
+}
+
 func GetCfgSysCronJobTime() string {
 	return CurrentCfg.ConfigSysPolicy.CronJobTimeInterval
 }

src/feedconsumer/consumer.go

@@ -3,7 +3,9 @@ package feedconsumer
 import (
 	"encoding/json"
 	"errors"
+	"strconv"
 	"sync"
+	"time"
 
 	"github.com/confluentinc/confluent-kafka-go/kafka"
 	"github.com/rs/zerolog"
@@ -67,7 +69,7 @@ func (cfc *KnoxFeedConsumer) setupKafkaConfig() {
 	sessionTimeoutMs := viper.GetString("feed-consumer.kafka.session-timeout-ms")
 	autoOffsetReset := viper.GetString("feed-consumer.kafka.auto-offset-reset")
 
-	groupID := viper.GetString("feed-consumer.kafka.group-id")
+	groupID := viper.GetString("feed-consumer.kafka.group-id") + strconv.FormatUint(uint64(time.Now().Unix()), 10)
 	cfc.topics = viper.GetStringSlice("feed-consumer.kafka.topics")
 	cfc.eventsBuffer = viper.GetInt("feed-consumer.kafka.events.buffer")
 

src/libs/common.go

@@ -330,7 +330,9 @@ func WriteKubeArmorPolicyToYamlFile(namespace string, policies []types.KubeArmor
 	fileName := GetEnv("POLICY_DIR", "./") + "system_policies.yaml"
 
 	if err := os.Remove(fileName); err != nil {
-		log.Error().Msg(err.Error())
+		if !strings.Contains(err.Error(), "no such file or directory") {
+			log.Error().Msg(err.Error())
+		}
 	}
 
 	f, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0600)

src/libs/dbHandler.go

@@ -26,7 +26,7 @@ func updateTimeInterval(lastDoc map[string]interface{}) {
 	}
 }
 
-func GetNetworkLogsFromDB(cfg types.ConfigDB, timeSelection string) []map[string]interface{} {
+func GetNetworkLogsFromDB(cfg types.ConfigDB, timeSelection string, trigger int) []map[string]interface{} {
 	results := []map[string]interface{}{}
 
 	endTime = time.Now().Unix()
@@ -39,6 +39,11 @@ func GetNetworkLogsFromDB(cfg types.ConfigDB, timeSelection string) []map[string
 				return results
 			}
 			results = docs
+
+			if len(results) != 0 && len(results) < trigger {
+				log.Info().Msgf("The number of network logs [%d] is less than trigger [%d]", len(results), trigger)
+				return results
+			}
 		} else {
 			// given time selection from ~ to
 			times := strings.Split(timeSelection, "|")
@@ -169,7 +174,7 @@ var LastSyslogID int64 = 0
 var syslogStartTime int64 = 0
 var syslogEndTime int64 = 0
 
-func GetSystemLogsFromDB(cfg types.ConfigDB, timeSelection string) []map[string]interface{} {
+func GetSystemLogsFromDB(cfg types.ConfigDB, timeSelection string, trigger int) []map[string]interface{} {
 	results := []map[string]interface{}{}
 
 	syslogEndTime = time.Now().Unix()
@@ -182,6 +187,11 @@ func GetSystemLogsFromDB(cfg types.ConfigDB, timeSelection string) []map[string]
 				return results
 			}
 			results = docs
+
+			if len(results) != 0 && len(results) < trigger {
+				log.Info().Msgf("The number of system logs [%d] is less than trigger [%d]", len(results), trigger)
+				return results
+			}
 		} else {
 			// given time selection from ~ to
 			times := strings.Split(timeSelection, "|")
@@ -242,7 +252,7 @@ var LastSysAlertID int64 = 0
 var sysAlertStartTime int64 = 0
 var sysAlertEndTime int64 = 0
 
-func GetSystemAlertsFromDB(cfg types.ConfigDB, timeSelection string) []map[string]interface{} {
+func GetSystemAlertsFromDB(cfg types.ConfigDB, timeSelection string, trigger int) []map[string]interface{} {
 	results := []map[string]interface{}{}
 
 	sysAlertEndTime = time.Now().Unix()
@@ -255,6 +265,12 @@ func GetSystemAlertsFromDB(cfg types.ConfigDB, timeSelection string) []map[strin
 				return results
 			}
 			results = docs
+
+			// TOOD: checking alert
+			// if len(results) != 0 && len(results) < trigger {
+			// 	log.Info().Msgf("The number of system alerts [%d] is less than trigger [%d]", len(results), trigger)
+			// 	return results
+			// }
 		} else {
 			// given time selection from ~ to
 			times := strings.Split(timeSelection, "|")