Merge branch 'master' into dev

accuknox · Sep 2, 2021 · 0797199 · 0797199
2 parents 37c97da + 83fa44a
commit 0797199
Show file tree

Hide file tree

Showing 13 changed files with 326 additions and 351 deletions.
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
@@ -2,9 +2,9 @@ name: ci-test
 
 on:
   push:
-    branches: [master,dev]
+    branches: [master, dev]
   pull_request:
-    branches: [master,dev]
+    branches: [master, dev]
 
 jobs:
   unit-test:
@@ -18,4 +18,50 @@ jobs:
           go-version: v1.16
 
       - name: Unit Test
-        run: ./tests/test-go-unit.sh
+        run: ./tests/test-go-unit.sh
+
+  system-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: v1.16
+
+      - name: Setup Env
+        run: |
+          # install kernel-headers
+          sudo apt-get update
+          sudo apt-get install -y linux-headers-$(uname -r)
+          # install kubeadm
+          sudo apt-get update
+          sudo apt-get install -y apt-transport-https curl
+          curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+          sudo touch /etc/apt/sources.list.d/kubernetes.list
+          echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
+          sudo apt-get update
+          sudo apt-get install -y kubelet=1.21.3-00 kubeadm=1.21.3-00 kubectl=1.21.3-00
+          sudo apt-mark hold kubelet kubeadm kubectl
+          # install apparmor and auditd
+          sudo apt-get install -y apparmor apparmor-utils auditd
+          sudo systemctl start apparmor; sudo systemctl start auditd
+          # turn off swap
+          sudo swapoff -a
+          # initialize kubernetes
+          sudo kubeadm init --pod-network-cidr=10.244.0.0/16 | tee -a ~/k8s_init.log
+          # copy k8s config
+          mkdir -p $HOME/.kube
+          sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+          sudo chown $USER:$USER $HOME/.kube/config
+          export KUBECONFIG=$HOME/.kube/config
+          echo "export KUBECONFIG=$HOME/.kube/config" | tee -a ~/.bashrc
+          # install flannel
+          kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.13.0/Documentation/kube-flannel.yml
+          # disable master isolation
+          kubectl taint nodes --all node-role.kubernetes.io/master-
+          # Install grpcurl
+          go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest
+      - name: Run Test Script
+        run: ./tests/test-scenarios-local.sh
diff --git a/deployments b/deployments
diff --git a/src/conf/local.yaml b/src/conf/local.yaml
@@ -5,15 +5,15 @@ application:
     operation-trigger: 100
     cron-job-time-interval: "0h0m10s"         # format: XhYmZs 
     network-log-limit: 100000
-    network-log-from: "hubble"                # db|hubble
+    network-log-from: "kafka"                # db|hubble|kafka
     network-log-file: "./flow.json"           # file path
     network-policy-to: "db|file"              # db, file
     network-policy-dir: "./"
   system:
     operation-mode: 1                         # 1: cronjob | 2: one-time-job
     operation-trigger: 100
     cron-job-time-interval: "0h0m10s"         # format: XhYmZs
-    system-log-from: "kubearmor"                     # db|kubearmor
+    system-log-from: "kafka"                     # db|kubearmor|kafka
     system-log-limit: 100000
     system-log-file: "./log.json"             # file path
     system-policy-to: "db|file"               # db, file

diff --git a/src/config/configManager_test.go b/src/config/configManager_test.go
@@ -2,11 +2,8 @@ package config
 
 import (
 	"bytes"
-	"encoding/json"
 	"testing"
 
-	"github.com/DATA-DOG/go-sqlmock"
-	"github.com/accuknox/knoxAutoPolicy/src/libs"
 	types "github.com/accuknox/knoxAutoPolicy/src/types"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
@@ -29,7 +26,6 @@ func TestLoadConfigDB(t *testing.T) {
 	assert.NotEmpty(t, cfg.DBHost, "DB host should not be empty")
 	assert.NotEmpty(t, cfg.DBPort, "DB host should not be empty")
 
-	assert.NotEmpty(t, cfg.TableConfiguration, "Table configuration should not be empty")
 	assert.NotEmpty(t, cfg.TableNetworkLog, "Table network_log should not be empty")
 	assert.NotEmpty(t, cfg.TableNetworkPolicy, "Table network_policy should not be empty")
 	assert.NotEmpty(t, cfg.TableSystemLog, "Table system_log should not be empty")
@@ -93,255 +89,3 @@ func TestSetLogFile(t *testing.T) {
 
 	assert.Equal(t, CurrentCfg.ConfigNetPolicy.NetworkLogFile, "test_log.log", "network log file should be \"test_log.log\"")
 }
-
-func TestAddConfiguration(t *testing.T) {
-	// prepare mock mysql
-	_, mock := libs.NewMock()
-
-	newCfg := types.Configuration{}
-	newCfg.ConfigName = "test_config"
-	newCfg.ConfigNetPolicy.NetPolicyCIDRBits = 32
-
-	configDBPtr := &newCfg.ConfigDB
-	configDB, _ := json.Marshal(configDBPtr)
-
-	configHubblePtr := &newCfg.ConfigCiliumHubble
-	configCilium, _ := json.Marshal(configHubblePtr)
-
-	configKubeArmorPtr := &newCfg.ConfigKubeArmorRelay
-	configKubeArmor, _ := json.Marshal(configKubeArmorPtr)
-
-	configFilterPtr := &newCfg.ConfigNetPolicy.NetLogFilters
-	configFilter, _ := json.Marshal(configFilterPtr)
-
-	prep := mock.ExpectPrepare("INSERT INTO auto_policy_config")
-	prep.ExpectExec().WithArgs(
-		"test_config",   //config_name
-		0,               //status
-		configDB,        //config_db
-		configCilium,    //config_cilium_hubble
-		configKubeArmor, //config_kubearmor_relay
-		0,               //network_operation_mode
-		"",              //network_cronjob_time_interval
-		"",              //network_one_time_job_time_selection
-		"",              //network_log_from
-		"",              //network_log_file
-		"",              //network_policy_to
-		"",              //network_policy_dir
-		configFilter,    //network_policy_log_filters
-		0,               //network_policy_types
-		0,               //network_policy_rule_types
-		32,              //network_policy_cidr_bits
-		0,               //network_policy_l3_level
-		0,               //network_policy_l4_level
-		0,               //network_policy_l7_level
-		0,               //system_operation_mode
-		"",              //system_cronjob_time_interval
-		"",              //system_one_time_job_time_selection
-		"",              //system_log_from
-		"",              //system_log_file
-		"",              //system_policy_to
-		"",              //system_policy_dir
-		0,               //system_policy_types
-		configFilter,    //system_policy_log_filters
-		false,           //system_policy_proc_fromsource
-		false,           //system_policy_file_fromsource
-		"",              //cluster_info_from
-		"",              //cluster_mgmt_url
-	).WillReturnResult(sqlmock.NewResult(0, 1))
-
-	// add configuration
-	err := AddConfiguration(newCfg)
-	assert.NoError(t, err)
-
-	if err = mock.ExpectationsWereMet(); err != nil {
-		t.Errorf("unmet expectation error: %s", err)
-	}
-}
-
-func TestGetConfigurations(t *testing.T) {
-	// prepare mock mysql
-	_, mock := libs.NewMock()
-
-	testCfg := types.Configuration{}
-	testCfg.ConfigName = "test_config"
-	testCfg.ConfigNetPolicy.NetPolicyCIDRBits = 32
-
-	configDBPtr := &testCfg.ConfigDB
-	configDB, _ := json.Marshal(configDBPtr)
-
-	configHubblePtr := &testCfg.ConfigCiliumHubble
-	configCilium, _ := json.Marshal(configHubblePtr)
-
-	configKubeArmorPtr := &testCfg.ConfigKubeArmorRelay
-	configKubeArmor, _ := json.Marshal(configKubeArmorPtr)
-
-	configFilterPtr := &testCfg.ConfigNetPolicy.NetLogFilters
-	configFilter, _ := json.Marshal(configFilterPtr)
-
-	rows := mock.NewRows([]string{
-		"id",
-		"config_name",
-		"status",
-		"config_db",
-		"config_cilium_hubble",
-		"config_kubearmor_relay",
-		"network_operation_mode",
-		"network_cronjob_time_interval",
-		"network_one_time_job_time_selection",
-		"network_log_from",
-		"network_log_file",
-		"network_policy_to",
-		"network_policy_dir",
-		"network_policy_log_filters",
-		"network_policy_types",
-		"network_policy_rule_types",
-		"network_policy_cidr_bits",
-		"network_policy_l3_level",
-		"network_policy_l4_level",
-		"network_policy_l7_level",
-		"system_operation_mode",
-		"system_cronjob_time_interval",
-		"system_one_time_job_time_selection",
-		"system_log_from",
-		"system_log_file",
-		"system_policy_to",
-		"system_policy_dir",
-		"system_policy_types",
-		"system_policy_log_filters",
-		"system_policy_proc_fromsource",
-		"system_policy_file_fromsource",
-		"cluster_info_from",
-		"cluster_mgmt_url"}).
-		AddRow(
-			1,               //id
-			"test_config",   //config_name
-			0,               //status
-			configDB,        //config_db
-			configCilium,    //config_cilium_hubble
-			configKubeArmor, //config_kubearmor_relay
-			0,               //network_operation_mode
-			"",              //network_cronjob_time_interval
-			"",              //network_one_time_job_time_selection
-			"",              //network_log_from
-			"",              //network_log_file
-			"",              //network_policy_to
-			"",              //network_policy_dir
-			configFilter,    //network_policy_log_filters
-			0,               //network_policy_types
-			0,               //network_policy_rule_types
-			32,              //network_policy_cidr_bits
-			0,               //network_policy_l3_level
-			0,               //network_policy_l4_level
-			0,               //network_policy_l7_level
-			0,               //system_operation_mode
-			"",              //system_cronjob_time_interval
-			"",              //system_one_time_job_time_selection
-			"",              //system_log_from
-			"",              //system_log_file
-			"",              //system_policy_to
-			"",              //system_policy_dir
-			0,               //system_policy_types
-			configFilter,    //system_policy_log_filters
-			false,           //system_policy_proc_fromsource
-			false,           //system_policy_file_fromsource
-			"",              //cluster_info_from
-			"",              //cluster_mgmt_url
-		)
-
-	query := "SELECT (.+) FROM auto_policy_config WHERE config_name = ?"
-	mock.ExpectQuery(query).WillReturnRows(rows)
-
-	// get configuration by name
-	results, err := GetConfigurations(testCfg.ConfigName)
-	assert.NoError(t, err)
-	assert.Equal(t, results[0].ConfigName, "test_config")
-
-	if err = mock.ExpectationsWereMet(); err != nil {
-		t.Errorf("unmet expectation error: %s", err)
-	}
-}
-
-func TestUpdateConfiguration(t *testing.T) {
-	// prepare mock mysql
-	_, mock := libs.NewMock()
-
-	testCfg := types.Configuration{}
-	testCfg.ConfigName = "test_config"
-	testCfg.ConfigNetPolicy.NetPolicyCIDRBits = 24
-
-	configDBPtr := &testCfg.ConfigDB
-	configDB, _ := json.Marshal(configDBPtr)
-
-	configHubblePtr := &testCfg.ConfigCiliumHubble
-	configCilium, _ := json.Marshal(configHubblePtr)
-
-	configKubeArmorPtr := &testCfg.ConfigKubeArmorRelay
-	configKubeArmor, _ := json.Marshal(configKubeArmorPtr)
-
-	configFilterPtr := &testCfg.ConfigNetPolicy.NetLogFilters
-	configFilter, _ := json.Marshal(configFilterPtr)
-
-	prep := mock.ExpectPrepare("UPDATE auto_policy_config")
-	prep.ExpectExec().WithArgs(
-		configDB,        //config_db
-		configCilium,    //config_cilium_hubble
-		configKubeArmor, //config_kubearmor_relay
-		0,               //network_operation_mode
-		"",              //network_cronjob_time_interval
-		"",              //network_one_time_job_time_selection
-		"",              //network_log_from
-		"",              //network_log_file
-		"",              //network_policy_to
-		"",              //network_policy_dir
-		configFilter,    //network_policy_log_filters
-		0,               //network_policy_types
-		0,               //network_policy_rule_types
-		24,              //network_policy_cidr_bits
-		0,               //network_policy_l3_level
-		0,               //network_policy_l4_level
-		0,               //network_policy_l7_level
-		0,               //system_operation_mode
-		"",              //system_cronjob_time_interval
-		"",              //system_one_time_job_time_selection
-		"",              //system_log_from
-		"",              //system_log_file
-		"",              //system_policy_to
-		"",              //system_policy_dir
-		0,               //system_policy_types
-		configFilter,    //system_policy_log_filters
-		false,           //system_policy_proc_fromsource
-		false,           //system_policy_file_fromsource
-		"",              //cluster_info_from
-		"",              //cluster_mgmt_url).
-		"test_config",   //config_name
-	).WillReturnResult(sqlmock.NewResult(0, 1))
-
-	// update configuration by name
-	err := UpdateConfiguration("test_config", testCfg)
-	assert.NoError(t, err)
-
-	if err = mock.ExpectationsWereMet(); err != nil {
-		t.Errorf("unmet expectation error: %s", err)
-	}
-}
-
-func TestDeleteConfiguration(t *testing.T) {
-	// prepare mock mysql
-	_, mock := libs.NewMock()
-
-	testCfg := types.Configuration{}
-	testCfg.ConfigName = "test_config"
-
-	prep := mock.ExpectPrepare("DELETE FROM auto_policy_config")
-	prep.ExpectExec().WithArgs("test_config").
-		WillReturnResult(sqlmock.NewResult(0, 1))
-
-	// update configuration by name
-	err := DeleteConfiguration("test_config")
-	assert.NoError(t, err)
-
-	if err = mock.ExpectationsWereMet(); err != nil {
-		t.Errorf("unmet expectation error: %s", err)
-	}
-}