Skip to content

Commit

Permalink
Merge pull request #105 from seungsoo-lee/master
Browse files Browse the repository at this point in the history
Update gRPC & time selection
  • Loading branch information
seungsoo-lee committed Jan 20, 2021
2 parents 33155a8 + 7c3eb60 commit bd6d93d
Show file tree
Hide file tree
Showing 26 changed files with 2,352 additions and 353 deletions.
33 changes: 33 additions & 0 deletions database/mysql/init/flow_management.sql
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS `network_flow` (

CREATE TABLE IF NOT EXISTS `discovered_policy` (
`id` int NOT NULL AUTO_INCREMENT,

`apiVersion` varchar(20) DEFAULT NULL,
`kind` varchar(20) DEFAULT NULL,
`name` varchar(50) DEFAULT NULL,
Expand All @@ -40,6 +41,38 @@ CREATE TABLE IF NOT EXISTS `discovered_policy` (
`status` varchar(10) DEFAULT NULL,
`outdated` varchar(50) DEFAULT NULL,
`spec` JSON DEFAULT NULL,

`generatedTime` int DEFAULT NULL,
PRIMARY KEY (`id`)
);

CREATE TABLE IF NOT EXISTS `auto_policy_config` (
`id` int NOT NULL AUTO_INCREMENT,

`config_name` varchar(50) DEFAULT NULL,
`status` int DEFAULT '0',

`config_db` JSON DEFAULT NULL,
`config_cilium_hubble` JSON DEFAULT NULL,

`operation_mode` int DEFAULT NULL,
`cronjob_time_interval` varchar(50) DEFAULT NULL,
`one_time_job_time_selection` varchar(50) DEFAULT NULL,

`network_log_from` varchar(50) DEFAULT NULL,
`discovered_policy_to` varchar(50) DEFAULT NULL,
`policy_dir` varchar(50) DEFAULT NULL,

`discovery_policy_types` int DEFAULT NULL,
`discovery_rule_types` int DEFAULT NULL,

`cidr_bits` int DEFAULT NULL,
`ignoring_flows` JSON DEFAULT NULL,

`l3_aggregation_level` int DEFAULT NULL,
`l4_aggregation_level` int DEFAULT NULL,
`l7_aggregation_level` int DEFAULT NULL,
`http_url_threshold` int DEFAULT NULL,

PRIMARY KEY (`id`)
);
36 changes: 21 additions & 15 deletions scripts/startService.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,39 @@ export KNOX_AUTO_HOME=`dirname $(realpath "$0")`/..

# database info
export DB_DRIVER=mysql
export DB_HOST=127.0.0.1
export DB_PORT=3306
export DB_USER=root
export DB_PASS=password
export DB_NAME=flow_management
export DB_HOST=127.0.0.1

# table info
# database table info
export TB_NETWORK_FLOW=network_flow
export TB_DISCOVERED_POLICY=discovered_policy
export TB_CONFIGURATION=auto_policy_config

# output dir info
export OUT_DIR=$KNOX_AUTO_HOME/policies/
# cilium hubble info (if want to connect with hubble relay directly)
export HUBBLE_URL=127.0.0.1
export HUBBLE_PORT=4245

# available discovery modes: egress | ingress | egress+ingress
export DISCOVERY_MODE=egress+ingress
# operation mode: cronjob: 1
# onetime job: 2
export OPERATION_MODE=2
export CRON_JOB_TIME_INTERVAL="@every 0h0m5s"

# available network log source: hubble | db
export NETWORK_LOG_FROM=db
export DISCOVERED_POLICY_TO="db|file"
export POLICY_DIR=$KNOX_AUTO_HOME/policies/

# cilium hubble info (if connect to hubble directly)
export HUBBLE_URL=127.0.0.1
export HUBBLE_PORT=4245
# available discovery modes:
# all (egress+ingress): 3
# egress only: 1
# ingress only: 2
export DISCOVERY_POLICY_TYPES=1
export DISCOVERY_RULE_TYPES=1

# operation mode: c=cronjob | a=at once
if [ $# -eq 1 ]
then
export OPERATION_MODE=$1
fi
# skip namepsace info
export IGNORING_SELECTOR_NAMESPACES="kube-system|knox-auto-policy|cilium|hipster"

$KNOX_AUTO_HOME/src/knoxAutoPolicy
$KNOX_AUTO_HOME/src/knoxAutoPolicy
9 changes: 9 additions & 0 deletions src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@ build:
# for build_image -> CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags '-s -w' -o knoxAutoPolicy main.go
go build -o knoxAutoPolicy main.go

.PHONY: protoinstall
protoinstall:
go install google.golang.org/protobuf/cmd/protoc-gen-go

.PHONY: pb
pb:
protoc -I=. --go_out . --go_opt paths=source_relative --go-grpc_out . --go-grpc_opt paths=source_relative protos/v1/config/config.proto
protoc -I=. --go_out . --go_opt paths=source_relative --go-grpc_out . --go-grpc_opt paths=source_relative protos/v1/worker/worker.proto

.PHONY: clean
clean:
rm -f knoxAutoPolicy go.sum
Expand Down
172 changes: 172 additions & 0 deletions src/core/configManager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
package core

import (
"errors"
"net"
"strings"

"github.com/accuknox/knoxAutoPolicy/src/libs"
types "github.com/accuknox/knoxAutoPolicy/src/types"
)

// Cfg ...
var Cfg types.Configuration

func init() {
// initially, default -> applied
LoadDefaultConfig()
libs.AddConfiguration(Cfg.ConfigDB, Cfg)
}

// LoadConfigDB ...
func LoadConfigDB() types.ConfigDB {
cfgDB := types.ConfigDB{}

cfgDB.DBDriver = libs.GetEnv("DB_DRIVER", "mysql")
cfgDB.DBUser = libs.GetEnv("DB_USER", "root")
cfgDB.DBPass = libs.GetEnv("DB_PASS", "password")
cfgDB.DBName = libs.GetEnv("DB_NAME", "flow_management")

if libs.IsK8sEnv() {
cfgDB.DBHost = libs.GetEnv("DB_HOST", "database.knox-auto-policy.svc.cluster.local")
dbAddr, err := net.LookupIP(cfgDB.DBHost)
if err == nil {
cfgDB.DBHost = dbAddr[0].String()
} else {
cfgDB.DBHost = libs.GetExternalIPAddr()
}
} else {
cfgDB.DBHost = libs.GetEnv("DB_HOST", "database")
dbAddr, err := net.LookupIP(cfgDB.DBHost)
if err == nil {
cfgDB.DBHost = dbAddr[0].String()
} else {
cfgDB.DBHost = libs.GetExternalIPAddr()
}
}
cfgDB.DBPort = libs.GetEnv("DB_PORT", "3306")

cfgDB.TableNetworkFlow = libs.GetEnv("TB_NETWORK_FLOW", "network_flow")
cfgDB.TableDiscoveredPolicy = libs.GetEnv("TB_DISCOVERED_POLICY", "discovered_policy")
cfgDB.TableConfiguration = libs.GetEnv("TB_CONFIGURATION", "auto_policy_config")

return cfgDB
}

// LoadConfigCiliumHubble ...
func LoadConfigCiliumHubble() types.ConfigCiliumHubble {
cfgHubble := types.ConfigCiliumHubble{}

if libs.IsK8sEnv() {
cfgHubble.HubbleURL = libs.GetEnv("HUBBLE_URL", "hubble-relay.cilium.svc.cluster.local")
addr, err := net.LookupIP(cfgHubble.HubbleURL)
if err == nil {
cfgHubble.HubbleURL = addr[0].String()
} else {
cfgHubble.HubbleURL = libs.GetExternalIPAddr()
}
} else {
cfgHubble.HubbleURL = libs.GetEnv("HUBBLE_URL", "127.0.0.1")
addr, err := net.LookupIP(cfgHubble.HubbleURL)
if err == nil {
cfgHubble.HubbleURL = addr[0].String()
} else {
cfgHubble.HubbleURL = libs.GetExternalIPAddr()
}
}
cfgHubble.HubblePort = libs.GetEnv("HUBBLE_PORT", "80")

return cfgHubble
}

// LoadDefaultConfig ...
func LoadDefaultConfig() {
Cfg = types.Configuration{}

// basic
Cfg.ConfigName = "default"
Cfg.Status = 1

Cfg.ConfigDB = LoadConfigDB()
Cfg.ConfigCiliumHubble = LoadConfigCiliumHubble()

// set worker
Cfg.OperationMode = libs.GetEnvInt("OPERATION_MODE", 1)
Cfg.CronJobTimeInterval = libs.GetEnv("CRON_JOB_TIME_INTERVAL", "@every 0h0m5s")
Cfg.OneTimeJobTimeSelection = "" // e.g., 2021-01-20 07:00:23|2021-01-20 07:00:25

// input & output
Cfg.NetworkLogFrom = libs.GetEnv("NETWORK_LOG_FROM", "db")
Cfg.DiscoveredPolicyTo = libs.GetEnv("DISCOVERED_POLICY_TO", "db")
Cfg.PolicyDir = libs.GetEnv("POLICY_DIR", "./")

// discovery types
Cfg.DiscoveryPolicyTypes = libs.GetEnvInt("DISCOVERY_POLICY_TYPES", 3)
Cfg.DiscoveryPolicyTypes = libs.GetEnvInt("DISCOVERY_RULE_TYPES", 1)

// cidr bits
Cfg.CIDRBits = 32

// ignoring flows
skipNamespacesStr := libs.GetEnv("IGNORING_SELECTOR_NAMESPACES", "")
igFlow1 := types.IgnoringFlows{IgSelectorNamespaces: strings.Split(skipNamespacesStr, "|")}
igFlow2 := types.IgnoringFlows{
IgSelectorLabels: []string{"pod-template-hash",
"controller-revision-hash", // from istana robot-shop
"statefulset.kubernetes.io/pod-name"}, // from istana robot-shop
}
Cfg.IgnoringFlows = []types.IgnoringFlows{igFlow1, igFlow2}

// aggregation level
Cfg.L3AggregationLevel = 3
Cfg.L4AggregationLevel = 3
Cfg.L7AggregationLevel = 3
Cfg.HTTPUrlThreshold = 3
}

// AddConfiguration function
func AddConfiguration(newConfig types.Configuration) error {
return libs.AddConfiguration(Cfg.ConfigDB, newConfig)
}

// GetConfigurations function
func GetConfigurations(configName string) ([]types.Configuration, error) {
return libs.GetConfigurations(Cfg.ConfigDB, configName)
}

// UpdateConfiguration function
func UpdateConfiguration(configName string, updateConfig types.Configuration) error {
return libs.UpdateConfiguration(Cfg.ConfigDB, configName, updateConfig)
}

// DeleteConfiguration function
func DeleteConfiguration(configName string) error {
return libs.DeleteConfiguration(Cfg.ConfigDB, configName)
}

// ApplyConfiguration ...
func ApplyConfiguration(configName string) error {
if Cfg.ConfigName == configName {
return errors.New("Not applied " + configName + " due to same configuration name")
}

if err := libs.ApplyConfiguration(Cfg.ConfigDB, Cfg.ConfigName, configName); err != nil {
return err
}

appliedConfigs, err := libs.GetConfigurations(Cfg.ConfigDB, configName)
if err != nil {
return err
}

// check if db info is null
appliedCfg := appliedConfigs[0]
if appliedCfg.ConfigDB.DBHost == "" {
appliedCfg.ConfigDB = Cfg.ConfigDB
}

// update current Cfg
Cfg = appliedCfg

return nil
}
1 change: 1 addition & 0 deletions src/core/configManager_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package core
22 changes: 11 additions & 11 deletions src/core/deduplicator.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,15 +343,15 @@ func UpdateHTTP(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.Knox
if includeAllRules {
// case 2-1: policy has the lower selector count? outdated
if len(newPolicy.Spec.Selector.MatchLabels) < len(latestPolicy.Spec.Selector.MatchLabels) {
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

continue
}

// annotate the outdated policy
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand Down Expand Up @@ -413,7 +413,7 @@ func UpdateToPorts(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.K
if includeAllRules {
// case 2-1: policy has the lower selector count? outdated
if len(newPolicy.Spec.Selector.MatchLabels) < len(latestPolicy.Spec.Selector.MatchLabels) {
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand All @@ -428,7 +428,7 @@ func UpdateToPorts(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.K
}

// annotate the outdated policy
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand Down Expand Up @@ -491,7 +491,7 @@ func UpdateMatchLabels(newPolicy types.KnoxNetworkPolicy, existingPolicies []typ
if len(newPolicy.Spec.Selector.MatchLabels) < len(latestPolicy.Spec.Selector.MatchLabels) ||
newTargetLabelsCount < existTargetLabelsCount {
// case 2-2: policy has the lower target matchLabels count? outdated
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand All @@ -506,7 +506,7 @@ func UpdateMatchLabels(newPolicy types.KnoxNetworkPolicy, existingPolicies []typ
}

// annotate the outdated policy
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand Down Expand Up @@ -560,7 +560,7 @@ func UpdateEntity(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.Kn
if includeAllEntities {
// case 2-1: policy has the lower selector count? outdated
if len(newPolicy.Spec.Selector.MatchLabels) < len(latestPolicy.Spec.Selector.MatchLabels) {
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand All @@ -575,7 +575,7 @@ func UpdateEntity(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.Kn
}

// annotate the outdated fqdn policy
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand Down Expand Up @@ -629,7 +629,7 @@ func UpdateService(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.K
if includeAllService {
// case 2-1: policy has the lower selector count? outdated
if len(newPolicy.Spec.Selector.MatchLabels) < len(latestPolicy.Spec.Selector.MatchLabels) {
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand All @@ -644,7 +644,7 @@ func UpdateService(newPolicy types.KnoxNetworkPolicy, existingPolicies []types.K
}

// annotate the outdated fqdn policy
libs.UpdateOutdatedPolicy(latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, latestPolicy.Metadata["name"], newPolicy.Metadata["name"])
updated = true
}

Expand Down Expand Up @@ -793,7 +793,7 @@ func updateExistCIDRtoNewFQDN(existingPolicies []types.KnoxNetworkPolicy, newPol
}
}

libs.UpdateOutdatedPolicy(existCIDR.Metadata["name"], fqdnPolicy.Metadata["name"])
libs.UpdateOutdatedPolicy(Cfg.ConfigDB, existCIDR.Metadata["name"], fqdnPolicy.Metadata["name"])
}
}
}
Expand Down
Loading

0 comments on commit bd6d93d

Please sign in to comment.