Skip to content

Commit

Permalink
Update L3/L7 aggregation level
Browse files Browse the repository at this point in the history 
Description

Fixes: #101
  • Loading branch information
@seungsoo-lee
seungsoo-lee committed Jan 24, 2021
1 parent ceef5d3 commit 5dcd639
Show file tree
Hide file tree
Showing 8 changed files with 139 additions and 140 deletions.
2 changes: 0 additions & 2 deletions database/mysql/init/flow_management.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,5 @@ CREATE TABLE IF NOT EXISTS `auto_policy_config` (
`l3_aggregation_level` int DEFAULT NULL,
`l4_aggregation_level` int DEFAULT NULL,
`l7_aggregation_level` int DEFAULT NULL,
`http_url_threshold` int DEFAULT NULL,

PRIMARY KEY (`id`)
);
12 changes: 10 additions & 2 deletions src/core/configManager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ var Cfg types.Configuration
// SkipNamespaces ...
var SkipNamespaces []string

// HTTPUrlThreshold
var HTTPUrlThreshold int

func init() {
// initially, default -> applied
LoadDefaultConfig()
Expand Down Expand Up @@ -116,9 +119,14 @@ func LoadDefaultConfig() {

// aggregation level
Cfg.L3AggregationLevel = 3
Cfg.L4AggregationLevel = 3
Cfg.L4Compression = 3
Cfg.L7AggregationLevel = 3
Cfg.HTTPUrlThreshold = 3

if Cfg.L7AggregationLevel == 3 {
HTTPUrlThreshold = 3
} else if Cfg.L7AggregationLevel == 2 {
HTTPUrlThreshold = 5
}
}

// AddConfiguration function
Expand Down
9 changes: 7 additions & 2 deletions src/core/httpAggregator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ func (n *Node) aggregateChildNodes() {
}

// step 1: #child nodes > threshold
if len(n.childNodes) > Cfg.HTTPUrlThreshold {
if len(n.childNodes) > HTTPUrlThreshold {
childPaths := []string{}
for _, childNode := range n.childNodes {
childPaths = append(childPaths, childNode.path)
Expand Down Expand Up @@ -433,7 +433,7 @@ func aggreateHTTPPathsNaive(paths []string) []string {

for key, paths := range depthToPaths {
// if threshold over, aggregate it
if len(paths) >= Cfg.HTTPUrlThreshold {
if len(paths) >= HTTPUrlThreshold {
aggregatedPaths = append(aggregatedPaths, key+"/.*")
} else {
for _, path := range paths {
Expand Down Expand Up @@ -480,6 +480,11 @@ func AggregatePaths(treeMap map[string]*Node, paths []string) []string {

// AggregateHTTPRules function
func AggregateHTTPRules(mergedSrcPerMergedDst map[string][]MergedPortDst) {
// if level 1, do not aggregate http path
if Cfg.L7AggregationLevel == 1 {
return
}

for mergedSrc, dsts := range mergedSrcPerMergedDst {
for i, dst := range dsts {
// check if dst is for HTTP rules
Expand Down
104 changes: 54 additions & 50 deletions src/core/networkPolicy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1397,62 +1397,64 @@ func mergingDstSpecs(mergedSrcsPerDst map[Dst][]string) map[string][]MergedPortD
}
}

for mergedSrc, dsts := range dstsPerMergedSrc {
// convert dst -> dstSimple, and count each dstSimple
dstSimpleCounts := map[DstSimple]int{}

for _, dst := range dsts {
dstSimple := DstSimple{Namespace: dst.Namespace,
PodName: dst.PodName,
Additional: dst.Additional,
Action: dst.Action}

if val, ok := dstSimpleCounts[dstSimple]; !ok {
dstSimpleCounts[dstSimple] = 1
} else {
dstSimpleCounts[dstSimple] = val + 1
if Cfg.L4Compression == 1 {
for mergedSrc, dsts := range dstsPerMergedSrc {
// convert dst -> dstSimple, and count each dstSimple
dstSimpleCounts := map[DstSimple]int{}

for _, dst := range dsts {
dstSimple := DstSimple{Namespace: dst.Namespace,
PodName: dst.PodName,
Additional: dst.Additional,
Action: dst.Action}

if val, ok := dstSimpleCounts[dstSimple]; !ok {
dstSimpleCounts[dstSimple] = 1
} else {
dstSimpleCounts[dstSimple] = val + 1
}
}
}

// sort dstCount by descending order
type dstCount struct {
DstSimple DstSimple
Count int
}

var dstCounts []dstCount
for dst, count := range dstSimpleCounts {
dstCounts = append(dstCounts, dstCount{dst, count})
}
// sort dstCount by descending order
type dstCount struct {
DstSimple DstSimple
Count int
}

sort.Slice(dstCounts, func(i, j int) bool {
return dstCounts[i].Count > dstCounts[j].Count
})
var dstCounts []dstCount
for dst, count := range dstSimpleCounts {
dstCounts = append(dstCounts, dstCount{dst, count})
}

if mergedSrcPerMergedDst[mergedSrc] == nil {
mergedSrcPerMergedDst[mergedSrc] = []MergedPortDst{}
}
sort.Slice(dstCounts, func(i, j int) bool {
return dstCounts[i].Count > dstCounts[j].Count
})

// if dst is matched dstSimple, remove it from origin dst list
for _, dstCount := range dstCounts {
if dstCount.Count >= 2 { // at least match count >= 2
for _, dst := range dsts {
simple := DstSimple{Namespace: dst.Namespace,
PodName: dst.PodName,
Additional: dst.Additional,
Action: dst.Action}
if mergedSrcPerMergedDst[mergedSrc] == nil {
mergedSrcPerMergedDst[mergedSrc] = []MergedPortDst{}
}

if dstCount.DstSimple == simple {
// merge protocol + port
mergedSrcPerMergedDst[mergedSrc] = mergingProtocolPorts(mergedSrcPerMergedDst[mergedSrc], dst)
// and then, remove dst
dsts = removeDstFromSlice(dsts, dst)
// if dst is matched dstSimple, remove it from origin dst list
for _, dstCount := range dstCounts {
if dstCount.Count >= 2 { // at least match count >= 2
for _, dst := range dsts {
simple := DstSimple{Namespace: dst.Namespace,
PodName: dst.PodName,
Additional: dst.Additional,
Action: dst.Action}

if dstCount.DstSimple == simple {
// merge protocol + port
mergedSrcPerMergedDst[mergedSrc] = mergingProtocolPorts(mergedSrcPerMergedDst[mergedSrc], dst)
// and then, remove dst
dsts = removeDstFromSlice(dsts, dst)
}
}
}
}
}

dstsPerMergedSrc[mergedSrc] = dsts
dstsPerMergedSrc[mergedSrc] = dsts
}
}

// if not merged dsts remains, append it by default
Expand All @@ -1462,11 +1464,13 @@ func mergingDstSpecs(mergedSrcsPerDst map[Dst][]string) map[string][]MergedPortD
}
}

// fqdn merging
mergeFQDN(mergedSrcPerMergedDst)
if Cfg.L4Compression == 1 {
// fqdn merging
mergeFQDN(mergedSrcPerMergedDst)

// cidr merging
mergeCIDR(mergedSrcPerMergedDst)
// cidr merging
mergeCIDR(mergedSrcPerMergedDst)
}

// entities merged (for Cilium)
mergeEntities(mergedSrcPerMergedDst)
Expand Down
17 changes: 6 additions & 11 deletions src/libs/mysqlHandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,8 @@ func AddConfiguration(cfg types.ConfigDB, newConfig types.Configuration) error {
"ignoring_flows," +
"l3_aggregation_level," +
"l4_aggregation_level," +
"l7_aggregation_level," +
"http_url_threshold) " +
"values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
"l7_aggregation_level) " +
"values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")

if err != nil {
return err
Expand Down Expand Up @@ -367,9 +366,8 @@ func AddConfiguration(cfg types.ConfigDB, newConfig types.Configuration) error {
newConfig.CIDRBits,
ignoringFlows,
newConfig.L3AggregationLevel,
newConfig.L4AggregationLevel,
newConfig.L4Compression,
newConfig.L7AggregationLevel,
newConfig.HTTPUrlThreshold,
)

if err != nil {
Expand Down Expand Up @@ -431,9 +429,8 @@ func GetConfigurations(cfg types.ConfigDB, configName string) ([]types.Configura
&cfg.CIDRBits,
&ignoringFlowByte,
&cfg.L3AggregationLevel,
&cfg.L4AggregationLevel,
&cfg.L4Compression,
&cfg.L7AggregationLevel,
&cfg.HTTPUrlThreshold,
); err != nil {
return nil, err
}
Expand Down Expand Up @@ -482,8 +479,7 @@ func UpdateConfiguration(cfg types.ConfigDB, configName string, updateConfig typ
"ignoring_flows=?," +
"l3_aggregation_level=?," +
"l4_aggregation_level=?," +
"l7_aggregation_level=?," +
"http_url_threshold=? " +
"l7_aggregation_level=? " +
"WHERE config_name=?")

if err != nil {
Expand Down Expand Up @@ -524,9 +520,8 @@ func UpdateConfiguration(cfg types.ConfigDB, configName string, updateConfig typ
updateConfig.CIDRBits,
ignoringFlows,
updateConfig.L3AggregationLevel,
updateConfig.L4AggregationLevel,
updateConfig.L4Compression,
updateConfig.L7AggregationLevel,
updateConfig.HTTPUrlThreshold,
configName,
)

Expand Down
Loading

0 comments on commit 5dcd639

Please sign in to comment.