Skip to content

Commit

Permalink
refactored and timeout increase
Browse files Browse the repository at this point in the history 
Signed-off-by: Vivek Shankar <vshankar@progress.com>
  • Loading branch information
@vivekshankar1
vivekshankar1 committed Oct 25, 2021
1 parent d41b5d6 commit 40c6ffc
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 129 deletions.
2 changes: 1 addition & 1 deletion .expeditor/verify_private.pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -717,7 +717,7 @@ steps:
- label: "ontop backup"
command:
- integration/run_test integration/tests/backup_ontop.sh
timeout_in_minutes: 25
timeout_in_minutes: 35
expeditor:
executor:
linux:
Expand Down
214 changes: 112 additions & 102 deletions components/nodemanager-service/managers/azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,10 +551,7 @@ func isNameMaching(s string, arr []string, exclude bool) bool {
if len(arr) > 0 {
initBool = false
for _, val := range arr {
initBool = initBool || strings.Contains(
strings.ToLower(s),
strings.ToLower(val),
)
initBool = initBool || isMatch(s, val)
}
}
}
Expand Down Expand Up @@ -746,16 +743,61 @@ func handleTagFilters(filters []*common.Filter) (map[string][]string, map[string
return includeTags, excludeTags
}

func isMatch(s string, p string) bool {

runeInput := []rune(strings.ToLower(s))
runePattern := []rune(strings.ToLower(p))

lenInput := len(runeInput)
lenPattern := len(runePattern)

isMatchingMatrix := make([][]bool, lenInput+1)

for i := range isMatchingMatrix {
isMatchingMatrix[i] = make([]bool, lenPattern+1)
}

isMatchingMatrix[0][0] = true
for i := 1; i < lenInput; i++ {
isMatchingMatrix[i][0] = false
}

if lenPattern > 0 {
if runePattern[0] == '*' {
isMatchingMatrix[0][1] = true
}
}

for j := 2; j <= lenPattern; j++ {
if runePattern[j-1] == '*' {
isMatchingMatrix[0][j] = isMatchingMatrix[0][j-1]
}

}

for i := 1; i <= lenInput; i++ {
for j := 1; j <= lenPattern; j++ {

if runePattern[j-1] == '*' {
isMatchingMatrix[i][j] = isMatchingMatrix[i-1][j] || isMatchingMatrix[i][j-1]
}

if runePattern[j-1] == '?' || runeInput[i-1] == runePattern[j-1] {
isMatchingMatrix[i][j] = isMatchingMatrix[i-1][j-1]
}
}
}

return isMatchingMatrix[lenInput][lenPattern]
}

func filterIncludeTags(includeTags map[string][]string, subsList []*manager.ManagerNode) []*manager.ManagerNode {
if len(includeTags) > 0 {
var filteredsubsList = []*manager.ManagerNode{}
for _, val := range subsList {
if sArray, ok := includeTags["name"]; ok {
for _, v := range sArray {
if strings.Contains(
strings.ToLower(val.Name),
strings.ToLower(v),
) {
if isMatch(val.Name, v) {
filteredsubsList = append(filteredsubsList, val)
}
}
Expand Down Expand Up @@ -875,15 +917,7 @@ func contains(s []string, e string) bool {
func (creds *Creds) QueryVMs(ctx context.Context, filters []*common.Filter) (map[string][]*manager.ManagerNode, error) {
var err error
vmList := make(map[string][]*manager.ManagerNode, 0)
subs := make([]*manager.ManagerNode, 0)
for _, filter := range filters {
if filter.Key == "subscription_id" {
subs = make([]*manager.ManagerNode, 0)
for _, subID := range filter.Values {
subs = append(subs, &manager.ManagerNode{Id: subID})
}
}
}
subs := filterBySubID(filters)
if len(subs) == 0 {
subs, err = creds.GetSubscriptions(ctx, filters)
if err != nil {
Expand Down Expand Up @@ -916,19 +950,11 @@ func (creds *Creds) QueryVMs(ctx context.Context, filters []*common.Filter) (map
func (creds *Creds) QueryApis(ctx context.Context, filters []*common.Filter) (map[string][]*manager.ManagerNode, error) {
var err error
vmList := make(map[string][]*manager.ManagerNode, 0)
subs := make([]*manager.ManagerNode, 0)
for _, filter := range filters {
if filter.Key == "subscription_id" {
subs = make([]*manager.ManagerNode, 0)
for _, subID := range filter.Values {
subs = append(subs, &manager.ManagerNode{Id: subID})
}
}
}
subs := filterBySubID(filters)
if len(subs) == 0 {
subs, err = creds.GetSubscriptionsForApi(ctx, filters)
if err != nil {
return vmList, errors.Wrap(err, "QueryVMs unable to list subscriptions")
return vmList, errors.Wrap(err, "QueryAPIs unable to list subscriptions")
}
}
for _, v := range subs {
Expand All @@ -937,6 +963,19 @@ func (creds *Creds) QueryApis(ctx context.Context, filters []*common.Filter) (ma
return vmList, nil
}

func filterBySubID(filters []*common.Filter) []*manager.ManagerNode {
subs := make([]*manager.ManagerNode, 0)
for _, filter := range filters {
if filter.Key == "subscription_id" {
subs = make([]*manager.ManagerNode, 0)
for _, subID := range filter.Values {
subs = append(subs, &manager.ManagerNode{Id: subID})
}
}
}
return subs
}

// QueryVMState returns an array of ManagerNodes, one for each vm in the account, over all subscriptions
func (creds *Creds) QueryVMState(ctx context.Context) ([]pgdb.InstanceState, error) {
var err error
Expand Down Expand Up @@ -969,7 +1008,7 @@ func (creds *Creds) QueryVMState(ctx context.Context) ([]pgdb.InstanceState, err
}

// QueryField returns account tags, locations, subscriptions
func (creds *Creds) QueryField(ctx context.Context, filters []*common.Filter, field string) ([]string, error) {
func (creds *Creds) QueryField(ctx context.Context, filters []*common.Filter, field string, mgrType string) ([]string, error) {
var err error
resultArray := make([]string, 0)
subs := make([]*manager.ManagerNode, 0)
Expand All @@ -994,13 +1033,19 @@ func (creds *Creds) QueryField(ctx context.Context, filters []*common.Filter, fi
return nil, errors.Wrap(err, "QueryField unable to list locations")
}
case "names", "tags:name":
vmsResult, err := creds.QueryVMs(ctx, []*common.Filter{})
if err != nil {
return nil, errors.Wrap(err, "QueryField unable to query vms")
}
for _, vms := range vmsResult {
for _, vm := range vms {
resultArray = append(resultArray, vm.Name)
if mgrType == "azure-api" {
for _, sub := range subs {
resultArray = append(resultArray, sub.Name)
}
} else {
vmsResult, err := creds.QueryVMs(ctx, []*common.Filter{})
if err != nil {
return nil, errors.Wrap(err, "QueryField unable to query vms")
}
for _, vms := range vmsResult {
for _, vm := range vms {
resultArray = append(resultArray, vm.Name)
}
}
}
case "tags":
Expand All @@ -1018,15 +1063,9 @@ func (creds *Creds) QueryField(ctx context.Context, filters []*common.Filter, fi
}
default:
if strings.HasPrefix(field, "tags:") {
tags, err := creds.ListTags(ctx, subs)
resultArray, err = queryDefaultFieldTags(ctx, mgrType, subs, field, resultArray, creds)
if err != nil {
return nil, errors.Wrap(err, "QueryField unable to list tags")
}
key := strings.TrimPrefix(field, "tags:")
for k, v := range tags {
if k == key {
resultArray = append(resultArray, v...)
}
return nil, err
}
} else {
return resultArray, errorutils.ProcessInvalid(nil, fmt.Sprintf("invalid filter field %s", field))
Expand All @@ -1036,78 +1075,49 @@ func (creds *Creds) QueryField(ctx context.Context, filters []*common.Filter, fi
return resultArray, nil
}

func UniqueString(a []string) []string {
seen := map[string]bool{}
var b []string
for _, v := range a {
if _, ok := seen[v]; !ok {
seen[v] = true
b = append(b, v)
}
}
return b
}

func (creds *Creds) QueryFieldApi(ctx context.Context, filters []*common.Filter, field string) ([]string, error) {
var err error
resultArray := make([]string, 0)
subs := make([]*manager.ManagerNode, 0)
for _, filter := range filters {
if filter.Key == "subscription_id" {
subs = make([]*manager.ManagerNode, 0)
for _, subID := range filter.Values {
subs = append(subs, &manager.ManagerNode{Id: subID})
func queryDefaultFieldTags(ctx context.Context, mgrType string, subs []*manager.ManagerNode, field string, resultArray []string, creds *Creds) ([]string, error) {
key := strings.TrimPrefix(field, "tags:")
if mgrType == "azure-api" {
if field == "tags:Name" {
for _, sub := range subs {
resultArray = append(resultArray, sub.Name)
}
}
}
if len(subs) == 0 {
subs, err = creds.ListSubscriptions(ctx)
if err != nil {
return nil, errors.Wrap(err, "QueryField unable to list subscriptions")
}
}
switch field {
case "regions":
resultArray, err = creds.ListLocations(ctx, subs)
if err != nil {
return nil, errors.Wrap(err, "QueryField unable to list locations")
}
case "names", "tags:name", "tags:Name":
for _, sub := range subs {
resultArray = append(resultArray, sub.Name)
key := strings.TrimPrefix(field, "tags:")
for _, v := range subs {
for _, val := range v.Tags {
if val.Key == key {
resultArray = append(resultArray, val.Value)
}
}
}
case "tags":
resultArray = UniqueString(resultArray)
} else {
tags, err := creds.ListTags(ctx, subs)
if err != nil {
return nil, errors.Wrap(err, "QueryField unable to list tags")
}
for k := range tags {
resultArray = append(resultArray, k)
}

resultArray = append(resultArray, "name")
case "subscriptions":
for _, sub := range subs {
resultArray = append(resultArray, fmt.Sprintf("%s:%s", sub.Name, sub.Id))
}
default:
if strings.HasPrefix(field, "tags:") {
key := strings.TrimPrefix(field, "tags:")
for _, v := range subs {
for _, val := range v.Tags {
if val.Key == key {
resultArray = append(resultArray, val.Value)
}
}
for k, v := range tags {
if k == key {
resultArray = append(resultArray, v...)
}
} else {
return resultArray, errorutils.ProcessInvalid(nil, fmt.Sprintf("invalid filter field %s", field))
}
}
resultArray = UniqueString(resultArray)
return resultArray, nil
}

func UniqueString(a []string) []string {
seen := map[string]bool{}
var b []string
for _, v := range a {
if _, ok := seen[v]; !ok {
seen[v] = true
b = append(b, v)
}
}
return b
}

func uniqueStringSliceCaseInsens(stringSlice []string) []string {
keys := make(map[string]bool)
list := []string{}
Expand Down
10 changes: 5 additions & 5 deletions components/nodemanager-service/managers/azure/azure_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ func TestHandleReFilterNameAndRegion(t *testing.T) {
assert.Equal(t, 0, len(filteredResources))

filters = []*common.Filter{
{Key: "name", Values: []string{"test-1"}},
{Key: "name", Values: []string{"test-1*"}},
}
filteredResources = reFilterNameAndRegion(filters, vmList)
assert.Equal(t, 1, len(filteredResources))
Expand All @@ -237,14 +237,14 @@ func TestHandleReFilterNameAndRegion(t *testing.T) {

filters = []*common.Filter{
{Key: "region", Values: []string{"eastus"}},
{Key: "name", Values: []string{"test-4"}},
{Key: "name", Values: []string{"test-4*"}},
}
filteredResources = reFilterNameAndRegion(filters, vmList)
assert.Equal(t, 1, len(filteredResources))

filters = []*common.Filter{
{Key: "region", Values: []string{"eastus"}},
{Key: "name", Values: []string{"test"}},
{Key: "name", Values: []string{"test*"}},
{Key: "name", Values: []string{"test-1A"}, Exclude: true},
}
filteredResources = reFilterNameAndRegion(filters, vmList)
Expand All @@ -260,7 +260,7 @@ func TestHandleReFilterNameAndRegion(t *testing.T) {
filters = []*common.Filter{
{Key: "region", Values: []string{"eastus"}},
{Key: "region", Values: []string{"eastus2"}},
{Key: "name", Values: []string{"test"}},
{Key: "name", Values: []string{"test*"}},
{Key: "name", Values: []string{"test-1A"}, Exclude: true},
}
filteredResources = reFilterNameAndRegion(filters, vmList)
Expand All @@ -281,7 +281,7 @@ func TestHandleReFilterNameAndRegion(t *testing.T) {
}},
}
filters = []*common.Filter{
{Key: "name", Values: []string{"ljkp"}, Exclude: false},
{Key: "name", Values: []string{"ljkp*"}, Exclude: false},
{Key: "name", Values: []string{"ljkp-win"}, Exclude: true},
{Key: "region", Values: []string{"eastus"}, Exclude: false},
}
Expand Down
25 changes: 4 additions & 21 deletions components/nodemanager-service/server/manager/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -405,10 +405,8 @@ func (srv *Server) SearchNodeFields(ctx context.Context, in *manager.FieldQuery)
switch mgr.Type {
case "aws-ec2", "aws-api":
return srv.searchAwsFields(ctx, in)
case "azure-vm":
return srv.searchAzureFields(ctx, in)
case "azure-api":
return srv.searchAzureApiFields(ctx, in)
case "azure-vm", "azure-api":
return srv.searchAzureFields(ctx, in, mgr.Type)
case "gcp-api":
return srv.searchGenericNodesFields(ctx, in, in.NodeManagerId)
case "automate":
Expand All @@ -418,30 +416,15 @@ func (srv *Server) SearchNodeFields(ctx context.Context, in *manager.FieldQuery)
}
}

func (srv *Server) searchAzureFields(ctx context.Context, in *manager.FieldQuery) (*manager.Fields, error) {
myazure, err := managers.GetAzureManagerFromID(ctx, in.NodeManagerId, srv.DB, srv.secretsClient)
if err != nil {
return nil, errorutils.FormatErrorMsg(err, in.NodeManagerId)
}
if in.Query == nil {
return nil, &errorutils.InvalidError{Msg: "Please provide a query."}
}
results, err := myazure.QueryField(ctx, in.GetQuery().GetFilterMap(), in.Field)
if err != nil {
return nil, errorutils.FormatErrorMsg(err, "")
}
return &manager.Fields{Fields: results}, nil
}

func (srv *Server) searchAzureApiFields(ctx context.Context, in *manager.FieldQuery) (*manager.Fields, error) {
func (srv *Server) searchAzureFields(ctx context.Context, in *manager.FieldQuery, mgrType string) (*manager.Fields, error) {
myazure, err := managers.GetAzureManagerFromID(ctx, in.NodeManagerId, srv.DB, srv.secretsClient)
if err != nil {
return nil, errorutils.FormatErrorMsg(err, in.NodeManagerId)
}
if in.Query == nil {
return nil, &errorutils.InvalidError{Msg: "Please provide a query."}
}
results, err := myazure.QueryFieldApi(ctx, in.GetQuery().GetFilterMap(), in.Field)
results, err := myazure.QueryField(ctx, in.GetQuery().GetFilterMap(), in.Field, mgrType)
if err != nil {
return nil, errorutils.FormatErrorMsg(err, "")
}
Expand Down

0 comments on commit 40c6ffc

Please sign in to comment.