Improve tags performance (#1526)

Co-authored-by: Erik De Neve <erik.de.neve@telenet.be>
grafana · Mar 21, 2023 · dd67988 · dd67988
1 parent 07c02ca
commit dd67988
Show file tree

Hide file tree

Showing 9 changed files with 191 additions and 103 deletions.
diff --git a/pkg/datasource/zabbix.go b/pkg/datasource/zabbix.go
@@ -7,9 +7,10 @@ import (
 	"github.com/alexanderzobnin/grafana-zabbix/pkg/timeseries"
 
 	"github.com/alexanderzobnin/grafana-zabbix/pkg/zabbix"
+	"golang.org/x/net/context"
+
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
-	"golang.org/x/net/context"
 )
 
 // ZabbixAPIQuery handles query requests to Zabbix API
@@ -52,7 +53,15 @@ func (ds *ZabbixDatasourceInstance) queryNumericItems(ctx context.Context, query
 	itemFilter := query.Item.Filter
 	showDisabled := query.Options.ShowDisabledItems
 
-	items, err := ds.zabbix.GetItems(ctx, groupFilter, hostFilter, appFilter, itemTagFilter, itemFilter, "num", showDisabled)
+	var items []*zabbix.Item
+	var err error
+	zabbixVersion, err := ds.zabbix.GetVersion(ctx)
+	if zabbixVersion >= 54 {
+		items, err = ds.zabbix.GetItems(ctx, groupFilter, hostFilter, itemTagFilter, itemFilter, "num", showDisabled)
+	} else {
+		items, err = ds.zabbix.GetItemsBefore54(ctx, groupFilter, hostFilter, appFilter, itemFilter, "num", showDisabled)
+	}
+
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/zabbix/methods.go b/pkg/zabbix/methods.go
@@ -2,6 +2,7 @@ package zabbix
 
 import (
 	"context"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -84,11 +85,49 @@ func (ds *Zabbix) GetItems(
 	ctx context.Context,
 	groupFilter string,
 	hostFilter string,
-	appFilter string,
 	itemTagFilter string,
 	itemFilter string,
 	itemType string,
 	showDisabled bool,
+) ([]*Item, error) {
+	var allItems []*Item
+	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
+	if err != nil {
+		return nil, err
+	}
+	if len(hosts) == 0 {
+		return allItems, nil
+	}
+
+	hostids := make([]string, 0)
+	for _, host := range hosts {
+		hostids = append(hostids, host.ID)
+	}
+
+	if isRegex(itemTagFilter) {
+		tags, err := ds.GetItemTags(ctx, groupFilter, hostFilter, itemTagFilter)
+		if err != nil {
+			return nil, err
+		}
+		var tagStrs []string
+		for _, t := range tags {
+			tagStrs = append(tagStrs, itemTagToString(t))
+		}
+		itemTagFilter = strings.Join(tagStrs, ",")
+	}
+	allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled, itemTagFilter)
+
+	return filterItemsByQuery(allItems, itemFilter)
+}
+
+func (ds *Zabbix) GetItemsBefore54(
+	ctx context.Context,
+	groupFilter string,
+	hostFilter string,
+	appFilter string,
+	itemFilter string,
+	itemType string,
+	showDisabled bool,
 ) ([]*Item, error) {
 	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
 	if err != nil {
@@ -100,80 +139,75 @@ func (ds *Zabbix) GetItems(
 	}
 
 	apps, err := ds.GetApps(ctx, groupFilter, hostFilter, appFilter)
-	// Apps not supported in Zabbix 5.4 and higher
-	isZabbix54orHigher := isAppMethodNotFoundError(err)
-	if isZabbix54orHigher {
-		apps = []Application{}
-	} else if err != nil {
+	if err != nil {
 		return nil, err
 	}
+
 	appids := make([]string, 0)
 	for _, app := range apps {
 		appids = append(appids, app.ID)
 	}
 
 	var allItems []*Item
 	if len(appids) > 0 {
-		allItems, err = ds.GetAllItems(ctx, nil, appids, itemType, showDisabled)
+		allItems, err = ds.GetAllItems(ctx, nil, appids, itemType, showDisabled, "")
 	} else if len(hostids) > 0 {
-		allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled)
-	}
-
-	if isZabbix54orHigher && itemTagFilter != "" {
-		allItems, err = filterItemsByTag(allItems, itemTagFilter)
-		if err != nil {
-			return nil, err
-		}
+		allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled, "")
 	}
 
 	return filterItemsByQuery(allItems, itemFilter)
 }
 
-func filterItemsByTag(items []*Item, filter string) ([]*Item, error) {
+func filterItemsByQuery(items []*Item, filter string) ([]*Item, error) {
 	re, err := parseFilter(filter)
 	if err != nil {
 		return nil, err
 	}
 
 	filteredItems := make([]*Item, 0)
 	for _, i := range items {
-		if len(i.Tags) == 0 && filter == "/.*/" {
-			filteredItems = append(filteredItems, i)
-		}
-
-		if len(i.Tags) > 0 {
-			tags := make([]string, 0)
-			for _, t := range i.Tags {
-				tags = append(tags, itemTagToString(t))
+		name := i.Name
+		if re != nil {
+			match, err := re.MatchString(name)
+			if err != nil {
+				return nil, err
 			}
-			for _, t := range tags {
-				if re != nil {
-					match, err := re.MatchString(t)
-					if err != nil {
-						return nil, err
-					}
-					if match {
-						filteredItems = append(filteredItems, i)
-						break
-					}
-				} else if t == filter {
-					filteredItems = append(filteredItems, i)
-					break
-				}
+			if match {
+				filteredItems = append(filteredItems, i)
 			}
+		} else if name == filter {
+			filteredItems = append(filteredItems, i)
 		}
+
 	}
 
 	return filteredItems, nil
 }
 
-func filterItemsByQuery(items []*Item, filter string) ([]*Item, error) {
+func (ds *Zabbix) GetApps(ctx context.Context, groupFilter string, hostFilter string, appFilter string) ([]Application, error) {
+	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
+	if err != nil {
+		return nil, err
+	}
+	hostids := make([]string, 0)
+	for _, host := range hosts {
+		hostids = append(hostids, host.ID)
+	}
+	allApps, err := ds.GetAllApps(ctx, hostids)
+	if err != nil {
+		return nil, err
+	}
+
+	return filterAppsByQuery(allApps, appFilter)
+}
+
+func filterAppsByQuery(items []Application, filter string) ([]Application, error) {
 	re, err := parseFilter(filter)
 	if err != nil {
 		return nil, err
 	}
 
-	filteredItems := make([]*Item, 0)
+	filteredItems := make([]Application, 0)
 	for _, i := range items {
 		name := i.Name
 		if re != nil {
@@ -193,7 +227,7 @@ func filterItemsByQuery(items []*Item, filter string) ([]*Item, error) {
 	return filteredItems, nil
 }
 
-func (ds *Zabbix) GetApps(ctx context.Context, groupFilter string, hostFilter string, appFilter string) ([]Application, error) {
+func (ds *Zabbix) GetItemTags(ctx context.Context, groupFilter string, hostFilter string, tagFilter string) ([]ItemTag, error) {
 	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
 	if err != nil {
 		return nil, err
@@ -202,32 +236,38 @@ func (ds *Zabbix) GetApps(ctx context.Context, groupFilter string, hostFilter st
 	for _, host := range hosts {
 		hostids = append(hostids, host.ID)
 	}
-	allApps, err := ds.GetAllApps(ctx, hostids)
-	if err != nil {
-		return nil, err
+
+	var allItems []*Item
+	itemType := "num"
+	showDisabled := false
+	allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled, "")
+
+	var allTags []ItemTag
+	for _, item := range allItems {
+		allTags = append(allTags, item.Tags...)
 	}
 
-	return filterAppsByQuery(allApps, appFilter)
+	return filterTags(allTags, tagFilter)
 }
 
-func filterAppsByQuery(items []Application, filter string) ([]Application, error) {
+func filterTags(items []ItemTag, filter string) ([]ItemTag, error) {
 	re, err := parseFilter(filter)
 	if err != nil {
 		return nil, err
 	}
 
-	filteredItems := make([]Application, 0)
+	filteredItems := make([]ItemTag, 0)
 	for _, i := range items {
-		name := i.Name
+		tagStr := itemTagToString(i)
 		if re != nil {
-			match, err := re.MatchString(name)
+			match, err := re.MatchString(tagStr)
 			if err != nil {
 				return nil, err
 			}
 			if match {
 				filteredItems = append(filteredItems, i)
 			}
-		} else if name == filter {
+		} else if tagStr == filter {
 			filteredItems = append(filteredItems, i)
 		}
 
@@ -314,7 +354,7 @@ func filterGroupsByQuery(items []Group, filter string) ([]Group, error) {
 	return filteredItems, nil
 }
 
-func (ds *Zabbix) GetAllItems(ctx context.Context, hostids []string, appids []string, itemtype string, showDisabled bool) ([]*Item, error) {
+func (ds *Zabbix) GetAllItems(ctx context.Context, hostids []string, appids []string, itemtype string, showDisabled bool, itemTagFilter string) ([]*Item, error) {
 	params := ZabbixAPIParams{
 		"output":         []string{"itemid", "name", "key_", "value_type", "hostid", "status", "state", "units", "valuemapid", "delay"},
 		"sortfield":      "name",
@@ -334,6 +374,24 @@ func (ds *Zabbix) GetAllItems(ctx context.Context, hostids []string, appids []st
 
 	if ds.version >= 54 {
 		params["selectTags"] = "extend"
+		if len(itemTagFilter) > 0 {
+			allTags := strings.Split(itemTagFilter, ",")
+			re := regexp.MustCompile(`(?m).*?([a-zA-Z0-9\s\-_]*):\s*([a-zA-Z0-9\-_\/:]*)`)
+			var tagsParams []map[string]string
+			for i := 0; i < len(allTags); i++ {
+				res := re.FindAllStringSubmatch(allTags[i], -1)
+				for i := range res {
+					tagParam := map[string]string{
+						"tag":      res[i][1],
+						"value":    res[i][2],
+						"operator": "1",
+					}
+					tagsParams = append(tagsParams, tagParam)
+				}
+			}
+			params["tags"] = tagsParams
+			params["evaltype"] = 2
+		}
 	}
 
 	if showDisabled == false {

diff --git a/pkg/zabbix/utils.go b/pkg/zabbix/utils.go
@@ -88,6 +88,11 @@ func parseFilter(filter string) (*regexp2.Regexp, error) {
 	return regexp2.Compile(pattern, regexp2.RE2)
 }
 
+func isRegex(filter string) bool {
+	regex := regexp.MustCompile(`^/(.+)/([imncsxrde]*)$`)
+	return regex.MatchString(filter)
+}
+
 func itemTagToString(tag ItemTag) string {
 	if tag.Value != "" {
 		return fmt.Sprintf("%s: %s", tag.Tag, tag.Value)

diff --git a/src/datasource/components/QueryEditor/MetricsQueryEditor.tsx b/src/datasource/components/QueryEditor/MetricsQueryEditor.tsx
@@ -70,14 +70,16 @@ export const MetricsQueryEditor = ({ query, datasource, onChange }: Props) => {
   }, [query.group.filter, query.host.filter]);
 
   const loadTagOptions = async (group: string, host: string) => {
-    if (!datasource.zabbix.isZabbix54OrHigher()) {
+    const tagsAvailable = await datasource.zabbix.isZabbix54OrHigher();
+    if (!tagsAvailable) {
       return [];
     }
 
     const groupFilter = datasource.replaceTemplateVars(group);
     const hostFilter = datasource.replaceTemplateVars(host);
     const items = await datasource.zabbix.getAllItems(groupFilter, hostFilter, null, null, {});
     const tags: ZBXItemTag[] = _.flatten(items.map((item: ZBXItem) => item.tags || []));
+    // const tags: ZBXItemTag[] = await datasource.zabbix.getItemTags(groupFilter, hostFilter, null);
 
     const tagList = _.uniqBy(tags, (t) => t.tag + t.value || '').map((t) => itemTagToString(t));
     let options: Array<SelectableValue<string>> = tagList?.map((tag) => ({

diff --git a/src/datasource/components/QueryEditor/TriggersQueryEditor.tsx b/src/datasource/components/QueryEditor/TriggersQueryEditor.tsx
@@ -85,7 +85,8 @@ export const TriggersQueryEditor = ({ query, datasource, onChange }: Props) => {
   }, [query.group.filter, query.host.filter]);
 
   const loadTagOptions = async (group: string, host: string) => {
-    if (!datasource.zabbix.isZabbix54OrHigher()) {
+    const tagsAvailable = await datasource.zabbix.isZabbix54OrHigher();
+    if (!tagsAvailable) {
       return [];
     }
 

diff --git a/src/datasource/components/VariableQueryEditor.tsx b/src/datasource/components/VariableQueryEditor.tsx
@@ -87,10 +87,10 @@ export class ZabbixVariableQueryEditor extends PureComponent<VariableQueryProps,
     this.props.onChange(queryModel, `Zabbix - ${queryType}`);
   };
 
-  render() {
+  async render() {
     const { selectedQueryType, legacyQuery, group, host, application, itemTag, item } = this.state;
     const { datasource } = this.props;
-    const supportsItemTags = datasource?.zabbix?.isZabbix54OrHigher() || false;
+    const supportsItemTags = (await datasource?.zabbix?.isZabbix54OrHigher()) || false;
 
     return (
       <>

diff --git a/src/datasource/utils.ts b/src/datasource/utils.ts
@@ -41,7 +41,7 @@ export function expandItemName(name: string, key: string): string {
   return name;
 }
 
-export function expandItems(items) {
+export function expandItems(items: any[]) {
   _.forEach(items, (item) => {
     item.item = item.name;
     item.name = expandItemName(item.item, item.key_);