Labels support for pmwebd and pmdaprometheus

src/pmdas/prometheus/pmdaprometheus.python

@@ -373,7 +373,7 @@ class Metric(object):
 				for instance in self.instances:
 					self.instance_by_name[instance["name"]] = {
 						"name": instance["prometheus_name"],
-						"label": instance["label"]
+						"label": instance["labels"]
 					}
 		except ValueError as e:
 			raise e
@@ -395,7 +395,7 @@ class Metric(object):
 			self.obj = pmdaMetric(self.pmid, self.type, c_api.PM_INDOM_NULL,
 								  self.sem, self.__units_val)
 
-	def label_func(self, inst):
+	def label_func(self, inst=c_api.PM_IN_NULL):
 		''' Label value of this instance '''
 		ret_label = "{}"
 		if self.__indom_cache:
@@ -733,6 +733,7 @@ class PrometheusSource(object):
 					break
 			indom = IndomCache(indom_idx, MAX_INDOM, self.pmda)
 			metric.indom_cache = indom
+			self.pmda.metrics_by_indom[indom.indom] = metric
 		# Stores the cached indoms
 		self.pmda.indom_cache.refresh()
 
@@ -771,20 +772,23 @@ class PrometheusSource(object):
 		self.pmda.add_metric(metric.full_name, metric.obj,
 								   metric.description)
 		self.metrics_by_id[metric.idx] = metric
+		self.pmda.metrics_by_pmid[metric.pmid] = metric
 
 	def __remove_metric(self, name):
 		try:
 			id = self.__metric_cache.lookup_name(name)
 			metric = self.metrics_by_id[id]
 			self.pmda.remove_metric(metric.full_name, metric.obj)
 			self.metrics_by_id.pop(id)
+			self.pmda.metrics_by_pmid.pop(metric.pmid)
 		except KeyError:
 			pass
 
 	def remove_all(self):
 		for id in self.metrics_by_id:
 			metric = self.metrics_by_id[id]
 			self.pmda.remove_metric(metric.full_name, metric.obj)
+			self.pmda.metrics_by_pmid.pop(metric.pmid)
 		self.metrics_by_id = {}
 
 	def parse_metrics_values(self, raw_data):
@@ -871,6 +875,10 @@ class PrometheusPMDA(PMDA):
 		self.source_by_name = {}
 		self.source_by_path = {}
 
+		# PMID to metrics map used for labels (callback)
+		self.metrics_by_pmid = {}
+		self.metrics_by_indom = {}
+
 		# Cluster cache stores the indoms of clusters
 		# Indom values start from 2 as Indom 1 is reserved for internal metrics
 		# Indom 2 is reserved for caching  the indoms metrics with indoms
@@ -906,6 +914,7 @@ class PrometheusPMDA(PMDA):
 		metric_info.create()
 		self.add_metric(metric_info.full_name, metric_info.obj,
 						metric_info.description)
+		self.metrics_by_pmid[metric_info.pmid] = metric_info
 
 		metric_info = Metric(self.pmda_name + '.config', 0, self)
 		metric_info.name = 'metadata_refresh_frequency'
@@ -915,6 +924,7 @@ class PrometheusPMDA(PMDA):
 		metric_info.create()
 		self.add_metric(metric_info.full_name, metric_info.obj,
 						metric_info.description)
+		self.metrics_by_pmid[metric_info.pmid] = metric_info
 
 		metric_info = Metric(self.pmda_name + '.config', 0, self)
 		metric_info.name = 'endpoint_fetch_timeout'
@@ -924,6 +934,8 @@ class PrometheusPMDA(PMDA):
 		metric_info.create()
 		self.add_metric(metric_info.full_name, metric_info.obj,
 						metric_info.description)
+		self.metrics_by_pmid[metric_info.pmid] = metric_info
+
 
 	def load_prometheus_sources(self):
 		''' Initializes the PrometheusSource for all the Prometheus sources'''
@@ -992,22 +1004,20 @@ class PrometheusPMDA(PMDA):
 		self.numfetch += 1
 
 	def prometheus_label(self, ident, type):
-		# self.log("Label: ident: + str(ident) + ", type: " + str(type))
-		#if ident == c_api.PM_LABEL_CLUSTER:	# ident is cluster
-		#	if ident in self.source_by_cluster:
-		#		return self.source_by_cluster[ident].labels
-		#elif ident == c_api.PM_LABEL_ITEM:	# ident is PMID
-		#	# Note: these labels are for metrics
-		#	return self.metrics_by_id[ident].label_func()
+		if type == c_api.PM_LABEL_ITEM:
+			if ident in self.metrics_by_pmid:
+				return self.metrics_by_pmid[ident].label_func()
+			else:
+				self.log("Invalid pmid:%d requested"
+					"on label with type:%d"%(ident, type))
 		return "{}"
 
 	def prometheus_label_callback(self, indom, inst):
-		# self.log("label_cb: indom: " + str(indom) + ", inst: " + str(inst))
-		# Note: ident is now an InDom identifier and not a PMID
-		#if ident in self.indoms_by_id:
-		#	# TODO: maybe needs to become metrics_by_indom[ident]?
-		#	#       or does the IndomCache need a label_func()?
-		#	return self.metrics_by_id[ident].label_func(inst)
+		if indom in self.metrics_by_indom:
+			return self.metrics_by_indom[indom].label_func(inst)
+		else:
+			self.log("Invalid indom:%d requested "
+				"on label callback with inst:%d"%(indom, inst))
 		return "{}"
 
 	def prometheus_fetch_callback(self, cluster, item, inst):

src/pmwebapi/pmwebapi.cxx

@@ -434,13 +434,20 @@ metric_list_traverse (const char *metric, void *closure)
         /* Quietly skip this metric. */
         return;
     }
+    pmLabelSet *labelSet;
+    string label = "{}";
+    rc = pmGetItemLabels(metric_id, &labelSet);
+    if (rc < 0) {
+        return;
+    } else if (rc != 0 && labelSet->jsonlen > 0) {
+        label = labelSet->json;
+    }
 
     if (mltc->num_metrics > 0) {
         *mltc->mhdb << ",\n";
     }
 
     *mltc->mhdb << "{";
-
     json_key_value (*mltc->mhdb, "name", string (metric), ",");
     rc = pmLookupText (metric_id, PM_TEXT_ONELINE, &metric_text);
     if (rc == 0) {
@@ -460,8 +467,8 @@ metric_list_traverse (const char *metric, void *closure)
                     string (metric_desc.sem == PM_SEM_COUNTER ? "counter" : metric_desc.sem == PM_SEM_INSTANT
                             ? "instant" : metric_desc.sem == PM_SEM_DISCRETE ? "discrete" : "unknown"), ",");
     json_key_value (*mltc->mhdb, "units", string (pmUnitsStr_r (&metric_desc.units, buffer, sizeof (buffer))), ",");
-    json_key_value (*mltc->mhdb, "type", string (pmTypeStr_r (metric_desc.type, buffer, sizeof (buffer))), "");
-
+    json_key_value (*mltc->mhdb, "type", string (pmTypeStr_r (metric_desc.type, buffer, sizeof (buffer))), ",");
+    *mltc->mhdb << "\"label\":" << label;
     *mltc->mhdb << "}";
     mltc->num_metrics++;
 }
@@ -999,12 +1006,28 @@ pmwebapi_respond_instance_list (struct MHD_Connection *connection,
         instlist = instances;
     }
 
+    pmLabelSet *labelSet;
+    int label_instances;
+    label_instances = pmGetInstancesLabels(inDom, &labelSet);
+    if (label_instances < 0)
+        goto out;
+
     output << "{";
     json_key_value (output, "indom", inDom, ",");
 
     output << "\"instances\":[\n";
     printed_instances = 0;
     for (i = 0; i < num_instances; i++) {
+        string label = "{}";
+        int j;
+        pmLabelSet *temp = labelSet;
+        for (j = 0; j < label_instances; j++) {
+            if ( (int) temp->inst == instlist[i] )
+                break;
+            temp++;
+        }
+        if (j < label_instances && temp->jsonlen > 0)
+            label = temp->json;
         char *instance_name;
         if (namelist != NULL) {
             instance_name = namelist[i];
@@ -1021,7 +1044,8 @@ pmwebapi_respond_instance_list (struct MHD_Connection *connection,
 
         output << "{";
         json_key_value (output, "instance", instlist[i], ",");
-        json_key_value (output, "name", string (instance_name));
+        json_key_value (output, "name", string (instance_name), ",");
+        output << "\"label\":" << label;
         output << "}";
         if (namelist == NULL) {
             free (instance_name);
@@ -1109,8 +1133,9 @@ metric_prometheus_traverse (const char *metric, void *closure)
         metric_id = c->metric_id_cache.find(metric)->second;
     } else {
         rc = pmLookupName (1, metrics, &metric_id);
-        if (rc != 1)
+        if (rc != 1) {
             return; // skip quietly
+        }
         c->metric_id_cache[metric] = metric_id;
         c->metric_name_cache[metric_id] = metric;
     }
@@ -1120,8 +1145,9 @@ metric_prometheus_traverse (const char *metric, void *closure)
 
     if (c->metric_desc_cache.find(metric_id) == c->metric_desc_cache.end()) {
         rc = pmLookupDesc (metric_id, &metric_desc);
-        if (rc != 0)
+        if (rc != 0) {
             return; // skip quietly
+        }
         c->metric_desc_cache[metric_id] = metric_desc;
     }
 
@@ -1133,7 +1159,7 @@ metric_prometheus_traverse (const char *metric, void *closure)
         }
         c->metric_text_cache[metric_id] = help_text;
     }
-
+    cout << metric_id<< endl;
     mptc->pmids.push_back(metric_id);
 }
 
@@ -1183,8 +1209,6 @@ metric_prometheus_batch_fetch(void *closure) {
         string pn = metric;
         replace (pn.begin(), pn.end(), '.', ':');
 
-        output << "# HELP " << pn << " " << help_text << endl;
-
         // Append a metric_desc.units-based suffix, and compute an
         // pmConvScale vector to match conventions as per
         // https://prometheus.io/docs/practices/naming/
@@ -1218,13 +1242,25 @@ metric_prometheus_batch_fetch(void *closure) {
         }
 
         // append semantics tag
-        if (metric_desc.sem == PM_SEM_COUNTER) {
+        if (metric_desc.sem == PM_SEM_COUNTER)
             pn += "_total";
+
+        output << "# HELP " << pn << " " << help_text << endl;
+
+        if (metric_desc.sem == PM_SEM_COUNTER) {
             output << "# TYPE " << pn << " counter" << endl;
         } else {
             output << "# TYPE " << pn << " gauge" << endl; // DISCRETE or INSTANT
         }
 
+        // Labels of the metric
+        pmLabelSet *labelSet;
+        int label_instances;
+        if (metric_desc.indom != PM_INDOM_NULL)
+            label_instances = pmGetInstancesLabels(metric_desc.indom, &labelSet);
+        else
+            label_instances = pmGetItemLabels(metric_id, &labelSet);
+
         // Iterate over the instances
         int i;
         if (c->metric_inst_cache.find(metric_id) == c->metric_inst_cache.end())
@@ -1245,20 +1281,37 @@ metric_prometheus_batch_fetch(void *closure) {
             int inst = v->inst;
             string labels;
             if (inst < 0) { // not an instanced metric
-                ;
+                if (label_instances > 0 && labelSet->jsonlen > 0)
+                    labels = "{" +json_to_prometheus(labelSet->json) + "}";
             } else if (inst_cache.find(inst) != inst_cache.end()){
                 labels = inst_cache[inst];
             } else {
                 char *name;
                 string instance_string;
+                string label;
                 rc = pmNameInDom (metric_desc.indom, inst, & name);
                 if (rc < 0)
                     instance_string="";
                 else {
                     instance_string= escapeString(name);
                     free (name);
                 }
-                labels = "{instance=\"" + instance_string + "\"}";
+                if (label_instances > 0) {
+                    int j;
+                    pmLabelSet *temp = labelSet;
+                    for (j=0; j < label_instances; j++) {
+                        if ((int) temp->inst == inst )
+                            break;
+                        temp++;
+                    }
+                    if (j < label_instances && temp->jsonlen > 0) {
+                        label = json_to_prometheus(temp->json);
+                    }
+                }
+                labels = "{instance=\"" + instance_string + "\"";
+                if (label.length() > 0)
+                    labels += ',' + label;
+                labels += '}';
                 inst_cache[inst] = labels;
             }
 

src/pmwebapi/pmwebapi.h

@@ -104,6 +104,7 @@ extern void json_quote (std::ostream & o, const std::string & value);
 extern struct MHD_Response *NOTMHD_compressible_response(struct MHD_Connection *connection,
                                                          const std::string& buf);
 extern std::string escapeString(const std::string&);
+extern std::string json_to_prometheus(const std::string&);
 
 // inlined right here
 

src/pmwebapi/util.cxx

@@ -196,6 +196,61 @@ std::string urlencode (const std::string &foo)
     return output.str ();
 }
 
+/* Convert the JSON key value pair into Prometheus label format
+   eg: {"key":"value"} ----> key="value"
+*/
+std::string json_to_prometheus(const std::string &input) {
+    stringstream output;
+    if (input.length() == 0)
+        return "";
+    int i = 1, length = input.length() - 1, labels_count = 0;
+    while (i < length) {
+        if (labels_count > 0)
+            output << ",";
+        int j = i;
+        string key, value;
+        // Find the key
+        if (input[j] == '\"') {
+            j += 1;
+            while (j < length){
+                if (input[j] == '\"' && input[j-1] != '\\')
+                    break;
+                j++;
+            }
+            key = input.substr(i+1, j-i-1);
+            i = j+2;
+        } else {
+            j = input.find_first_of(":", i + 1);
+            key = input.substr(i, j-i);
+            i = j+1;
+        }
+        j = i;
+        // Find value
+        if (input[j] == '\"') {
+            j += 1;
+            while (j < length){
+                if (input[j] == '\"' && input[j-1] != '\\')
+                    break;
+                j++;
+            }
+            value = input.substr(i+1, j-i-1);
+            i = j+2;
+        } else {
+            j = input.find_first_of(",", i + 1);
+            if (j == -1){
+                j = length;
+                value = input.substr(i, length - i);
+            }
+            else
+                value = input.substr(i, j-i);
+            i = j+1;
+        }
+        output << key << "=" << '\"' << value << '\"';
+        labels_count ++;
+    }
+    return output.str();
+}
+
 std::string escapeString(const std::string& input) {
     std::ostringstream ss;
     for (std::string::const_iterator iter = input.begin(); iter != input.end(); iter++) {