Skip to content

Commit

Permalink
Add additional metadata to the data response (#13036)
Browse files Browse the repository at this point in the history 
* Consolidate query params

* Add new option to show full dimensions in the json header (this will include dimensions, charts and chart labels)

* Group and pass parameters with query_params
  • Loading branch information
@stelfrag
stelfrag committed May 31, 2022
1 parent a13858f commit 3071aa0
Show file tree
Hide file tree
Showing 5 changed files with 132 additions and 37 deletions.
74 changes: 73 additions & 1 deletion web/api/formatters/json_wrapper.c
View file
Expand Up @@ -2,9 +2,26 @@

#include "json_wrapper.h"

struct value_output {
int c;
BUFFER *wb;
};

static int value_list_output(void *entry, void *data) {
struct value_output *ap = (struct value_output *)data;
BUFFER *wb = ap->wb;
char *output = (char *) entry;
if(ap->c) buffer_strcat(wb, ",");
buffer_strcat(wb, output);
(ap->c)++;
return 0;
}

void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
struct context_param *context_param_list, char *chart_label_key)
QUERY_PARAMS *rrdset_query_data)
{
struct context_param *context_param_list = rrdset_query_data->context_param_list;
char *chart_label_key = rrdset_query_data->chart_label_key;

RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
Expand Down Expand Up @@ -98,6 +115,61 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
}
buffer_strcat(wb, "],\n");

if (rrdset_query_data->show_dimensions) {
buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq);

char name[RRD_ID_LENGTH_MAX * 2 + 2];
char output[RRD_ID_LENGTH_MAX * 2 + 8];
char value[RRD_ID_LENGTH_MAX * 2 + 1];

struct value_output co = {.c = 0, .wb = wb};

DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
dictionary_set(dict, name, output, len+1);
}
dictionary_get_all(dict, value_list_output, &co);
dictionary_destroy(dict);

co.c = 0;
buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name);
dictionary_set(dict, name, output, len + 1);
}

dictionary_get_all(dict, value_list_output, &co);
dictionary_destroy(dict);

RRDSET *st;
co.c = 0;
buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
st = rd->rrdset;
if (likely(st->state)) {
struct label_index *labels = &st->state->labels;
if (labels->head) {
netdata_rwlock_rdlock(&labels->labels_rwlock);
for (struct label *label = labels->head; label; label = label->next) {
sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
dictionary_set(dict, name, output, len + 1);
}
netdata_rwlock_unlock(&labels->labels_rwlock);
}
}
}
dictionary_get_all(dict, value_list_output, &co);
dictionary_destroy(dict);
buffer_strcat(wb, "],\n");
}

// Composite charts
if (context_mode && temp_rd) {
buffer_sprintf(
Expand Down
3 changes: 2 additions & 1 deletion web/api/formatters/json_wrapper.h
View file
Expand Up @@ -5,7 +5,8 @@

#include "rrd2json.h"

extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key);
extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
QUERY_PARAMS *query_params);
extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);

#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
58 changes: 33 additions & 25 deletions web/api/formatters/rrd2json.c
View file
Expand Up @@ -211,7 +211,7 @@ int rrdset2value_api_v1(
int rrdset2anything_api_v1(
ONEWAYALLOC *owa
, RRDSET *st
, BUFFER *wb
, QUERY_PARAMS *query_params
, BUFFER *dimensions
, uint32_t format
, long points
Expand All @@ -221,16 +221,24 @@ int rrdset2anything_api_v1(
, long group_time
, uint32_t options
, time_t *latest_timestamp
, struct context_param *context_param_list
, char *chart_label_key
, int max_anomaly_rates
, int timeout
)
{
if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
BUFFER *wb = query_params->wb;
if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
st->last_accessed_time = now_realtime_sec();

RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
RRDR *r = rrd2rrdr(
owa,
st,
points,
after,
before,
group_method,
group_time,
options,
dimensions ? buffer_tostring(dimensions) : NULL,
query_params->context_param_list,
query_params->timeout);
if(!r) {
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
return HTTP_RESP_INTERNAL_SERVER_ERROR;
Expand All @@ -242,9 +250,9 @@ int rrdset2anything_api_v1(
}

if (st && st->state && st->state->is_ar_chart)
ml_process_rrdr(r, max_anomaly_rates);
ml_process_rrdr(r, query_params->max_anomaly_rates);

RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;

if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
buffer_no_cacheable(wb);
Expand All @@ -258,7 +266,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_SSV:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -271,7 +279,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_SSV_COMMA:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -284,7 +292,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_JS_ARRAY:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 0);
}
Expand All @@ -297,7 +305,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_CSV:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -310,7 +318,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_CSV_MARKDOWN:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -323,7 +331,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_CSV_JSON_ARRAY:
wb->contenttype = CT_APPLICATION_JSON;
if(options & RRDR_OPTION_JSON_WRAP) {
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
buffer_strcat(wb, "[\n");
rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
buffer_strcat(wb, "\n]");
Expand All @@ -340,7 +348,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_TSV:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -353,7 +361,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_HTML:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
Expand All @@ -371,9 +379,9 @@ int rrdset2anything_api_v1(
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 1, context_param_list);
rrdr2json(r, wb, options, 1, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand All @@ -383,9 +391,9 @@ int rrdset2anything_api_v1(
wb->contenttype = CT_APPLICATION_JSON;

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 1, context_param_list);
rrdr2json(r, wb, options, 1, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand All @@ -394,9 +402,9 @@ int rrdset2anything_api_v1(
case DATASOURCE_JSONP:
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 0, context_param_list);
rrdr2json(r, wb, options, 0, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand All @@ -407,9 +415,9 @@ int rrdset2anything_api_v1(
wb->contenttype = CT_APPLICATION_JSON;

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 0, context_param_list);
rrdr2json(r, wb, options, 0, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand Down
19 changes: 13 additions & 6 deletions web/api/formatters/rrd2json.h
View file
Expand Up @@ -4,6 +4,17 @@
#define NETDATA_RRD2JSON_H 1

#include "web/api/web_api_v1.h"

typedef struct query_params {
struct context_param *context_param_list;
BUFFER *wb;
char *chart_label_key;
int max_anomaly_rates;
int timeout;
int show_dimensions;
} QUERY_PARAMS;


#include "web/api/exporters/allmetrics.h"
#include "web/api/queries/rrdr.h"

Expand Down Expand Up @@ -56,8 +67,8 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
extern int rrdset2anything_api_v1(
ONEWAYALLOC *owa
, RRDSET *st
, BUFFER *wb
, BUFFER *dimensions
,
QUERY_PARAMS *query_params, BUFFER *dimensions
, uint32_t format
, long points
, long long after
Expand All @@ -66,10 +77,6 @@ extern int rrdset2anything_api_v1(
, long group_time
, uint32_t options
, time_t *latest_timestamp
, struct context_param *context_param_list
, char *chart_label_key
, int max_anomaly_rates
, int timeout
);

extern int rrdset2value_api_v1(
Expand Down
15 changes: 11 additions & 4 deletions web/api/web_api_v1.c
View file
Expand Up @@ -422,6 +422,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
char *chart_labels_filter = NULL;

int group = RRDR_GROUPING_AVERAGE;
int show_dimensions = 0;
uint32_t format = DATASOURCE_JSON;
uint32_t options = 0x00000000;

Expand All @@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
buffer_strcat(dimensions, "|");
buffer_strcat(dimensions, value);
}
else if(!strcmp(name, "show_dimensions")) show_dimensions = 1;
else if(!strcmp(name, "after")) after_str = value;
else if(!strcmp(name, "before")) before_str = value;
else if(!strcmp(name, "points")) points_str = value;
Expand Down Expand Up @@ -645,10 +647,15 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
buffer_strcat(w->response.data, "(");
}

ret = rrdset2anything_api_v1(owa, st, w->response.data, dimensions, format,
points, after, before, group, group_time,
options, &last_timestamp_in_data, context_param_list,
chart_label_key, max_anomaly_rates, timeout);
QUERY_PARAMS query_params = {
.context_param_list = context_param_list,
.timeout = timeout,
.max_anomaly_rates = max_anomaly_rates,
.show_dimensions = show_dimensions,
.wb = w->response.data};

ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
points, after, before, group, group_time, options, &last_timestamp_in_data);

free_context_param_list(owa, &context_param_list);

Expand Down

0 comments on commit 3071aa0

Please sign in to comment.