Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add additional metadata to the data response #13036

Merged
merged 3 commits into from May 31, 2022
Merged

Add additional metadata to the data response #13036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d87e36e
Consolidate query params
stelfrag May 25, 2022
a8fd55f
Add new option to show full dimensions in the json header (this will …
stelfrag May 25, 2022
a9d540f
Group and pass parameters with query_params
stelfrag May 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
74 changes: 73 additions & 1 deletion web/api/formatters/json_wrapper.c
View file
Open in desktop
Expand Up @@ -2,9 +2,26 @@

#include "json_wrapper.h"

struct value_output {
int c;
BUFFER *wb;
};

static int value_list_output(void *entry, void *data) {
struct value_output *ap = (struct value_output *)data;
BUFFER *wb = ap->wb;
char *output = (char *) entry;
if(ap->c) buffer_strcat(wb, ",");
buffer_strcat(wb, output);
(ap->c)++;
return 0;
}

void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
struct context_param *context_param_list, char *chart_label_key)
QUERY_PARAMS *rrdset_query_data)
{
struct context_param *context_param_list = rrdset_query_data->context_param_list;
char *chart_label_key = rrdset_query_data->chart_label_key;

RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
Expand Down Expand Up @@ -98,6 +115,61 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
}
buffer_strcat(wb, "],\n");

if (rrdset_query_data->show_dimensions) {
buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq);

char name[RRD_ID_LENGTH_MAX * 2 + 2];
char output[RRD_ID_LENGTH_MAX * 2 + 8];
char value[RRD_ID_LENGTH_MAX * 2 + 1];

struct value_output co = {.c = 0, .wb = wb};

DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
dictionary_set(dict, name, output, len+1);
}
dictionary_get_all(dict, value_list_output, &co);
dictionary_destroy(dict);

co.c = 0;
buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name);
dictionary_set(dict, name, output, len + 1);
}

dictionary_get_all(dict, value_list_output, &co);
dictionary_destroy(dict);

RRDSET *st;
co.c = 0;
buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
st = rd->rrdset;
if (likely(st->state)) {
struct label_index *labels = &st->state->labels;
if (labels->head) {
netdata_rwlock_rdlock(&labels->labels_rwlock);
for (struct label *label = labels->head; label; label = label->next) {
sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
dictionary_set(dict, name, output, len + 1);
}
netdata_rwlock_unlock(&labels->labels_rwlock);
}
}
}
dictionary_get_all(dict, value_list_output, &co);
dictionary_destroy(dict);
buffer_strcat(wb, "],\n");
}

// Composite charts
if (context_mode && temp_rd) {
buffer_sprintf(
Expand Down
3 changes: 2 additions & 1 deletion web/api/formatters/json_wrapper.h
View file
Open in desktop
Expand Up @@ -5,7 +5,8 @@

#include "rrd2json.h"

extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key);
extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
QUERY_PARAMS *query_params);
extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);

#endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
58 changes: 33 additions & 25 deletions web/api/formatters/rrd2json.c
View file
Open in desktop
Expand Up @@ -211,7 +211,7 @@ int rrdset2value_api_v1(
int rrdset2anything_api_v1(
ONEWAYALLOC *owa
, RRDSET *st
, BUFFER *wb
, QUERY_PARAMS *query_params
, BUFFER *dimensions
, uint32_t format
, long points
Expand All @@ -221,16 +221,24 @@ int rrdset2anything_api_v1(
, long group_time
, uint32_t options
, time_t *latest_timestamp
, struct context_param *context_param_list
, char *chart_label_key
, int max_anomaly_rates
, int timeout
)
{
if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
BUFFER *wb = query_params->wb;
if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
st->last_accessed_time = now_realtime_sec();

RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout);
RRDR *r = rrd2rrdr(
owa,
st,
points,
after,
before,
group_method,
group_time,
options,
dimensions ? buffer_tostring(dimensions) : NULL,
query_params->context_param_list,
query_params->timeout);
if(!r) {
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
return HTTP_RESP_INTERNAL_SERVER_ERROR;
Expand All @@ -242,9 +250,9 @@ int rrdset2anything_api_v1(
}

if (st && st->state && st->state->is_ar_chart)
ml_process_rrdr(r, max_anomaly_rates);
ml_process_rrdr(r, query_params->max_anomaly_rates);

RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;

if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
buffer_no_cacheable(wb);
Expand All @@ -258,7 +266,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_SSV:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -271,7 +279,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_SSV_COMMA:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -284,7 +292,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_JS_ARRAY:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 0);
}
Expand All @@ -297,7 +305,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_CSV:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -310,7 +318,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_CSV_MARKDOWN:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -323,7 +331,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_CSV_JSON_ARRAY:
wb->contenttype = CT_APPLICATION_JSON;
if(options & RRDR_OPTION_JSON_WRAP) {
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
buffer_strcat(wb, "[\n");
rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
buffer_strcat(wb, "\n]");
Expand All @@ -340,7 +348,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_TSV:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
rrdr_json_wrapper_end(r, wb, format, options, 1);
}
Expand All @@ -353,7 +361,7 @@ int rrdset2anything_api_v1(
case DATASOURCE_HTML:
if(options & RRDR_OPTION_JSON_WRAP) {
wb->contenttype = CT_APPLICATION_JSON;
rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
Expand All @@ -371,9 +379,9 @@ int rrdset2anything_api_v1(
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 1, context_param_list);
rrdr2json(r, wb, options, 1, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand All @@ -383,9 +391,9 @@ int rrdset2anything_api_v1(
wb->contenttype = CT_APPLICATION_JSON;

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 1, context_param_list);
rrdr2json(r, wb, options, 1, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand All @@ -394,9 +402,9 @@ int rrdset2anything_api_v1(
case DATASOURCE_JSONP:
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 0, context_param_list);
rrdr2json(r, wb, options, 0, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand All @@ -407,9 +415,9 @@ int rrdset2anything_api_v1(
wb->contenttype = CT_APPLICATION_JSON;

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key);
rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);

rrdr2json(r, wb, options, 0, context_param_list);
rrdr2json(r, wb, options, 0, query_params->context_param_list);

if(options & RRDR_OPTION_JSON_WRAP)
rrdr_json_wrapper_end(r, wb, format, options, 0);
Expand Down
19 changes: 13 additions & 6 deletions web/api/formatters/rrd2json.h
View file
Open in desktop
Expand Up @@ -4,6 +4,17 @@
#define NETDATA_RRD2JSON_H 1

#include "web/api/web_api_v1.h"

typedef struct query_params {
struct context_param *context_param_list;
BUFFER *wb;
char *chart_label_key;
int max_anomaly_rates;
int timeout;
int show_dimensions;
} QUERY_PARAMS;


#include "web/api/exporters/allmetrics.h"
#include "web/api/queries/rrdr.h"

Expand Down Expand Up @@ -56,8 +67,8 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
extern int rrdset2anything_api_v1(
ONEWAYALLOC *owa
, RRDSET *st
, BUFFER *wb
, BUFFER *dimensions
,
QUERY_PARAMS *query_params, BUFFER *dimensions
, uint32_t format
, long points
, long long after
Expand All @@ -66,10 +77,6 @@ extern int rrdset2anything_api_v1(
, long group_time
, uint32_t options
, time_t *latest_timestamp
, struct context_param *context_param_list
, char *chart_label_key
, int max_anomaly_rates
, int timeout
);

extern int rrdset2value_api_v1(
Expand Down
15 changes: 11 additions & 4 deletions web/api/web_api_v1.c
View file
Open in desktop
Expand Up @@ -422,6 +422,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
char *chart_labels_filter = NULL;

int group = RRDR_GROUPING_AVERAGE;
int show_dimensions = 0;
uint32_t format = DATASOURCE_JSON;
uint32_t options = 0x00000000;

Expand All @@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
buffer_strcat(dimensions, "|");
buffer_strcat(dimensions, value);
}
else if(!strcmp(name, "show_dimensions")) show_dimensions = 1;
else if(!strcmp(name, "after")) after_str = value;
else if(!strcmp(name, "before")) before_str = value;
else if(!strcmp(name, "points")) points_str = value;
Expand Down Expand Up @@ -645,10 +647,15 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
buffer_strcat(w->response.data, "(");
}

ret = rrdset2anything_api_v1(owa, st, w->response.data, dimensions, format,
points, after, before, group, group_time,
options, &last_timestamp_in_data, context_param_list,
chart_label_key, max_anomaly_rates, timeout);
QUERY_PARAMS query_params = {
.context_param_list = context_param_list,
.timeout = timeout,
.max_anomaly_rates = max_anomaly_rates,
.show_dimensions = show_dimensions,
.wb = w->response.data};

ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
points, after, before, group, group_time, options, &last_timestamp_in_data);

free_context_param_list(owa, &context_param_list);

Expand Down