Skip to content

Commit

Permalink
do not report dimensions that failed to be queried (#14447)
Browse files Browse the repository at this point in the history 
* do not report dimensions that failed to be queried

* renamed SELECTED to QUERIED to have clarity on what it means

* fix wrong placement of continue
  • Loading branch information
@ktsaou @Ferroin
ktsaou authored and Ferroin committed Feb 9, 2023
1 parent 357ed77 commit 47038b8
Show file tree
Hide file tree
Showing 8 changed files with 49 additions and 21 deletions.
10 changes: 5 additions & 5 deletions database/rrdcontext.c
View file
Expand Up @@ -2501,7 +2501,7 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
if (rrd_flag_check(rm, RRD_FLAG_HIDDEN)
|| (rm->rrddim && rrddim_option_check(rm->rrddim, RRDDIM_OPTION_HIDDEN))) {
options |= RRDR_DIMENSION_HIDDEN;
options &= ~RRDR_DIMENSION_SELECTED;
options &= ~RRDR_DIMENSION_QUERIED;
}

if (qt->query.pattern) {
Expand All @@ -2512,24 +2512,24 @@ static void query_target_add_metric(QUERY_TARGET_LOCALS *qtl, RRDMETRIC_ACQUIRED
|| (qtl->match_names && simple_pattern_matches(qt->query.pattern, string2str(rm->name)))
) {
// it matches the pattern
options |= (RRDR_DIMENSION_SELECTED | RRDR_DIMENSION_NONZERO);
options |= (RRDR_DIMENSION_QUERIED | RRDR_DIMENSION_NONZERO);
options &= ~RRDR_DIMENSION_HIDDEN;
}
else {
// it does not match the pattern
options |= RRDR_DIMENSION_HIDDEN;
options &= ~RRDR_DIMENSION_SELECTED;
options &= ~RRDR_DIMENSION_QUERIED;
}
}
else {
// we don't have a dimensions pattern
// so this is a selected dimension
// if it is not hidden
if(!(options & RRDR_DIMENSION_HIDDEN))
options |= RRDR_DIMENSION_SELECTED;
options |= RRDR_DIMENSION_QUERIED;
}

if((options & RRDR_DIMENSION_HIDDEN) && (options & RRDR_DIMENSION_SELECTED))
if((options & RRDR_DIMENSION_HIDDEN) && (options & RRDR_DIMENSION_QUERIED))
options &= ~RRDR_DIMENSION_HIDDEN;

if(!(options & RRDR_DIMENSION_HIDDEN) || (qt->request.options & RRDR_OPTION_PERCENTAGE)) {
Expand Down
5 changes: 5 additions & 0 deletions web/api/formatters/csv/csv.c
View file
Expand Up @@ -12,6 +12,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
// print the csv header
for(c = 0, i = 0; c < used ; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

if(!i) {
Expand All @@ -32,6 +33,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
// print the --- line after header
for(c = 0, i = 0; c < used ;c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

if(!i) {
Expand Down Expand Up @@ -89,6 +91,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0; c < used ;c++) {
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

NETDATA_DOUBLE n = cn[c];

if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
Expand All @@ -104,6 +108,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
// for each dimension
for(c = 0; c < used ;c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

buffer_strcat(wb, separator);
Expand Down
6 changes: 5 additions & 1 deletion web/api/formatters/json/json.c
View file
Expand Up @@ -111,6 +111,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
// print the header lines
for(c = 0, i = 0; c < used ; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

buffer_fast_strcat(wb, pre_label, pre_label_len);
Expand Down Expand Up @@ -180,7 +181,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
// google supports one annotation per row
int annotation_found = 0;
for(c = 0; c < used ; c++) {
if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

if(unlikely(co[c] & RRDR_VALUE_RESET)) {
buffer_fast_strcat(wb, overflow_annotation, overflow_annotation_len);
Expand Down Expand Up @@ -215,6 +216,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0; c < used ;c++) {
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

NETDATA_DOUBLE n;
if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
n = ar[c];
Expand All @@ -234,6 +237,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
// for each dimension
for(c = 0; c < used ;c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

NETDATA_DOUBLE n;
Expand Down
14 changes: 9 additions & 5 deletions web/api/formatters/json_wrapper.c
View file
Expand Up @@ -131,6 +131,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS

for(c = 0, i = 0; c < query_used ; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

if(i) buffer_strcat(wb, ", ");
Expand All @@ -155,6 +156,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS

for(c = 0, i = 0; c < query_used ; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

if(i) buffer_strcat(wb, ", ");
Expand Down Expand Up @@ -260,9 +262,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
for (c = 0, i = 0; c < query_used; c++) {
QUERY_METRIC *qm = &qt->query.array[c];

if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
continue;

if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
continue;

Expand Down Expand Up @@ -295,8 +296,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
for (c = 0, i = 0; c < query_used; c++) {
QUERY_METRIC *qm = &qt->query.array[c];

if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
continue;
if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
continue;

Expand Down Expand Up @@ -351,6 +352,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0; c < query_used ;c++) {
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;

NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
NETDATA_DOUBLE n = cn[c];

Expand All @@ -365,6 +368,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS

for(c = 0, i = 0; c < query_used ;c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

if(i) buffer_strcat(wb, ", ");
Expand Down
2 changes: 2 additions & 0 deletions web/api/formatters/value/value.c
View file
Expand Up @@ -22,6 +22,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for (c = 0; c < used; c++) {
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
NETDATA_DOUBLE n = cn[c];

if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
Expand All @@ -37,6 +38,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
// for each dimension
for (c = 0; c < used; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;

NETDATA_DOUBLE n = cn[c];
Expand Down
28 changes: 19 additions & 9 deletions web/api/queries/query.c
View file
Expand Up @@ -2273,9 +2273,11 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
r->internal.grouping_reset(r);

if(ops[c]) {
r->od[c] |= RRDR_DIMENSION_SELECTED;
r->od[c] |= RRDR_DIMENSION_QUERIED;
rrd2rrdr_query_execute(r, c, ops[c]);
}
else
continue;

global_statistics_rrdr_query_completed(
1,
Expand Down Expand Up @@ -2385,15 +2387,23 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
// free all resources used by the grouping method
r->internal.grouping_free(r);

// when all the dimensions are zero, we should return all of them
if(unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
// all the dimensions are zero
// mark them as NONZERO to send them all
for(size_t c = 0, max = qt->query.used; c < max ; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
r->od[c] |= RRDR_DIMENSION_NONZERO;
if(likely(dimensions_used)) {
// when all the dimensions are zero, we should return all of them
if (unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero &&
!(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
// all the dimensions are zero
// mark them as NONZERO to send them all
for (size_t c = 0, max = qt->query.used; c < max; c++) {
if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue;
r->od[c] |= RRDR_DIMENSION_NONZERO;
}
}

return r;
}

return r;
// we couldn't query any dimension
rrdr_free(owa, r);
return NULL;
}
2 changes: 1 addition & 1 deletion web/api/queries/rrdr.h
View file
Expand Up @@ -58,7 +58,7 @@ typedef enum rrdr_dimension_flag {
RRDR_DIMENSION_DEFAULT = 0x00,
RRDR_DIMENSION_HIDDEN = 0x04, // the dimension is hidden (not to be presented to callers)
RRDR_DIMENSION_NONZERO = 0x08, // the dimension is non zero (contains non-zero values)
RRDR_DIMENSION_SELECTED = 0x10, // the dimension is selected for evaluation in this RRDR
RRDR_DIMENSION_QUERIED = 0x10, // the dimension is selected for evaluation in this RRDR
} RRDR_DIMENSION_FLAGS;

// RRDR result options
Expand Down
3 changes: 3 additions & 0 deletions web/api/queries/weights.c
View file
Expand Up @@ -541,6 +541,9 @@ NETDATA_DOUBLE *rrd2rrdr_ks2(
if(unlikely(r->od[0] & RRDR_DIMENSION_HIDDEN))
goto cleanup;

if(unlikely(!(r->od[0] & RRDR_DIMENSION_QUERIED)))
goto cleanup;

if(unlikely(!(r->od[0] & RRDR_DIMENSION_NONZERO)))
goto cleanup;

Expand Down

0 comments on commit 47038b8

Please sign in to comment.