Increase Prometheus not found metric on tempo-vulture (#301)

* Add trace not found error Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com> * Remove hard-coded status code Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com> * Define and export the error Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com> * Update changelog Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com> * Refactor: reduce nesting Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com>
grafana · Nov 2, 2020 · eb987a8 · eb987a8
1 parent 806439d
commit eb987a8
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,3 +3,4 @@
 * [CHANGE] Bloom filters are now sharded to reduce size and improve caching, as blocks grow. This is a **breaking change** and all data stored before this change will **not** be queryable. [192](https://github.com/grafana/tempo/pull/192)
 * [ENHANCEMENT] CI checks for vendored dependencies using `make vendor-check`. Update CONTRIBUTING.md to reflect the same before checking in files in a PR. [#274](https://github.com/grafana/tempo/pull/274)
 * [ENHANCEMENT] Add warnings for suspect configs. [#294](https://github.com/grafana/tempo/pull/294)
+* [BUGFIX] Increase Prometheus `notfound` metric on tempo-vulture. [#301](https://github.com/grafana/tempo/pull/301)
diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go
@@ -114,6 +114,11 @@ func queryTempoAndAnalyze(baseURL string, backoff time.Duration, traceIDs []stri
 
 		glog.Error("tempo url ", baseURL+"/api/traces/"+id)
 		trace, err := util.QueryTrace(baseURL, id, tempoOrgID)
+		if err == util.ErrTraceNotFound {
+			glog.Error("trace not found ", id)
+			tm.notfound++
+			continue
+		}
 		if err != nil {
 			return nil, err
 		}

diff --git a/pkg/util/errors.go b/pkg/util/errors.go
@@ -2,12 +2,16 @@ package util
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
 
+// ErrTraceNotFound can be used when we don't find a trace
+var ErrTraceNotFound = errors.New("trace not found")
+
 // The MultiError type implements the error interface, and contains the
 // Errors used to construct it.
 type MultiError []error

diff --git a/pkg/util/query.go b/pkg/util/query.go
@@ -30,6 +30,10 @@ func QueryTrace(baseURL, id, orgID string) (*tempopb.Trace, error) {
 		}
 	}()
 
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, ErrTraceNotFound
+	}
+
 	trace := &tempopb.Trace{}
 	unmarshaller := &jsonpb.Unmarshaler{}
 	err = unmarshaller.Unmarshal(resp.Body, trace)