Increase Prometheus not found metric on tempo-vulture

CHANGELOG.md

@@ -3,3 +3,4 @@
 * [CHANGE] Bloom filters are now sharded to reduce size and improve caching, as blocks grow. This is a **breaking change** and all data stored before this change will **not** be queryable. [192](https://github.com/grafana/tempo/pull/192)
 * [ENHANCEMENT] CI checks for vendored dependencies using `make vendor-check`. Update CONTRIBUTING.md to reflect the same before checking in files in a PR. [#274](https://github.com/grafana/tempo/pull/274)
 * [ENHANCEMENT] Add warnings for suspect configs. [#294](https://github.com/grafana/tempo/pull/294)
+* [BUGFIX] Increase Prometheus `notfound` metric on tempo-vulture. [#301](https://github.com/grafana/tempo/pull/301)

cmd/tempo-vulture/main.go

@@ -114,6 +114,11 @@ func queryTempoAndAnalyze(baseURL string, backoff time.Duration, traceIDs []stri
 
 		glog.Error("tempo url ", baseURL+"/api/traces/"+id)
 		trace, err := util.QueryTrace(baseURL, id, tempoOrgID)
+		if err == util.ErrTraceNotFound {
+			glog.Error("trace not found ", id)
+			tm.notfound++
+			continue
+		}
 		if err != nil {
 			return nil, err
 		}

pkg/util/errors.go

@@ -2,12 +2,16 @@ package util
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
 
+// ErrTraceNotFound can be used when we don't find a trace
+var ErrTraceNotFound = errors.New("trace not found")
+
 // The MultiError type implements the error interface, and contains the
 // Errors used to construct it.
 type MultiError []error

pkg/util/query.go

@@ -30,6 +30,10 @@ func QueryTrace(baseURL, id, orgID string) (*tempopb.Trace, error) {
 		}
 	}()
 
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, ErrTraceNotFound
+	}
+
 	trace := &tempopb.Trace{}
 	unmarshaller := &jsonpb.Unmarshaler{}
 	err = unmarshaller.Unmarshal(resp.Body, trace)