Merge d9c9ac5 into 2b176f5

api/v2/api-v2.go

@@ -7,13 +7,15 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/m-lab/annotation-service/api"
+	"github.com/m-lab/annotation-service/metrics"
 	"github.com/m-lab/go/logx"
 )
 
@@ -81,6 +83,7 @@ func post(ctx context.Context, url string, encodedData []byte) (*http.Response,
 
 	httpReq, err := http.NewRequest("POST", url, bytes.NewReader(encodedData))
 	if err != nil {
+		log.Println(err)
 		return nil, err
 	}
 
@@ -107,6 +110,7 @@ func postWithRetry(ctx context.Context, url string, encodedData []byte) (*http.R
 		start := time.Now()
 		resp, err := post(ctx, url, encodedData)
 		if err != nil {
+			log.Println(err)
 			RequestTimeHistogram.WithLabelValues(err.Error()).Observe(float64(time.Since(start).Nanoseconds()) / 1e6)
 			return nil, err
 		}
@@ -119,13 +123,15 @@ func postWithRetry(ctx context.Context, url string, encodedData []byte) (*http.R
 			return resp, ErrStatusNotOK
 		}
 		if ctx.Err() != nil {
+			log.Println(ctx.Err())
 			RequestTimeHistogram.WithLabelValues("timeout").Observe(float64(time.Since(start).Nanoseconds()) / 1e6)
 			return nil, ctx.Err()
 		}
 		// This is a recoverable error, so we should retry.
 		RequestTimeHistogram.WithLabelValues("retry").Observe(float64(time.Since(start).Nanoseconds()) / 1e6)
 		err = waitOneSecond(ctx)
 		if err != nil {
+			log.Println(err)
 			return nil, err
 		}
 	}
@@ -145,16 +151,21 @@ func GetAnnotations(ctx context.Context, url string, date time.Time, ips []strin
 	}
 	encodedData, err := json.Marshal(req)
 	if err != nil {
+		log.Println(err)
+		metrics.ClientErrorTotal.WithLabelValues("request encoding error").Inc()
 		return nil, err
 	}
 
 	httpResp, err := postWithRetry(ctx, url, encodedData)
 	if err != nil {
+		log.Println(err)
 		if httpResp == nil || httpResp.Body == nil {
+			metrics.ClientErrorTotal.WithLabelValues("http empty response").Inc()
 			return nil, err
 		}
 		defer httpResp.Body.Close()
 		if err == ErrStatusNotOK {
+			metrics.ClientErrorTotal.WithLabelValues("http status not OK").Inc()
 			body, ioutilErr := ioutil.ReadAll(httpResp.Body)
 			if ioutilErr != nil {
 				return nil, ioutilErr
@@ -171,11 +182,15 @@ func GetAnnotations(ctx context.Context, url string, date time.Time, ips []strin
 	defer httpResp.Body.Close()
 	// Handle other status codes
 	if httpResp.StatusCode != http.StatusOK {
+		log.Println("http statuscode not OK")
+		metrics.ClientErrorTotal.WithLabelValues("http statuscode not OK").Inc()
 		return nil, errors.New(httpResp.Status)
 	}
 	// Copy response into a byte slice
 	body, err := ioutil.ReadAll(httpResp.Body)
 	if err != nil {
+		log.Println(err)
+		metrics.ClientErrorTotal.WithLabelValues("cannot read http response").Inc()
 		return nil, err
 	}
 
@@ -195,6 +210,8 @@ func GetAnnotations(ctx context.Context, url string, date time.Time, ips []strin
 		err = decoder.Decode(&resp)
 		if err != nil {
 			// This is a more serious error.
+			log.Println(err)
+			metrics.ClientErrorTotal.WithLabelValues("json decode error").Inc()
 			return nil, err
 		}
 	}

metrics/metrics.go

@@ -55,7 +55,11 @@ var (
 		Name: "annotator_Error_total",
 		Help: "The total number annotation errors.",
 	}, []string{"type"})
-
+	// ClientErrorTotal measures the different type of RPC client errors.
+	ClientErrorTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "annotator_client_error_total",
+		Help: "The total number client annotation RPC errors.",
+	}, []string{"type"})
 	// TODO make this an integer gauge
 	DatasetCount = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "annotator_num_datasets",