fix: add logs and metrics around pusher retries

pkg/pusher/metrics.go

@@ -17,7 +17,8 @@ type metrics struct {
 	SyncTime         prometheus.Histogram
 	ErrorTime        prometheus.Histogram
 
-	ReceiptDepth *prometheus.CounterVec
+	ReceiptDepth   *prometheus.CounterVec
+	ShallowRetries prometheus.Counter
 }
 
 func newMetrics() metrics {
@@ -72,6 +73,12 @@ func newMetrics() metrics {
 			},
 			[]string{"depth"},
 		),
+		ShallowRetries: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "shallow_retries",
+			Help:      "Chunks retried because receipts were too shallow",
+		}),
 	}
 }
 

pkg/pusher/pusher.go

@@ -155,8 +155,8 @@ LOOP:
 						s.metrics.TotalSynced.Inc()
 						s.metrics.SyncTime.Observe(time.Since(startTime).Seconds())
 						// only print this if there was no error while sending the chunk
-						logger.Tracef("pusher: pushed chunk %s to node %s", ch.Address().String(), storerPeer.String())
 						po := swarm.Proximity(ch.Address().Bytes(), storerPeer.Bytes())
+						logger.Tracef("pusher: pushed chunk %s to node %s with receipt depth %d", ch.Address().String(), storerPeer.String(), po)
 						s.metrics.ReceiptDepth.WithLabelValues(strconv.Itoa(int(po))).Inc()
 						delete(retryCounter, ch.Address().ByteString())
 					} else {
@@ -203,12 +203,13 @@ LOOP:
 					d := s.depther.NeighborhoodDepth()
 					if po < d {
 						mtx.Lock()
-						retryCounter[ch.Address().ByteString()]++
 						if retryCounter[ch.Address().ByteString()] < retryCount {
+							retryCounter[ch.Address().ByteString()]++
 							mtx.Unlock()
-							err = fmt.Errorf("pusher: shallow receipt depth %d, want at least %d", po, d)
 							po := swarm.Proximity(ch.Address().Bytes(), storerPeer.Bytes())
-							s.metrics.ReceiptDepth.WithLabelValues(strconv.Itoa(int(po))).Inc()
+							// QUESTION: where is this error logged?
+							err = fmt.Errorf("pusher: reshallow receipt depth %d from chunk %s, want at least %d", po, ch.Address().String(), d)
+							s.metrics.ShallowRetries.Inc()
 							return
 						}
 						mtx.Unlock()