retry metadata posting on 5xx errors

command/metadata.go

@@ -62,10 +62,18 @@ func runMetadataLoop(c *Context, termMetadataCh <-chan struct{}, quit <-chan str
 		}
 
 		for _, result := range results {
-			err := c.API.PutMetadata(c.Host.ID, result.namespace, result.metadata)
+			resp, err := c.API.PutMetadata(c.Host.ID, result.namespace, result.metadata)
+			// retry on 5XX errors
+			if resp != nil && resp.StatusCode >= 500 {
+				logger.Errorf("put metadata %q failed: status %s", result.namespace, resp.Status)
+				resultCh <- &metadataResult{
+					namespace: result.namespace,
+					metadata:  result.metadata,
+				}
+				continue
+			}
 			if err != nil {
-				logger.Errorf("put metadata %q failed: %s", result.namespace, err.Error())
-				// do not retry because huge metadata will be always rejected
+				logger.Errorf("put metadata %q failed: %v", result.namespace, err)
 				clearMetadataCache(c.Agent.MetadataGenerators, result.namespace)
 				continue
 			}

mackerel/metadata.go

@@ -1,9 +1,13 @@
 package mackerel
 
+import (
+	"net/http"
+)
+
 // PutMetadata creates or updates metadata of a host.
-func (api *API) PutMetadata(hostID string, namespace string, metadata interface{}) error {
+func (api *API) PutMetadata(hostID string, namespace string, metadata interface{}) (*http.Response, error) {
 	resp, err := api.putJSON("/api/v0/hosts/"+hostID+"/metadata/"+namespace, metadata)
 	defer closeResp(resp)
 
-	return err
+	return resp, err
 }