bridge/pkg/solana: retry VAA submission on transient errors

bridge/pkg/solana/watcher.go

@@ -9,6 +9,8 @@ import (
 
 	eth_common "github.com/ethereum/go-ethereum/common"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	agentv1 "github.com/certusone/wormhole/bridge/pkg/proto/agent/v1"
 
@@ -110,6 +112,31 @@ func (e *SolanaBridgeWatcher) Run(ctx context.Context) error {
 				cancel()
 				if err != nil {
 					logger.Error("failed to submit VAA", zap.Error(err), zap.String("digest", h))
+
+					st, ok := status.FromError(err)
+					if !ok {
+						panic("err not a status")
+					}
+
+					// For transient errors, we can put the VAA back into the queue such that it can
+					// be retried after the runnable has been rescheduled.
+					switch st.Code() {
+					case
+						// Our context was cancelled, likely because the watcher stream died.
+						codes.Canceled,
+						// The agent encountered a transient error, likely node unavailability.
+						codes.Unavailable,
+						codes.Aborted:
+
+						logger.Error("requeuing VAA", zap.Error(err), zap.String("digest", h))
+
+						// Tombstone goroutine
+						go func(v *vaa.VAA) {
+							time.Sleep(10 * time.Second)
+							e.vaaChan <- v
+						}(v)
+					}
+
 					break
 				}