Add metric for showing the errant GTIDs in VTOrc (#13281)

* feat: add metric for errant gtid

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: update the errant gtid map always, so that once errant gtids are fixed, they are cleared

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: fix the test to match the new expectations

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: add mutex to protect the access to the map storing errant gtids

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: update the errant gtid map in a goroutine

Signed-off-by: Manan Gupta <manan@planetscale.com>

---------

Signed-off-by: Manan Gupta <manan@planetscale.com>

go/test/endtoend/vtorc/api/api_test.go

@@ -17,7 +17,9 @@ limitations under the License.
 package api
 
 import (
+	"encoding/json"
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
 
@@ -197,5 +199,23 @@ func TestAPIEndpoints(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, 400, status, resp)
 		assert.Equal(t, "Filtering by shard without keyspace isn't supported\n", resp)
+
+		// Also verify that the metric for errant GTIDs is reporting the correct information.
+		_, resp, err = utils.MakeAPICall(t, vtorc, "/debug/vars")
+		require.NoError(t, err)
+		resultMap := make(map[string]any)
+		err = json.Unmarshal([]byte(resp), &resultMap)
+		require.NoError(t, err)
+		errantGTIDMap := reflect.ValueOf(resultMap["ErrantGtidMap"])
+		errantGtidTablets := errantGTIDMap.MapKeys()
+		require.Len(t, errantGtidTablets, 3)
+
+		errantGTIDinReplica := ""
+		for _, tabletKey := range errantGtidTablets {
+			if tabletKey.String() == replica.Alias {
+				errantGTIDinReplica = errantGTIDMap.MapIndex(tabletKey).Interface().(string)
+			}
+		}
+		require.NotEmpty(t, errantGTIDinReplica)
 	})
 }

go/vt/vtorc/inst/instance_dao.go

@@ -31,6 +31,7 @@ import (
 	"github.com/rcrowley/go-metrics"
 	"github.com/sjmudd/stopwatch"
 
+	"vitess.io/vitess/go/stats"
 	"vitess.io/vitess/go/vt/external/golib/sqlutils"
 
 	vitessmysql "vitess.io/vitess/go/mysql"
@@ -56,6 +57,12 @@ const (
 var instanceReadChan = make(chan bool, backendDBConcurrency)
 var instanceWriteChan = make(chan bool, backendDBConcurrency)
 
+var (
+	// Mutex to protect the access of the following variable
+	errantGtidMapMu = sync.Mutex{}
+	errantGtidMap   = make(map[string]string)
+)
+
 // Constant strings for Group Replication information
 // See https://dev.mysql.com/doc/refman/8.0/en/replication-group-members-table.html for additional information.
 const (
@@ -88,6 +95,11 @@ func init() {
 	_ = metrics.Register("instance.write", writeInstanceCounter)
 	_ = writeBufferLatency.AddMany([]string{"wait", "write"})
 	writeBufferLatency.Start("wait")
+	stats.NewStringMapFuncWithMultiLabels("ErrantGtidMap", "Metric to track the errant GTIDs detected by VTOrc", []string{"TabletAlias"}, "ErrantGtid", func() map[string]string {
+		errantGtidMapMu.Lock()
+		defer errantGtidMapMu.Unlock()
+		return errantGtidMap
+	})
 
 	go initializeInstanceDao()
 }
@@ -429,6 +441,12 @@ Cleanup:
 				instance.GtidErrant, err = vitessmysql.Subtract(redactedExecutedGtidSet.String(), redactedPrimaryExecutedGtidSet.String())
 			}
 		}
+		// update the errant gtid map
+		go func() {
+			errantGtidMapMu.Lock()
+			defer errantGtidMapMu.Unlock()
+			errantGtidMap[topoproto.TabletAliasString(tablet.Alias)] = instance.GtidErrant
+		}()
 	}
 
 	latency.Stop("instance")