REP-6804 Add option to read oplog directly

.github/workflows/all.yml

@@ -28,6 +28,41 @@ jobs:
           - mongodb_versions: [ '4.2.5', '6.0' ]
             topology: replset
 
+          - mongodb_versions: [ '4.0', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
+          - mongodb_versions: [ '4.2', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
+          - mongodb_versions: [ '4.4', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
+          - mongodb_versions: [ '5.0', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
+          - mongodb_versions: [ '6.0', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
+          - mongodb_versions: [ '7.0', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
+          - mongodb_versions: [ '8.0', '8.0' ]
+            topology: replset
+            srcChangeReader: tailOplog
+            dstChangeReader: tailOplog
+
         exclude:
           - mongodb_versions: [ '4.2', '4.2' ]
             toHashedIndexKey: true
@@ -69,6 +104,9 @@ jobs:
 
         toHashedIndexKey: [true, false]
 
+        srcChangeReader: [changeStream]
+        dstChangeReader: [changeStream]
+
         topology:
           - replset
           - replset-to-sharded
@@ -78,7 +116,7 @@ jobs:
     # versions need.
     runs-on: ubuntu-22.04
 
-    name: ${{ matrix.mongodb_versions[0] }} to ${{ matrix.mongodb_versions[1] }}, ${{ matrix.topology }}${{ matrix.toHashedIndexKey && ', hashed doc compare' || '' }}
+    name: ${{ matrix.mongodb_versions[0] }} to ${{ matrix.mongodb_versions[1] }}, ${{ matrix.topology }}${{ matrix.toHashedIndexKey && ', hashed doc compare' || '' }}, srcChangeReader=${{ matrix.srcChangeReader }}, dstChangeReader=${{ matrix.dstChangeReader }}
 
     steps:
       - run: uname -a
@@ -124,6 +162,9 @@ jobs:
         env:
           MVTEST_DOC_COMPARE_METHOD: ${{matrix.toHashedIndexKey && 'toHashedIndexKey' || ''}}
 
+          MVTEST_SRC_CHANGE_READER: ${{matrix.srcChangeReader}}
+          MVTEST_DST_CHANGE_READER: ${{matrix.dstChangeReader}}
+
           MVTEST_SRC: ${{ (matrix.topology == 'sharded') && env.shardedSrcConnStr || env.replsetSrcConnStr }}
           MVTEST_DST: ${{ (matrix.topology == 'sharded' || matrix.topology == 'replset-to-sharded') && env.shardedDstConnStr || env.replsetDstConnStr }}
 

README.md

@@ -145,6 +145,8 @@ The verifier will now check to completion to make sure that there are no inconsi
 | `--dstNamespace <namespaces>`           | destination namespaces to check                                                                                                                                                             |
 | `--metaDBName <name>`                   | name of the database in which to store verification metadata (default: "migration_verification_metadata")                                                                                   |
 | `--docCompareMethod`                    | How to compare documents. See below for details.                                                                                                                                        |
+| `--srcChangeReader`                     | How to read changes from the source. See below for details.             |
+| `--dstChangeReader`                     | How to read changes from the destination. See below for details.        |
 | `--start`                               | Start checking documents right away rather than waiting for a `/check` API request. |
 | `--verifyAll`                           | If set, verify all user namespaces                                                                                                                                                          |
 | `--clean`                               | If set, drop all previous verification metadata before starting                                                                                                                             |
@@ -198,8 +200,6 @@ connection strings in the following environment variables:
 
 The migration-verifier has two steps:
 
-
-
 1. The initial check
     1. The verifier partitions up the data into 400MB (configurable) chunks and spins up many worker goroutines (threads) to read from both the source and destination.
     2. The verifier compares the documents on the source and destination by bytes and if they are different, it then checks field by field in case the field ordering has changed (since field ordering isn't required to be the same for the migration to be a success)
@@ -392,6 +392,19 @@ Full-document verification methods allow migration-verifier to diagnose mismatch
 
 Additionally, because the amount of data sent to migration-verifier doesn’t actually reflect the documents’ size, no meaningful statistics are shown concerning the collection data size. Document counts, of course, are still shown.
 
+# Change reading methods
+
+NB: If the verifier restarts, it **MUST** use the same change reader options
+as before, or it will fail immediately.
+
+## `changeStream`
+
+The default. The verifier will read a change stream, which works seamlessly on sharded or unsharded clusters.
+
+## `tailOplog`
+
+The verifier will read the oplog continually instead of reading a change stream. This is generally faster, but it doesn’t work in sharded clusters.
+
 # Known Issues
 
 - The verifier may report missing documents on the destination that don’t actually appear to be missing (i.e., a nonexistent problem). This has been hard to reproduce. If missing documents are reported, it is good practice to check for false positives.

agg/agg.go

@@ -0,0 +1,208 @@
+// Package agg provides convenience types for aggregation operators.
+// This yields two major advantages over using bson.D or bson.M:
+// - simpler syntax
+// - auto-completion (i.e., via gopls)
+//
+// Guiding principles are:
+// - Prefer [1]any for unary operators (e.g., `$bsonSize`).
+// - Prefer struct types for operators with named parameters.
+// - Use functions sparingly, e.g., for “tuple” operators like `$in`.
+package agg
+
+import (
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type Eq []any
+
+var _ bson.Marshaler = Eq{}
+
+func (e Eq) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$eq", []any(e)}})
+}
+
+// ---------------------------------------------
+
+func In[T any](needle any, haystack []T) bson.D {
+	return bson.D{{"$in", bson.A{needle, haystack}}}
+}
+
+// ---------------------------------------------
+
+type BSONSize [1]any
+
+var _ bson.Marshaler = BSONSize{}
+
+func (b BSONSize) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$bsonSize", b[0]}})
+}
+
+// ---------------------------------------------
+
+type Type [1]any
+
+var _ bson.Marshaler = Type{}
+
+func (t Type) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$type", t[0]}})
+}
+
+// ---------------------------------------------
+
+type Not [1]any
+
+var _ bson.Marshaler = Type{}
+
+func (n Not) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{{"$not", n[0]}})
+}
+
+// ---------------------------------------------
+
+type And []any
+
+var _ bson.Marshaler = And{}
+
+func (a And) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$and", []any(a)},
+	})
+}
+
+// ---------------------------------------------
+
+type Or []any
+
+var _ bson.Marshaler = Or{}
+
+func (o Or) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$or", []any(o)},
+	})
+}
+
+// ---------------------------------------------
+
+type MergeObjects []any
+
+var _ bson.Marshaler = MergeObjects{}
+
+func (m MergeObjects) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$mergeObjects", []any(m)},
+	})
+}
+
+// ---------------------------------------------
+
+type Cond struct {
+	If, Then, Else any
+}
+
+var _ bson.Marshaler = Cond{}
+
+func (c Cond) D() bson.D {
+	return bson.D{
+		{"$cond", bson.D{
+			{"if", c.If},
+			{"then", c.Then},
+			{"else", c.Else},
+		}},
+	}
+}
+
+func (c Cond) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(c.D())
+}
+
+// ---------------------------------------------
+
+type Switch struct {
+	Branches []SwitchCase
+	Default  any
+}
+
+var _ bson.Marshaler = Switch{}
+
+type SwitchCase struct {
+	Case any
+	Then any
+}
+
+func (s Switch) D() bson.D {
+	return bson.D{{"$switch", bson.D{
+		{"branches", s.Branches},
+		{"default", s.Default},
+	}}}
+}
+
+func (s Switch) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(s.D())
+}
+
+// ---------------------------------------------
+
+type ArrayElemAt struct {
+	Array any
+	Index int
+}
+
+var _ bson.Marshaler = ArrayElemAt{}
+
+func (a ArrayElemAt) D() bson.D {
+	return bson.D{{"$arrayElemAt", bson.A{
+		a.Array,
+		a.Index,
+	}}}
+}
+
+func (a ArrayElemAt) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(a.D())
+}
+
+// ---------------------------------------------
+
+type Map struct {
+	Input, As, In any
+}
+
+var _ bson.Marshaler = Map{}
+
+func (m Map) D() bson.D {
+	return bson.D{
+		{"$map", bson.D{
+			{"input", m.Input},
+			{"as", m.As},
+			{"in", m.In},
+		}},
+	}
+}
+
+func (m Map) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(m.D())
+}
+
+// ------------------------------------------
+
+type Filter struct {
+	Input, As, Cond, Limit any
+}
+
+var _ bson.Marshaler = Filter{}
+
+func (f Filter) D() bson.D {
+	d := bson.D{
+		{"input", f.Input},
+		{"as", f.As},
+		{"cond", f.Cond},
+	}
+
+	if f.Limit != nil {
+		d = append(d, bson.E{"limit", f.Limit})
+	}
+	return bson.D{{"$filter", d}}
+}
+
+func (f Filter) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(f.D())
+}

agg/helpers/exist.go

@@ -0,0 +1,12 @@
+package helpers
+
+import (
+	"github.com/10gen/migration-verifier/agg"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type Exists [1]any
+
+func (e Exists) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(agg.Not{agg.Eq{"missing", agg.Type{e[0]}}})
+}

agg/helpers/string.go

@@ -0,0 +1,27 @@
+// Package helpers exposes functions that express common operations
+// that don’t map to a single aggregation operator.
+package helpers
+
+import (
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+// StringHasPrefix parallels Go’s strings.HasPrefix.
+type StringHasPrefix struct {
+	FieldRef any
+	Prefix   string
+}
+
+func (sp StringHasPrefix) MarshalBSON() ([]byte, error) {
+	return bson.Marshal(bson.D{
+		{"$eq", bson.A{
+			0,
+			bson.D{{"$indexOfCP", bson.A{
+				sp.FieldRef,
+				sp.Prefix,
+				0,
+				1,
+			}}},
+		}},
+	})
+}