diff --git a/README.md b/README.md index b53c4a8e..032193ef 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,8 @@ The migration-verifier is also rather resource-hungry. To mitigate this, try lim - The verifier, during its first generation, may report a confusing “Mismatches found” but then report 0 problems. This is a reporting bug in mongosync; if you see it, check the documents in `migration_verification_metadata.verification_tasks` for generation 1 (not generation 0). +- The verifier conflates “missing” documents with change events: if it finds a document missing and receives a change event for another document, the verifier records those together in its metadata. As a result, the verifier’s reporting can cause confusion: what its log calls “missing or changed” documents aren’t, in fact, necessarily failures; they could all just be change events that are pending a recheck. + # Limitations - The verifier’s iterative process can handle data changes while it is running, until you hit the writesOff endpoint. However, it cannot handle DDL commands. If the verifier receives a DDL change stream event (drop, dropDatabase, rename), the verification will fail. If an untracked DDL event (create, createIndexes, dropIndexes, modify) occurs, the verifier may miss the change. diff --git a/internal/verifier/summary.go b/internal/verifier/summary.go index 540c37ab..f847de65 100644 --- a/internal/verifier/summary.go +++ b/internal/verifier/summary.go @@ -73,15 +73,15 @@ func (verifier *Verifier) reportDocumentMismatches(ctx context.Context, strBuild failureTypesTable := tablewriter.NewWriter(strBuilder) failureTypesTable.SetHeader([]string{"Failure Type", "Count"}) - contentMismatch := 0 - missing := 0 + contentMismatchCount := 0 + missingOrChangedCount := 0 for _, task := range failedTasks { - contentMismatch += len(task.FailedDocs) - missing += len(task.Ids) + contentMismatchCount += len(task.FailedDocs) + missingOrChangedCount += len(task.Ids) } - failureTypesTable.Append([]string{"Documents With Differing Content", fmt.Sprintf("%v", contentMismatch)}) - failureTypesTable.Append([]string{"Documents Missing On Source or Dest", fmt.Sprintf("%v", missing)}) + failureTypesTable.Append([]string{"Documents With Differing Content", fmt.Sprintf("%v", contentMismatchCount)}) + failureTypesTable.Append([]string{"Missing or Changed Documents", fmt.Sprintf("%v", missingOrChangedCount)}) strBuilder.WriteString("Failure summary:\n") failureTypesTable.Render() @@ -89,7 +89,7 @@ func (verifier *Verifier) reportDocumentMismatches(ctx context.Context, strBuild mismatchedDocsTableRows := types.ToNumericTypeOf(0, verifier.failureDisplaySize) mismatchedDocsTable.SetHeader([]string{"ID", "Cluster", "Field", "Namespace", "Details"}) - printAll := int64(contentMismatch) < (verifier.failureDisplaySize + int64(0.25*float32(verifier.failureDisplaySize))) + printAll := int64(contentMismatchCount) < (verifier.failureDisplaySize + int64(0.25*float32(verifier.failureDisplaySize))) OUTA: for _, task := range failedTasks { for _, f := range task.FailedDocs { @@ -118,20 +118,20 @@ OUTA: mismatchedDocsTable.Render() } - missingDocsTable := tablewriter.NewWriter(strBuilder) - missingDocsTableRows := types.ToNumericTypeOf(0, verifier.failureDisplaySize) - missingDocsTable.SetHeader([]string{"Document ID", "Source NameSpace", "Destination Namespace"}) + missingOrChangedDocsTable := tablewriter.NewWriter(strBuilder) + missingOrChangedDocsTableRows := types.ToNumericTypeOf(0, verifier.failureDisplaySize) + missingOrChangedDocsTable.SetHeader([]string{"Document ID", "Source Namespace", "Destination Namespace"}) - printAll = int64(missing) < (verifier.failureDisplaySize + int64(0.25*float32(verifier.failureDisplaySize))) + printAll = int64(missingOrChangedCount) < (verifier.failureDisplaySize + int64(0.25*float32(verifier.failureDisplaySize))) OUTB: for _, task := range failedTasks { for _, _id := range task.Ids { - if !printAll && missingDocsTableRows >= verifier.failureDisplaySize { + if !printAll && missingOrChangedDocsTableRows >= verifier.failureDisplaySize { break OUTB } - missingDocsTableRows++ - missingDocsTable.Append([]string{ + missingOrChangedDocsTableRows++ + missingOrChangedDocsTable.Append([]string{ fmt.Sprintf("%v", _id), fmt.Sprintf("%v", task.QueryFilter.Namespace), fmt.Sprintf("%v", task.QueryFilter.To), @@ -139,14 +139,14 @@ OUTB: } } - if missingDocsTableRows > 0 { + if missingOrChangedDocsTableRows > 0 { strBuilder.WriteString("\n") if printAll { - strBuilder.WriteString("All documents present in source/destination missing in destination/source:\n") + strBuilder.WriteString("All documents marked missing or changed:\n") } else { - strBuilder.WriteString(fmt.Sprintf("First %d documents present in source/destination missing in destination/source:\n", verifier.failureDisplaySize)) + strBuilder.WriteString(fmt.Sprintf("First %d documents marked missing or changed:\n", verifier.failureDisplaySize)) } - missingDocsTable.Render() + missingOrChangedDocsTable.Render() } return true, anyAreIncomplete, nil