Skip to content

Commit

Permalink
fix: Correct broken document migration in 2.0.0
Browse files Browse the repository at this point in the history
Changelog: Title
Ticket: MEN-5718

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>
  • Loading branch information
alfrunes committed Jul 11, 2022
1 parent be0817c commit fce72de
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 41 deletions.
26 changes: 9 additions & 17 deletions store/mongo/migration_2_0_0.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2021 Northern.tech AS
// Copyright 2022 Northern.tech AS
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -43,18 +43,7 @@ func (m *migration_2_0_0) Up(from migrate.Version) error {
collections := map[string]struct {
Indexes []mongo.IndexModel
}{
DevicesCollectionName: {
Indexes: []mongo.IndexModel{
{
Keys: bson.D{
{Key: mstore.FieldTenantID, Value: 1},
{Key: dbFieldID, Value: 1},
},
Options: mopts.Index().
SetName(mstore.FieldTenantID + "_" + dbFieldID),
},
},
},
DevicesCollectionName: {},
SessionsCollectionName: {
Indexes: []mongo.IndexModel{
{
Expand Down Expand Up @@ -105,16 +94,18 @@ func (m *migration_2_0_0) Up(from migrate.Version) error {
Tenant: tenantID,
})

for collection := range collections {
for collection, idxes := range collections {
// get all the documents in the collection
findOptions := mopts.Find().
SetBatchSize(findBatchSize).
SetSort(bson.D{{Key: dbFieldID, Value: 1}})
coll := client.Database(m.db).Collection(collection)
collOut := client.Database(DbName).Collection(collection)
_, err := collOut.Indexes().CreateMany(ctx, collections[collection].Indexes)
if err != nil {
return err
if len(idxes.Indexes) > 0 {
_, err := collOut.Indexes().CreateMany(ctx, collections[collection].Indexes)
if err != nil {
return err
}
}

cur, err := coll.Find(ctx, bson.D{}, findOptions)
Expand Down Expand Up @@ -148,6 +139,7 @@ func (m *migration_2_0_0) Up(from migrate.Version) error {
writes = append(writes, mongo.NewInsertOneModel().SetDocument(item))
}
if len(writes) == findBatchSize {
_, err = collOut.BulkWrite(ctx, writes)
if err != nil {
return err
}
Expand Down
101 changes: 77 additions & 24 deletions store/mongo/migration_2_0_0_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2021 Northern.tech AS
// Copyright 2022 Northern.tech AS
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -16,10 +16,12 @@ package mongo

import (
"context"
"strconv"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"

"github.com/mendersoftware/go-lib-micro/mongo/migrate"
mstore "github.com/mendersoftware/go-lib-micro/store/v2"
Expand All @@ -37,36 +39,87 @@ func TestMigration_2_0_0(t *testing.T) {
db: DbName,
}
from := migrate.MakeVersion(0, 0, 0)
db := db.Client().Database(DbName)

err := m.Up(from)
require.NoError(t, err)

iv := db.Client().Database(DbName).
Collection(DevicesCollectionName).
Indexes()
collDevs := db.Client().Database(DbName).
Collection(DevicesCollectionName)
ctx := context.Background()
cur, err := iv.List(ctx)
require.NoError(t, err)

var idxes []index
err = cur.All(ctx, &idxes)
_, err := collDevs.InsertMany(ctx, func() []interface{} {
ret := make([]interface{}, findBatchSize+1)
for i := range ret {
ret[i] = bson.D{{Key: "_id", Value: strconv.Itoa(i)}}
}
return ret
}())
if !assert.NoError(t, err) {
t.FailNow()
}
err = m.Up(from)
require.NoError(t, err)
require.Len(t, idxes, 2)
for _, idx := range idxes {
if _, ok := idx.Keys[dbFieldID]; ok && len(idx.Keys) == 1 {
// Skip default index
continue

collNames, err := db.ListCollectionNames(ctx, bson.D{})
for _, collName := range collNames {
idxes, err := db.Collection(collName).
Indexes().
ListSpecifications(ctx)
require.NoError(t, err)
expectedLen := 1
switch collName {
case RecordingsCollectionName:
expectedLen = 3
case SessionsCollectionName, ControlCollectionName:
expectedLen = 2
}
switch idx.Name {
case mstore.FieldTenantID + "_" + dbFieldID:
assert.Equal(t, map[string]int{
dbFieldID: 1,
mstore.FieldTenantID: 1,
}, idx.Keys)
require.Len(t, idxes, expectedLen)
for _, idx := range idxes {
if idx.Name == "_id_" {
// Skip default index
continue
}
switch idx.Name {
case IndexNameLogsExpire:
var keys bson.D
assert.True(t, idx.ExpireAfterSeconds != nil, "Expected index to have TTL")
err := bson.Unmarshal(idx.KeysDocument, &keys)
require.NoError(t, err)
assert.Equal(t, bson.D{
{Key: "expire_ts", Value: int32(1)},
}, keys)

default:
assert.Failf(t, "index test failed", "Index name \"%s\" not recognized", idx.Name)
case mstore.FieldTenantID + "_" + dbFieldSessionID:
var keys bson.D
err := bson.Unmarshal(idx.KeysDocument, &keys)
require.NoError(t, err)
assert.Equal(t, bson.D{
{Key: mstore.FieldTenantID, Value: int32(1)},
{Key: dbFieldSessionID, Value: int32(1)},
}, keys)

case mstore.FieldTenantID + "_" + dbFieldDeviceID:
var keys bson.D
err := bson.Unmarshal(idx.KeysDocument, &keys)
require.NoError(t, err)
assert.Equal(t, bson.D{
{Key: mstore.FieldTenantID, Value: int32(1)},
{Key: dbFieldDeviceID, Value: int32(1)},
}, keys)

default:
assert.Failf(t, "index test failed", "Index name \"%s\" not recognized", idx.Name)
}
}
}
assert.Equal(t, "2.0.0", m.Version().String())

actual, err := collDevs.CountDocuments(ctx, bson.D{
{Key: mstore.FieldTenantID, Value: bson.D{{Key: "$exists", Value: true}}},
})
if assert.NoError(t, err) {
expected, err := collDevs.CountDocuments(ctx, bson.D{})
if assert.NoError(t, err) {
assert.Equalf(t, expected, actual,
"%d documents does not contain 'tenant_id' key", expected-actual)
}
}
}

0 comments on commit fce72de

Please sign in to comment.