Preserve partition order when returning (#27)

eapolinario · Dec 17, 2019 · c38a569 · c38a569
1 parent dbac744
commit c38a569
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 10 deletions.
diff --git a/datacatalog/pkg/repositories/gormimpl/artifact.go b/datacatalog/pkg/repositories/gormimpl/artifact.go
@@ -53,7 +53,11 @@ func (h *artifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.A
 	defer timer.Stop()
 
 	var artifact models.Artifact
-	result := h.db.Preload("ArtifactData").Preload("Partitions").Preload("Tags").
+	result := h.db.Preload("ArtifactData").
+		Preload("Partitions", func(db *gorm.DB) *gorm.DB {
+			return db.Order("partitions.created_at ASC") // preserve the order in which the partitions were created
+		}).
+		Preload("Tags").
 		Order("artifacts.created_at DESC").
 		First(
 			&artifact,
@@ -102,7 +106,11 @@ func (h *artifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, i
 		return []models.Artifact{}, h.errorTransformer.ToDataCatalogError(tx.Error)
 	}
 
-	tx = tx.Preload("ArtifactData").Preload("Partitions").Preload("Tags").Find(&artifacts)
+	tx = tx.Preload("ArtifactData").
+		Preload("Partitions", func(db *gorm.DB) *gorm.DB {
+			return db.Order("partitions.created_at ASC") // preserve the order in which the partitions were created
+		}).
+		Preload("Tags").Find(&artifacts)
 	if tx.Error != nil {
 		return []models.Artifact{}, h.errorTransformer.ToDataCatalogError(tx.Error)
 	}

diff --git a/datacatalog/pkg/repositories/gormimpl/artifact_test.go b/datacatalog/pkg/repositories/gormimpl/artifact_test.go
@@ -179,7 +179,7 @@ func TestGetArtifact(t *testing.T) {
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "artifact_data"  WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123)))) ORDER BY "artifact_data"."dataset_project" ASC`).WithReply(expectedArtifactDataResponse)
 	GlobalMock.NewMock().WithQuery(
-		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY "partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse)
+		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse)
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "tags"  WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid)))) ORDER BY "tags"."dataset_project" ASC`).WithReply(expectedTagResponse)
 	getInput := models.ArtifactKey{
@@ -221,7 +221,7 @@ func TestGetArtifactByID(t *testing.T) {
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "artifact_data"  WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123)))) ORDER BY "artifact_data"."dataset_project" ASC`).WithReply(expectedArtifactDataResponse)
 	GlobalMock.NewMock().WithQuery(
-		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY "partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse)
+		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse)
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "tags"  WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid)))) ORDER BY "tags"."dataset_project" ASC`).WithReply(expectedTagResponse)
 	getInput := models.ArtifactKey{
@@ -293,7 +293,7 @@ func TestListArtifactsWithPartition(t *testing.T) {
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "artifact_data"  WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(expectedArtifactDataResponse)
 	GlobalMock.NewMock().WithQuery(
-		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123)))`).WithReply(expectedPartitionResponse)
+		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC`).WithReply(expectedPartitionResponse)
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "tags"  WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid))))`).WithReply(expectedTagResponse)
 

diff --git a/datacatalog/pkg/repositories/gormimpl/dataset.go b/datacatalog/pkg/repositories/gormimpl/dataset.go
@@ -45,7 +45,9 @@ func (h *dataSetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Dat
 	defer timer.Stop()
 
 	var ds models.Dataset
-	result := h.db.Preload("PartitionKeys").First(&ds, &models.Dataset{DatasetKey: in})
+	result := h.db.Preload("PartitionKeys", func(db *gorm.DB) *gorm.DB {
+		return db.Order("partition_keys.created_at ASC") // preserve the order in which the partitions were created
+	}).First(&ds, &models.Dataset{DatasetKey: in})
 
 	if result.Error != nil {
 		logger.Debugf(ctx, "Unable to find Dataset: [%+v], err: %v", in, result.Error)

diff --git a/datacatalog/pkg/repositories/gormimpl/dataset_test.go b/datacatalog/pkg/repositories/gormimpl/dataset_test.go
@@ -165,7 +165,7 @@ func TestGetDataset(t *testing.T) {
 	samplePartitionKey["dataset_uuid"] = getDatasetUUID()
 	expectedPartitionKeyResponse = append(expectedPartitionKeyResponse, samplePartitionKey, samplePartitionKey)
 
-	GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys"  WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid))) ORDER BY "partition_keys"."dataset_uuid" ASC`).WithReply(expectedPartitionKeyResponse)
+	GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys"  WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid))) ORDER BY partition_keys.created_at ASC,"partition_keys"."dataset_uuid" ASC`).WithReply(expectedPartitionKeyResponse)
 	datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope())
 	actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey)
 	assert.NoError(t, err)

diff --git a/datacatalog/pkg/repositories/gormimpl/tag.go b/datacatalog/pkg/repositories/gormimpl/tag.go
@@ -42,8 +42,12 @@ func (h *tagRepo) Get(ctx context.Context, in models.TagKey) (models.Tag, error)
 	defer timer.Stop()
 
 	var tag models.Tag
-	result := h.db.Preload("Artifact").Preload("Artifact.ArtifactData").
-		Preload("Artifact.Partitions").Preload("Artifact.Tags").
+	result := h.db.Preload("Artifact").
+		Preload("Artifact.ArtifactData").
+		Preload("Artifact.Partitions", func(db *gorm.DB) *gorm.DB {
+			return db.Order("partitions.created_at ASC") // preserve the order in which the partitions were created
+		}).
+		Preload("Artifact.Tags").
 		Order("tags.created_at DESC").
 		First(&tag, &models.Tag{
 			TagKey: in,

diff --git a/datacatalog/pkg/repositories/gormimpl/tag_test.go b/datacatalog/pkg/repositories/gormimpl/tag_test.go
@@ -77,7 +77,7 @@ func TestGetTag(t *testing.T) {
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "artifact_data"  WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(getDBArtifactDataResponse(artifact))
 	GlobalMock.NewMock().WithQuery(
-		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123)))`).WithReply(getDBPartitionResponse(artifact))
+		`SELECT * FROM "partitions"  WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(getDBPartitionResponse(artifact))
 	GlobalMock.NewMock().WithQuery(
 		`SELECT * FROM "tags"  WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid))))`).WithReply(getDBTagResponse(artifact))
 	getInput := models.TagKey{