Skip to content

Commit

Permalink
Merge pull request #36936 from hashicorp/b-dms-replication-task-settings
Browse files Browse the repository at this point in the history 
`aws_dms_replication_config`, `aws_dms_replication_task`: suppress differences on task settings
  • Loading branch information
@gdavison
gdavison committed Apr 17, 2024
2 parents 9198fc2 + d91e526 commit 1b41728
Show file tree
Hide file tree
Showing 16 changed files with 2,605 additions and 674 deletions.
19 changes: 19 additions & 0 deletions .changelog/36936.txt
View file
@@ -0,0 +1,19 @@
```release-note:bug
resource/aws_dms_replication_task: Allows leaving `replication_task_settings` unset to use default settings.
```

```release-note:bug
resource/aws_dms_replication_task: Suppresses differences in partial `replication_task_settings` JSON documents.
```

```release-note:bug
resource/aws_dms_replication_task: Adds validation to `replication_task_settings` to disallow `Logging.CloudWatchLogGroup` and `Logging.CloudWatchLogStream`.
```

```release-note:bug
resource/aws_dms_replication_config: Suppresses differences in partial `replication_settings` JSON documents.
```

```release-note:bug
resource/aws_dms_replication_config: Adds validation to `replication_settings` to disallow `Logging.CloudWatchLogGroup` and `Logging.CloudWatchLogStream`.
```
24 changes: 24 additions & 0 deletions internal/service/dms/dms_test.go
View file
@@ -0,0 +1,24 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package dms_test

import (
"testing"

"github.com/hashicorp/terraform-plugin-testing/helper/resource"
"github.com/hashicorp/terraform-provider-aws/internal/acctest"
"github.com/hashicorp/terraform-provider-aws/names"
)

func init() {
acctest.RegisterServiceErrorCheckFunc(names.DMSServiceID, testAccErrorCheckSkip)
}

// testAccErrorCheckSkip skips DMS tests that have error messages indicating unsupported features
func testAccErrorCheckSkip(t *testing.T) resource.ErrorCheckFunc {
return acctest.ErrorCheckSkipMessagesContaining(t,
// Serverless DMS in GovCloud
"SERVERLESS feature is not available",
)
}
144 changes: 135 additions & 9 deletions internal/service/dms/endpoint_test.go
View file
Expand Up @@ -9,6 +9,7 @@ import (
"testing"

"github.com/YakDriver/regexache"
dms "github.com/aws/aws-sdk-go/service/databasemigrationservice"
sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest"
"github.com/hashicorp/terraform-plugin-testing/helper/resource"
"github.com/hashicorp/terraform-plugin-testing/terraform"
Expand Down Expand Up @@ -2211,6 +2212,7 @@ func TestAccDMSEndpoint_pauseReplicationTasks(t *testing.T) {
endpointNameSource := "aws_dms_endpoint.source"
endpointNameTarget := "aws_dms_endpoint.target"
replicationTaskName := "aws_dms_replication_task.test"
var task dms.ReplicationTask

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t) },
Expand All @@ -2223,7 +2225,7 @@ func TestAccDMSEndpoint_pauseReplicationTasks(t *testing.T) {
Check: resource.ComposeTestCheckFunc(
testAccCheckEndpointExists(ctx, endpointNameSource),
testAccCheckEndpointExists(ctx, endpointNameTarget),
testAccCheckReplicationTaskExists(ctx, replicationTaskName),
testAccCheckReplicationTaskExists(ctx, replicationTaskName, &task),
resource.TestCheckResourceAttr(replicationTaskName, "status", "running"),
),
},
Expand All @@ -2232,7 +2234,7 @@ func TestAccDMSEndpoint_pauseReplicationTasks(t *testing.T) {
Check: resource.ComposeTestCheckFunc(
testAccCheckEndpointExists(ctx, endpointNameSource),
testAccCheckEndpointExists(ctx, endpointNameTarget),
testAccCheckReplicationTaskExists(ctx, replicationTaskName),
testAccCheckReplicationTaskExists(ctx, replicationTaskName, &task),
resource.TestCheckResourceAttr(replicationTaskName, "status", "running"),
),
},
Expand Down Expand Up @@ -4680,7 +4682,7 @@ resource "aws_kms_key" "test" {
}

func testAccEndpointConfig_pauseReplicationTasks(rName string, pause bool) string {
return acctest.ConfigCompose(testAccRDSClustersConfig_base(rName), fmt.Sprintf(`
return acctest.ConfigCompose(testAccEndpointConfig_rdsClusterBase(rName), fmt.Sprintf(`
resource "aws_dms_endpoint" "source" {
database_name = "tftest"
endpoint_id = "%[1]s-source"
Expand Down Expand Up @@ -4723,12 +4725,26 @@ resource "aws_dms_replication_instance" "test" {
}
resource "aws_dms_replication_task" "test" {
migration_type = "full-load-and-cdc"
replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn
replication_task_id = %[1]q
replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}"
source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn
table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"testrule\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}"
migration_type = "full-load-and-cdc"
replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn
replication_task_id = %[1]q
source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn
table_mappings = jsonencode(
{
"rules" = [
{
"rule-type" = "selection",
"rule-id" = "1",
"rule-name" = "testrule",
"object-locator" = {
"schema-name" = "%%",
"table-name" = "%%"
},
"rule-action" = "include"
}
]
}
)
start_replication_task = true
Expand All @@ -4742,3 +4758,113 @@ resource "aws_dms_replication_task" "test" {
}
`, rName, pause))
}

// testAccEndpointConfig_rdsClusterBase configures a pair of Aurora RDS clusters (and instances) ready for replication.
func testAccEndpointConfig_rdsClusterBase(rName string) string {
return acctest.ConfigCompose(acctest.ConfigVPCWithSubnets(rName, 2), fmt.Sprintf(`
resource "aws_db_subnet_group" "test" {
name = %[1]q
subnet_ids = aws_subnet.test[*].id
}
resource "aws_security_group" "test" {
name = %[1]q
vpc_id = aws_vpc.test.id
ingress {
from_port = 0
to_port = 0
protocol = -1
cidr_blocks = ["0.0.0.0/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = {
Name = %[1]q
}
}
data "aws_rds_engine_version" "default" {
engine = "aurora-mysql"
}
data "aws_rds_orderable_db_instance" "test" {
engine = data.aws_rds_engine_version.default.engine
engine_version = data.aws_rds_engine_version.default.version
preferred_instance_classes = ["db.t3.small", "db.t3.medium", "db.t3.large"]
}
resource "aws_rds_cluster_parameter_group" "test" {
name = "%[1]s-pg-cluster"
family = data.aws_rds_engine_version.default.parameter_group_family
description = "DMS cluster parameter group"
parameter {
name = "binlog_format"
value = "ROW"
apply_method = "pending-reboot"
}
parameter {
name = "binlog_row_image"
value = "Full"
apply_method = "pending-reboot"
}
parameter {
name = "binlog_checksum"
value = "NONE"
apply_method = "pending-reboot"
}
}
resource "aws_rds_cluster" "source" {
cluster_identifier = "%[1]s-aurora-cluster-source"
engine = data.aws_rds_orderable_db_instance.test.engine
engine_version = data.aws_rds_orderable_db_instance.test.engine_version
database_name = "tftest"
master_username = "tftest"
master_password = "mustbeeightcharaters"
skip_final_snapshot = true
vpc_security_group_ids = [aws_security_group.test.id]
db_subnet_group_name = aws_db_subnet_group.test.name
db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.test.name
}
resource "aws_rds_cluster_instance" "source" {
identifier = "%[1]s-source-primary"
cluster_identifier = aws_rds_cluster.source.id
engine = data.aws_rds_orderable_db_instance.test.engine
engine_version = data.aws_rds_orderable_db_instance.test.engine_version
instance_class = data.aws_rds_orderable_db_instance.test.instance_class
db_subnet_group_name = aws_db_subnet_group.test.name
}
resource "aws_rds_cluster" "target" {
cluster_identifier = "%[1]s-aurora-cluster-target"
engine = data.aws_rds_orderable_db_instance.test.engine
engine_version = data.aws_rds_orderable_db_instance.test.engine_version
database_name = "tftest"
master_username = "tftest"
master_password = "mustbeeightcharaters"
skip_final_snapshot = true
vpc_security_group_ids = [aws_security_group.test.id]
db_subnet_group_name = aws_db_subnet_group.test.name
}
resource "aws_rds_cluster_instance" "target" {
identifier = "%[1]s-target-primary"
cluster_identifier = aws_rds_cluster.target.id
engine = data.aws_rds_orderable_db_instance.test.engine
engine_version = data.aws_rds_orderable_db_instance.test.engine_version
instance_class = data.aws_rds_orderable_db_instance.test.instance_class
db_subnet_group_name = aws_db_subnet_group.test.name
}
`, rName))
}
13 changes: 9 additions & 4 deletions internal/service/dms/replication_config.go
View file
Expand Up @@ -113,11 +113,16 @@ func ResourceReplicationConfig() *schema.Resource {
Required: true,
ForceNew: true,
},
// "replication_settings" is equivalent to "replication_task_settings" on "aws_dms_replication_task"
// All changes to this field and supporting tests should be mirrored in "aws_dms_replication_task"
"replication_settings": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ValidateFunc: validation.StringIsJSON,
Type: schema.TypeString,
Optional: true,
Computed: true,
ValidateDiagFunc: validation.AllDiag(
validation.ToDiagFunc(validation.StringIsJSON),
validateReplicationSettings,
),
DiffSuppressFunc: suppressEquivalentTaskSettings,
DiffSuppressOnRefresh: true,
},
Expand Down

0 comments on commit 1b41728

Please sign in to comment.