DMS Serverless task using AWS CLI v2

Subhash Raut Subhash Raut

Subhash Raut

Database Migration Specialist Solution Architect | AWS

发布日期: 2023年8月2日
+ 关注

This step by step cli guide to perform AWS DMS serverless migration

  • Create source MySQL endpoint
  • Create Target PostgreSQL endpoint
  • Create AWS DMS replication config
  • Start AWS DMS replication - fresh start
  • Stop AWS DMS replication
  • Modify AWS DMS replication config
  • Resume AWS DMS replication
  • Start AWS DMS replication - reload target
  • Check AWS DMS replication table statistics
  • Cleanup - delete AWS DMS replication config, source and target endpoints.

# mysql source endpoint 
aws dms create-endpoint \
--endpoint-identifier climysqlsource \
--endpoint-type source \
--engine-name mysql \
--server-name rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com \
--username dbuser \
--password <password>  \
--port 3306 \
--tags Key="Name",Value="cli dms mysql source endpoint"


{
    "Endpoint": 
    {
        "EndpointIdentifier": "climysqlsource",
        "EndpointType": "SOURCE",
        "EngineName": "mysql",
        "EngineDisplayName": "MySQL",
        "Username": "dbuser",
        "ServerName": "rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
        "Port": 3306,
        "TransferFiles": false,
        "ReceiveTransferredFiles": false,
        "Status": "active",
        "KmsKeyId": "arn:aws:kms:ap-southeast-2:123456789012:key/ab12cd3e-fg45-6hij-789k-l0l1m234n5o6",
        "EndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "SslMode": "none",
        "MySQLSettings": 
        {
            "Port": 3306,
            "ServerName": "rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
            "Username": "dbuser"
        }
    }
}



# describe the endpoint
aws dms describe-endpoints --query "Endpoints[?EngineName=='mysql' && contains(EndpointIdentifier,'cli')]"


[
    {
        "EndpointIdentifier": "climysqlsource",
        "EndpointType": "SOURCE",
        "EngineName": "mysql",
        "EngineDisplayName": "MySQL",
        "Username": "dbuser",
        "ServerName": "rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
        "Port": 3306,
        "TransferFiles": false,
        "ReceiveTransferredFiles": false,
        "Status": "active",
        "KmsKeyId": "arn:aws:kms:ap-southeast-2:123456789012:key/ab12cd3e-fg45-6hij-789k-l0l1m234n5o6",
        "EndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "SslMode": "none",
        "MySQLSettings": {
            "Port": 3306,
            "ServerName": "rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
            "Username": "dbuser"
        }
    }
]


# psql target endpoint 
aws dms create-endpoint \
--endpoint-identifier clipsqltarget \
--endpoint-type target \
--engine-name postgres \
--server-name rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com \
--username dbuser \
--password <password>  \
--port 5432 \
--database-name mydata \
--tags Key="Name",Value="cli dms psql target endpoint" Key="Application",Value="AWS CLI2"


{
    "Endpoint": 
    {
        "EndpointIdentifier": "clipsqltarget",
        "EndpointType": "TARGET",
        "EngineName": "postgres",
        "EngineDisplayName": "PostgreSQL",
        "Username": "dbuser",
        "ServerName": "rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
        "Port": 5432,
        "DatabaseName": "mydata",
        "TransferFiles": false,
        "ReceiveTransferredFiles": false,
        "Status": "active",
        "KmsKeyId": "arn:aws:kms:ap-southeast-2:123456789012:key/ab12cd3e-fg45-6hij-789k-l0l1m234n5o6",
        "EndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "SslMode": "none",
        "PostgreSQLSettings": 
        {
            "DatabaseName": "mydata",
            "Port": 5432,
            "ServerName": "rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
            "Username": "dbuser"
        }
    }
}



# describe the endpoint
aws dms describe-endpoints --query "Endpoints[?EngineName=='postgres' && contains(EndpointIdentifier,'cli')]"


[
    {
        "EndpointIdentifier": "clipsqltarget",
        "EndpointType": "TARGET",
        "EngineName": "postgres",
        "EngineDisplayName": "PostgreSQL",
        "Username": "dbuser",
        "ServerName": "rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
        "Port": 5432,
        "DatabaseName": "mydata",
        "TransferFiles": false,
        "ReceiveTransferredFiles": false,
        "Status": "active",
        "KmsKeyId": "arn:aws:kms:ap-southeast-2:123456789012:key/ab12cd3e-fg45-6hij-789k-l0l1m234n5o6",
        "EndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "SslMode": "none",
        "PostgreSQLSettings": {
            "DatabaseName": "mydata",
            "Port": 5432,
            "ServerName": "rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
            "Username": "dbuser"
        }
    }
]

#find the precreated DMS replication subnetgroup

领英推荐

aws dms describe-replication-subnet-groups --query "ReplicationSubnetGroups[?contains(ReplicationSubnetGroupIdentifier,'dms')].{ReplicationSubnetGroupIdentifier:ReplicationSubnetGroupIdentifier}"
[
    {
        "ReplicationSubnetGroupIdentifier": "dmsreplicationsubnetgroup-2b2cdefghijklm7no"
    }
]

#find the precreated EC2 Securitygroup 

aws ec2 describe-security-groups --query "SecurityGroups[?contains(GroupName,'EC2')].{grpname:GroupName,id:GroupId}"


[
    {
        "grpname": "rds-myClientSG-12AB3CDEFGHIJ",
        "id": "sg-1123ab4566cd7ef89"
    }
]

#create replication config 

aws dms create-replication-config \
--replication-config-identifier climysql2psql \
--source-endpoint-arn arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F \
--target-endpoint-arn arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA \
--compute-config '{
  "MaxCapacityUnits": 4,
  "MultiAZ": false,
  "ReplicationSubnetGroupId": "dmsreplicationsubnetgroup-2b2cdefghijklm7no",
  "VpcSecurityGroupIds": ["sg-1123ab4566cd7ef89"]
}' \
--replication-type full-load-and-cdc \
--table-mappings '{
    "rules": [
        {
            "rule-id": "783282541",
            "rule-name": "783282541",
            "rule-type": "selection",
            "rule-action": "include",
            "object-locator": {
                "schema-name": "mydata",
                "table-name": "newtable"
            },
            "filters": []
        },
        {
            "rule-id": 1152034124,
            "rule-name": "lkbtwrp3-a00w9q",
            "rule-type": "transformation",
            "rule-target": "schema",
            "rule-action": "rename",
            "object-locator": {
                "schema-name": "mydata"
            },
            "value": "public"
        }
    ]
}' \
--replication-settings '{
    "TargetMetadata": {
    "TargetSchema": "",
    "SupportLobs": true,
    "FullLobMode": false,
    "LobChunkSize": 64,
    "LimitedSizeLobMode": true,
    "LobMaxSize": 32,
    "InlineLobMaxSize": 0,
    "LoadMaxFileSize": 0,
    "ParallelLoadThreads": 0,
    "ParallelLoadBufferSize": 0,
    "BatchApplyEnabled": false,
    "TaskRecoveryTableEnabled": false,
    "ParallelLoadQueuesPerThread": 0,
    "ParallelApplyThreads": 0,
    "ParallelApplyBufferSize": 0,
    "ParallelApplyQueuesPerThread": 0
  },
  "FullLoadSettings": {
    "TargetTablePrepMode": "DROP_AND_CREATE",
    "CreatePkAfterFullLoad": false,
    "StopTaskCachedChangesApplied": false,
    "StopTaskCachedChangesNotApplied": false,
    "MaxFullLoadSubTasks": 8,
    "TransactionConsistencyTimeout": 600,
    "CommitRate": 10000
  },
  "Logging": {
    "EnableLogging": true,
    "EnableLogContext": false,
    "LogComponents": [
      { "Id": "DATA_STRUCTURE", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "COMMUNICATION", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "IO", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "COMMON", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "FILE_FACTORY", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "FILE_TRANSFER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "REST_SERVER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "ADDONS", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TARGET_LOAD", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TARGET_APPLY", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "SOURCE_UNLOAD", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "SOURCE_CAPTURE", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TRANSFORMATION", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "SORTER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TASK_MANAGER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TABLES_MANAGER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "METADATA_MANAGER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "PERFORMANCE", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "VALIDATOR_EXT", "Severity": "LOGGER_SEVERITY_DEFAULT" }
    ],
    "CloudWatchLogGroup": null,
    "CloudWatchLogStream": null
  },
  "ControlTablesSettings": {
    "historyTimeslotInMinutes": 5,
    "ControlSchema": "",
    "HistoryTimeslotInMinutes": 5,
    "HistoryTableEnabled": false,
    "SuspendedTablesTableEnabled": false,
    "StatusTableEnabled": false,
    "FullLoadExceptionTableEnabled": false
  },
  "StreamBufferSettings": {
    "StreamBufferCount": 3,
    "StreamBufferSizeInMB": 8,
    "CtrlStreamBufferSizeInMB": 5
  },
  "ChangeProcessingDdlHandlingPolicy": {
    "HandleSourceTableDropped": true,
    "HandleSourceTableTruncated": true,
    "HandleSourceTableAltered": true
  },
  "ErrorBehavior": {
    "DataErrorPolicy": "LOG_ERROR",
    "EventErrorPolicy": "IGNORE",
    "DataTruncationErrorPolicy": "LOG_ERROR",
    "DataErrorEscalationPolicy": "SUSPEND_TABLE",
    "DataErrorEscalationCount": 0,
    "TableErrorPolicy": "SUSPEND_TABLE",
    "TableErrorEscalationPolicy": "STOP_TASK",
    "TableErrorEscalationCount": 0,
    "RecoverableErrorCount": -1,
    "RecoverableErrorInterval": 5,
    "RecoverableErrorThrottling": true,
    "RecoverableErrorThrottlingMax": 1800,
    "RecoverableErrorStopRetryAfterThrottlingMax": true,
    "ApplyErrorDeletePolicy": "IGNORE_RECORD",
    "ApplyErrorInsertPolicy": "LOG_ERROR",
    "ApplyErrorUpdatePolicy": "LOG_ERROR",
    "ApplyErrorEscalationPolicy": "LOG_ERROR",
    "ApplyErrorEscalationCount": 0,
    "ApplyErrorFailOnTruncationDdl": false,
    "FullLoadIgnoreConflicts": true,
    "FailOnTransactionConsistencyBreached": false,
    "FailOnNoTablesCaptured": true
  },
  "ChangeProcessingTuning": {
    "BatchApplyPreserveTransaction": true,
    "BatchApplyTimeoutMin": 1,
    "BatchApplyTimeoutMax": 30,
    "BatchApplyMemoryLimit": 500,
    "BatchSplitSize": 0,
    "MinTransactionSize": 1000,
    "CommitTimeout": 1,
    "MemoryLimitTotal": 1024,
    "MemoryKeepTime": 60,
    "StatementCacheSize": 50
  },
  "PostProcessingRules": null,
  "CharacterSetSettings": null,
  "LoopbackPreventionSettings": null,
  "BeforeImageSettings": null,
  "FailTaskWhenCleanTaskResourceFailed": false,
  "TTSettings": {
    "EnableTT": false,
    "FailTaskOnTTFailure": false,
    "TTS3Settings": null,
    "TTRecordSettings": null
  }
}' \
--tags Key="Name",Value="cli DMS serverless config mysql2psql" Key="Application",Value="AWS CLI2"


{
    "ReplicationConfig": 
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "ComputeConfig": 
        {
            "MaxCapacityUnits": 4,
            "MultiAZ": false,
            "ReplicationSubnetGroupId": "dmsreplicationsubnetgroup-2b2cdefghijklm7no",
            "VpcSecurityGroupIds": [
                "sg-1123ab4566cd7ef89"
            ]
        },
        "ReplicationSettings": "{\"TargetMetadata\":{\"TargetSchema\":\"\",\"SupportLobs\":true,\"FullLobMode\":false,\"LobChunkSize\":64,\"LimitedSizeLobMode\":true,\"LobMaxSize\":32,\"InlineLobMaxSize\":0,\"LoadMaxFileSize\":0,\"ParallelLoadThreads\":0,\"ParallelLoadBufferSize\":0,\"BatchApplyEnabled\":false,\"TaskRecoveryTableEnabled\":false,\"ParallelLoadQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0},\"FullLoadSettings\":{\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"CreatePkAfterFullLoad\":false,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"MaxFullLoadSubTasks\":8,\"TransactionConsistencyTimeout\":600,\"CommitRate\":10000},\"Logging\":{\"EnableLogging\":true,\"EnableLogContext\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}],\"CloudWatchLogGroup\":null,\"CloudWatchLogStream\":null},\"ControlTablesSettings\":{\"historyTimeslotInMinutes\":5,\"ControlSchema\":\"\",\"HistoryTimeslotInMinutes\":5,\"HistoryTableEnabled\":false,\"SuspendedTablesTableEnabled\":false,\"StatusTableEnabled\":false,\"FullLoadExceptionTableEnabled\":false},\"StreamBufferSettings\":{\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8,\"CtrlStreamBufferSizeInMB\":5},\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true,\"HandleSourceTableAltered\":true},\"ErrorBehavior\":{\"DataErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorEscalationCount\":0,\"TableErrorPolicy\":\"SUSPEND_TABLE\",\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorEscalationCount\":0,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"RecoverableErrorStopRetryAfterThrottlingMax\":true,\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorFailOnTruncationDdl\":false,\"FullLoadIgnoreConflicts\":true,\"FailOnTransactionConsistencyBreached\":false,\"FailOnNoTablesCaptured\":true},\"ChangeProcessingTuning\":{\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMin\":1,\"BatchApplyTimeoutMax\":30,\"BatchApplyMemoryLimit\":500,\"BatchSplitSize\":0,\"MinTransactionSize\":1000,\"CommitTimeout\":1,\"MemoryLimitTotal\":1024,\"MemoryKeepTime\":60,\"StatementCacheSize\":50},\"PostProcessingRules\":null,\"CharacterSetSettings\":null,\"LoopbackPreventionSettings\":null,\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"TTSettings\":{\"EnableTT\":false,\"FailTaskOnTTFailure\":false,\"TTS3Settings\":null,\"TTRecordSettings\":null}}",
        "TableMappings": "{\n    \"rules\": [\n        {\n            \"rule-id\": \"783282541\",\n            \"rule-name\": \"783282541\",\n            \"rule-type\": \"selection\",\n            \"rule-action\": \"include\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\",\n                \"table-name\": \"newtable\"\n            },\n            \"filters\": []\n        },\n        {\n            \"rule-id\": 1152034124,\n            \"rule-name\": \"lkbtwrp3-a00w9q\",\n            \"rule-type\": \"transformation\",\n            \"rule-target\": \"schema\",\n            \"rule-action\": \"rename\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\"\n            },\n            \"value\": \"public\"\n        }\n    ]\n}",
        "ReplicationConfigCreateTime": "2023-07-31T15:28:56.540000+10:00",
        "ReplicationConfigUpdateTime": "2023-07-31T15:28:56.540000+10:00"
    }
}



# describe the newly created replication config
aws dms describe-replication-configs --query "ReplicationConfigs[?contains(ReplicationConfigIdentifier,'cli')]"


[
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "ComputeConfig": {
            "MaxCapacityUnits": 4,
            "MultiAZ": false,
            "ReplicationSubnetGroupId": "dmsreplicationsubnetgroup-2b2cdefghijklm7no",
            "VpcSecurityGroupIds": [
                "sg-1123ab4566cd7ef89"
            ]
        },
        "ReplicationSettings": "{\"TargetMetadata\":{\"TargetSchema\":\"\",\"SupportLobs\":true,\"FullLobMode\":false,\"LobChunkSize\":64,\"LimitedSizeLobMode\":true,\"LobMaxSize\":32,\"InlineLobMaxSize\":0,\"LoadMaxFileSize\":0,\"ParallelLoadThreads\":0,\"ParallelLoadBufferSize\":0,\"BatchApplyEnabled\":false,\"TaskRecoveryTableEnabled\":false,\"ParallelLoadQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0},\"FullLoadSettings\":{\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"CreatePkAfterFullLoad\":false,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"MaxFullLoadSubTasks\":8,\"TransactionConsistencyTimeout\":600,\"CommitRate\":10000},\"Logging\":{\"EnableLogging\":true,\"EnableLogContext\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}],\"CloudWatchLogGroup\":null,\"CloudWatchLogStream\":null},\"ControlTablesSettings\":{\"historyTimeslotInMinutes\":5,\"ControlSchema\":\"\",\"HistoryTimeslotInMinutes\":5,\"HistoryTableEnabled\":false,\"SuspendedTablesTableEnabled\":false,\"StatusTableEnabled\":false,\"FullLoadExceptionTableEnabled\":false},\"StreamBufferSettings\":{\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8,\"CtrlStreamBufferSizeInMB\":5},\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true,\"HandleSourceTableAltered\":true},\"ErrorBehavior\":{\"DataErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorEscalationCount\":0,\"TableErrorPolicy\":\"SUSPEND_TABLE\",\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorEscalationCount\":0,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"RecoverableErrorStopRetryAfterThrottlingMax\":true,\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorFailOnTruncationDdl\":false,\"FullLoadIgnoreConflicts\":true,\"FailOnTransactionConsistencyBreached\":false,\"FailOnNoTablesCaptured\":true},\"ChangeProcessingTuning\":{\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMin\":1,\"BatchApplyTimeoutMax\":30,\"BatchApplyMemoryLimit\":500,\"BatchSplitSize\":0,\"MinTransactionSize\":1000,\"CommitTimeout\":1,\"MemoryLimitTotal\":1024,\"MemoryKeepTime\":60,\"StatementCacheSize\":50},\"PostProcessingRules\":null,\"CharacterSetSettings\":null,\"LoopbackPreventionSettings\":null,\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"TTSettings\":{\"EnableTT\":false,\"FailTaskOnTTFailure\":false,\"TTS3Settings\":null,\"TTRecordSettings\":null}}",
        "TableMappings": "{\n    \"rules\": [\n        {\n            \"rule-id\": \"783282541\",\n            \"rule-name\": \"783282541\",\n            \"rule-type\": \"selection\",\n            \"rule-action\": \"include\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\",\n                \"table-name\": \"newtable\"\n            },\n            \"filters\": []\n        },\n        {\n            \"rule-id\": 1152034124,\n            \"rule-name\": \"lkbtwrp3-a00w9q\",\n            \"rule-type\": \"transformation\",\n            \"rule-target\": \"schema\",\n            \"rule-action\": \"rename\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\"\n            },\n            \"value\": \"public\"\n        }\n    ]\n}",
        "ReplicationConfigCreateTime": "2023-07-31T15:28:56.540000+10:00"
    }
] 



#start the replication (dms serverless task)
aws dms start-replication \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC \
--start-replication-type start-replication


{
    "Replication": {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "Status": "initializing",
        "ProvisionData": {
            "ProvisionedCapacityUnits": 0,
            "IsNewProvisioningAvailable": false
        },
        "FailureMessages": [],
        "ReplicationStats": {
            "FullLoadProgressPercent": 0,
            "ElapsedTimeMillis": 0,
            "TablesLoaded": 0,
            "TablesLoading": 0,
            "TablesQueued": 0,
            "TablesErrored": 0
        },
        "StartReplicationType": "start-replication",
        "ReplicationCreateTime": "2023-07-31T15:28:56.540000+10:00",
        "ReplicationUpdateTime": "2023-07-31T15:28:56.540000+10:00"
    }
}



# query replications
aws dms describe-replications --query "Replications[?contains(ReplicationConfigIdentifier,'cli')]"


[
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "Status": "preparing_metadata_resources",
        "ProvisionData": {
            "ProvisionedCapacityUnits": 0,
            "IsNewProvisioningAvailable": false
        },
        "FailureMessages": [],
        "ReplicationStats": {
            "FullLoadProgressPercent": 0,
            "ElapsedTimeMillis": 0,
            "TablesLoaded": 0,
            "TablesLoading": 0,
            "TablesQueued": 0,
            "TablesErrored": 0
        },
        "StartReplicationType": "start-replication",
        "ReplicationCreateTime": "2023-07-21T16:11:01.687000+10:00",
        "ReplicationUpdateTime": "2023-07-21T16:21:45.586000+10:00"
    }
]



# stop replication 
aws dms stop-replication \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC 


{
    "Replication": 
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "Status": "stopping",
        "ProvisionData": 
        {
            "ProvisionState": "provisioned",
            "ProvisionedCapacityUnits": 4,
            "IsNewProvisioningAvailable": false
        },
        "FailureMessages": [],
        "ReplicationStats": 
        {
            "FullLoadProgressPercent": 100,
            "ElapsedTimeMillis": 1988,
            "TablesLoaded": 1,
            "TablesLoading": 0,
            "TablesQueued": 0,
            "TablesErrored": 0,
            "FreshStartDate": "2023-07-21T16:57:46.566000+10:00",
            "StartDate": "2023-07-21T16:57:46.566000+10:00",
            "FullLoadStartDate": "2023-07-21T16:57:47.089000+10:00",
            "FullLoadFinishDate": "2023-07-21T16:57:49.078000+10:00"
        },
        "StartReplicationType": "start-replication",
        "ReplicationCreateTime": "2023-07-21T16:11:01.687000+10:00",
        "ReplicationUpdateTime": "2023-07-21T16:58:17.988000+10:00"
    }
}


# modify config to turn off logging
aws dms modify-replication-config \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC \
--replication-settings '{
    "TargetMetadata": {
    "TargetSchema": "",
    "SupportLobs": true,
    "FullLobMode": false,
    "LobChunkSize": 64,
    "LimitedSizeLobMode": true,
    "LobMaxSize": 32,
    "InlineLobMaxSize": 0,
    "LoadMaxFileSize": 0,
    "ParallelLoadThreads": 0,
    "ParallelLoadBufferSize": 0,
    "BatchApplyEnabled": false,
    "TaskRecoveryTableEnabled": false,
    "ParallelLoadQueuesPerThread": 0,
    "ParallelApplyThreads": 0,
    "ParallelApplyBufferSize": 0,
    "ParallelApplyQueuesPerThread": 0
  },
  "FullLoadSettings": {
    "TargetTablePrepMode": "DROP_AND_CREATE",
    "CreatePkAfterFullLoad": false,
    "StopTaskCachedChangesApplied": false,
    "StopTaskCachedChangesNotApplied": false,
    "MaxFullLoadSubTasks": 8,
    "TransactionConsistencyTimeout": 600,
    "CommitRate": 10000
  },
  "Logging": {
    "EnableLogging": false,
    "EnableLogContext": false,
    "LogComponents": [
      { "Id": "DATA_STRUCTURE", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "COMMUNICATION", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "IO", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "COMMON", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "FILE_FACTORY", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "FILE_TRANSFER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "REST_SERVER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "ADDONS", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TARGET_LOAD", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TARGET_APPLY", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "SOURCE_UNLOAD", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "SOURCE_CAPTURE", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TRANSFORMATION", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "SORTER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TASK_MANAGER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "TABLES_MANAGER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "METADATA_MANAGER", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "PERFORMANCE", "Severity": "LOGGER_SEVERITY_DEFAULT" },
      { "Id": "VALIDATOR_EXT", "Severity": "LOGGER_SEVERITY_DEFAULT" }
    ],
    "CloudWatchLogGroup": null,
    "CloudWatchLogStream": null
  },
  "ControlTablesSettings": {
    "historyTimeslotInMinutes": 5,
    "ControlSchema": "",
    "HistoryTimeslotInMinutes": 5,
    "HistoryTableEnabled": false,
    "SuspendedTablesTableEnabled": false,
    "StatusTableEnabled": false,
    "FullLoadExceptionTableEnabled": false
  },
  "StreamBufferSettings": {
    "StreamBufferCount": 3,
    "StreamBufferSizeInMB": 8,
    "CtrlStreamBufferSizeInMB": 5
  },
  "ChangeProcessingDdlHandlingPolicy": {
    "HandleSourceTableDropped": true,
    "HandleSourceTableTruncated": true,
    "HandleSourceTableAltered": true
  },
  "ErrorBehavior": {
    "DataErrorPolicy": "LOG_ERROR",
    "EventErrorPolicy": "IGNORE",
    "DataTruncationErrorPolicy": "LOG_ERROR",
    "DataErrorEscalationPolicy": "SUSPEND_TABLE",
    "DataErrorEscalationCount": 0,
    "TableErrorPolicy": "SUSPEND_TABLE",
    "TableErrorEscalationPolicy": "STOP_TASK",
    "TableErrorEscalationCount": 0,
    "RecoverableErrorCount": -1,
    "RecoverableErrorInterval": 5,
    "RecoverableErrorThrottling": true,
    "RecoverableErrorThrottlingMax": 1800,
    "RecoverableErrorStopRetryAfterThrottlingMax": true,
    "ApplyErrorDeletePolicy": "IGNORE_RECORD",
    "ApplyErrorInsertPolicy": "LOG_ERROR",
    "ApplyErrorUpdatePolicy": "LOG_ERROR",
    "ApplyErrorEscalationPolicy": "LOG_ERROR",
    "ApplyErrorEscalationCount": 0,
    "ApplyErrorFailOnTruncationDdl": false,
    "FullLoadIgnoreConflicts": true,
    "FailOnTransactionConsistencyBreached": false,
    "FailOnNoTablesCaptured": true
  },
  "ChangeProcessingTuning": {
    "BatchApplyPreserveTransaction": true,
    "BatchApplyTimeoutMin": 1,
    "BatchApplyTimeoutMax": 30,
    "BatchApplyMemoryLimit": 500,
    "BatchSplitSize": 0,
    "MinTransactionSize": 1000,
    "CommitTimeout": 1,
    "MemoryLimitTotal": 1024,
    "MemoryKeepTime": 60,
    "StatementCacheSize": 50
  },
  "PostProcessingRules": null,
  "CharacterSetSettings": null,
  "LoopbackPreventionSettings": null,
  "BeforeImageSettings": null,
  "FailTaskWhenCleanTaskResourceFailed": false,
  "TTSettings": {
    "EnableTT": false,
    "FailTaskOnTTFailure": false,
    "TTS3Settings": null,
    "TTRecordSettings": null
  }
}' 


{
    "ReplicationConfig": {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "ComputeConfig": {
            "MaxCapacityUnits": 4,
            "MultiAZ": false,
            "ReplicationSubnetGroupId": "dmsreplicationsubnetgroup-2b2cdefghijklm7no",
            "VpcSecurityGroupIds": [
                "sg-1123ab4566cd7ef89"
            ]
        },
        "ReplicationSettings": "{\"Logging\":{\"EnableLogging\":true,\"EnableLogContext\":false,\"LogComponents\":[{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"TRANSFORMATION\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"SOURCE_UNLOAD\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"IO\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"TARGET_LOAD\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"PERFORMANCE\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"SOURCE_CAPTURE\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"SORTER\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"REST_SERVER\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"VALIDATOR_EXT\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"TARGET_APPLY\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"TASK_MANAGER\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"TABLES_MANAGER\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"METADATA_MANAGER\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"FILE_FACTORY\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"COMMON\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"ADDONS\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"DATA_STRUCTURE\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"COMMUNICATION\"},{\"Severity\":\"LOGGER_SEVERITY_DEFAULT\",\"Id\":\"FILE_TRANSFER\"}],\"CloudWatchLogGroup\":null,\"CloudWatchLogStream\":null},\"StreamBufferSettings\":{\"StreamBufferCount\":3,\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferSizeInMB\":8},\"ErrorBehavior\":{\"FailOnNoTablesCaptured\":true,\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"FailOnTransactionConsistencyBreached\":false,\"RecoverableErrorThrottlingMax\":1800,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"ApplyErrorEscalationCount\":0,\"RecoverableErrorStopRetryAfterThrottlingMax\":true,\"RecoverableErrorThrottling\":true,\"ApplyErrorFailOnTruncationDdl\":false,\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"RecoverableErrorCount\":-1,\"DataErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"RecoverableErrorInterval\":5,\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"TableErrorEscalationCount\":0,\"FullLoadIgnoreConflicts\":true,\"DataErrorPolicy\":\"LOG_ERROR\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"ValidationSettings\":{\"ValidationPartialLobSize\":0,\"EnableValidation\":false,\"ThreadCount\":5,\"PartitionSize\":10000,\"RecordFailureDelayLimitInMinutes\":0,\"SkipLobColumns\":false,\"FailureMaxCount\":10000,\"HandleCollationDiff\":false,\"ValidationQueryCdcDelaySeconds\":0,\"ValidationMode\":\"ROW_LEVEL\",\"TableFailureMaxCount\":1000,\"ValidationOnly\":false},\"TTSettings\":{\"TTS3Settings\":null,\"TTRecordSettings\":null,\"FailTaskOnTTFailure\":false,\"EnableTT\":false},\"FullLoadSettings\":{\"CommitRate\":10000,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"MaxFullLoadSubTasks\":8,\"TransactionConsistencyTimeout\":600,\"CreatePkAfterFullLoad\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\"},\"TargetMetadata\":{\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"TargetSchema\":\"\",\"InlineLobMaxSize\":0,\"ParallelLoadQueuesPerThread\":0,\"SupportLobs\":true,\"LobChunkSize\":64,\"TaskRecoveryTableEnabled\":false,\"ParallelLoadThreads\":0,\"LobMaxSize\":32,\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"ParallelLoadBufferSize\":0},\"BeforeImageSettings\":null,\"ControlTablesSettings\":{\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false,\"HistoryTableEnabled\":false,\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false},\"LoopbackPreventionSettings\":null,\"CharacterSetSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingTuning\":{\"StatementCacheSize\":50,\"CommitTimeout\":1,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"BatchApplyTimeoutMax\":30,\"MinTransactionSize\":1000,\"MemoryKeepTime\":60,\"BatchApplyMemoryLimit\":500,\"MemoryLimitTotal\":1024},\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true,\"HandleSourceTableAltered\":true}}",
        "TableMappings": "{\n    \"rules\": [\n        {\n            \"rule-id\": \"783282541\",\n            \"rule-name\": \"783282541\",\n            \"rule-type\": \"selection\",\n            \"rule-action\": \"include\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\",\n                \"table-name\": \"newtable\"\n            },\n            \"filters\": []\n        },\n        {\n            \"rule-id\": 1152034124,\n            \"rule-name\": \"lkbtwrp3-a00w9q\",\n            \"rule-type\": \"transformation\",\n            \"rule-target\": \"schema\",\n            \"rule-action\": \"rename\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\"\n            },\n            \"value\": \"public\"\n        }\n    ]\n}",
        "ReplicationConfigCreateTime": "2023-07-21T16:11:01.687000+10:00",
        "ReplicationConfigUpdateTime": "2023-07-21T16:55:58.862000+10:00"
    }
}



# resume replication
aws dms start-replication \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC \
--start-replication-type resume-processing


{
    "Replication": 
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "Status": "initializing",
        "ProvisionData": 
        {
            "ProvisionState": "provisioned",
            "ProvisionedCapacityUnits": 4,
            "IsNewProvisioningAvailable": false
        },
        "FailureMessages": [],
        "ReplicationStats": 
        {
            "FullLoadProgressPercent": 100,
            "ElapsedTimeMillis": 242,
            "TablesLoaded": 1,
            "TablesLoading": 0,
            "TablesQueued": 0,
            "TablesErrored": 0,
            "FreshStartDate": "2023-07-31T16:10:36.974000+10:00",
            "StartDate": "2023-07-31T16:10:36.974000+10:00",
            "StopDate": "2023-07-31T16:13:47.002000+10:00",
            "FullLoadStartDate": "2023-07-31T16:10:38.977000+10:00",
            "FullLoadFinishDate": "2023-07-31T16:10:39.220000+10:00"
        },
        "StartReplicationType": "resume-processing",
        "RecoveryCheckpoint": "checkpoint:V1#1#mysql-bin-changelog.009875:153:-1:153:0#0#0#*#0#0",
        "ReplicationCreateTime": "2023-07-31T15:28:56.540000+10:00",
        "ReplicationUpdateTime": "2023-07-31T16:14:40.753000+10:00"
    }
}



# reload target 
aws dms start-replication \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC \
--start-replication-type reload-target


{
    "Replication": 
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "Status": "initializing",
        "ProvisionData": 
        {
            "ProvisionState": "provisioned",
            "ProvisionedCapacityUnits": 4,
            "IsNewProvisioningAvailable": false
        },
        "FailureMessages": [],
        "ReplicationStats": 
        {
            "FullLoadProgressPercent": 100,
            "ElapsedTimeMillis": 281,
            "TablesLoaded": 1,
            "TablesLoading": 0,
            "TablesQueued": 0,
            "TablesErrored": 0,
            "FreshStartDate": "2023-07-31T16:06:35.627000+10:00",
            "StartDate": "2023-07-31T16:06:35.627000+10:00",
            "StopDate": "2023-07-31T16:09:55.643000+10:00",
            "FullLoadStartDate": "2023-07-31T16:06:37.834000+10:00",
            "FullLoadFinishDate": "2023-07-31T16:06:38.115000+10:00"
        },
        "StartReplicationType": "reload-target",
        "RecoveryCheckpoint": "checkpoint:V1#3#mysql-bin-changelog.009874:487:-1:514:42408507080932:-:0#0#0#*#0#0",
        "ReplicationCreateTime": "2023-07-31T15:28:56.540000+10:00",
        "ReplicationUpdateTime": "2023-07-31T16:10:12.123000+10:00"
    }
}



# check table statistics
aws dms describe-replication-table-statistics \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC


{
    "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
    "ReplicationTableStatistics": 
    [
        {
            "SchemaName": "mydata",
            "TableName": "newtable",
            "Inserts": 0,
            "Deletes": 0,
            "Updates": 0,
            "Ddls": 0,
            "AppliedInserts": 0,
            "AppliedDeletes": 0,
            "AppliedUpdates": 0,
            "AppliedDdls": 0,
            "FullLoadRows": 29,
            "FullLoadCondtnlChkFailedRows": 0,
            "FullLoadErrorRows": 0,
            "FullLoadStartTime": "2023-07-31T16:10:38.977000+10:00",
            "FullLoadEndTime": "2023-07-31T16:10:39.220000+10:00",
            "FullLoadReloaded": false,
            "LastUpdateTime": "2023-07-31T16:10:42.116000+10:00",
            "TableState": "Table completed",
            "ScheduledForReload": false,
            "ValidationPendingRecords": 0,
            "ValidationFailedRecords": 0,
            "ValidationSuspendedRecords": 0,
            "ValidationState": "Not enabled"
        }
    ]
}



# cleanup
# first stop the task
aws dms stop-replication \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC 


{
    "Replication": 
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "Status": "stopping",
        "ProvisionData": 
        {
            "ProvisionState": "provisioned",
            "ProvisionedCapacityUnits": 4,
            "IsNewProvisioningAvailable": false
        },
        "FailureMessages": [],
        "ReplicationStats": 
        {
            "FullLoadProgressPercent": 100,
            "ElapsedTimeMillis": 1924,
            "TablesLoaded": 1,
            "TablesLoading": 0,
            "TablesQueued": 0,
            "TablesErrored": 0,
            "FreshStartDate": "2023-07-31T16:06:35.627000+10:00",
            "StartDate": "2023-07-31T16:06:35.627000+10:00",
            "FullLoadStartDate": "2023-07-31T16:06:36.191000+10:00",
            "FullLoadFinishDate": "2023-07-31T16:06:38.115000+10:00"
        },
        "StartReplicationType": "start-replication",
        "ReplicationCreateTime": "2023-07-31T15:28:56.540000+10:00",
        "ReplicationUpdateTime": "2023-07-31T16:07:40.251000+10:00"
    }
}



# delete replication config
aws dms delete-replication-config \
--replication-config-arn arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC


{
    "ReplicationConfig": 
    {
        "ReplicationConfigIdentifier": "climysql2psql",
        "ReplicationConfigArn": "arn:aws:dms:ap-southeast-2:123456789012:replication-config:ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC",
        "SourceEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "TargetEndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "ReplicationType": "full-load-and-cdc",
        "ComputeConfig": 
        {
            "MaxCapacityUnits": 4,
            "MultiAZ": false,
            "ReplicationSubnetGroupId": "dmsreplicationsubnetgroup-2b2cdefghijklm7no",
            "VpcSecurityGroupIds": [
                "sg-1123ab4566cd7ef89"
            ]
        },
        "ReplicationSettings": "{\"TargetMetadata\":{\"TargetSchema\":\"\",\"SupportLobs\":true,\"FullLobMode\":false,\"LobChunkSize\":64,\"LimitedSizeLobMode\":true,\"LobMaxSize\":32,\"InlineLobMaxSize\":0,\"LoadMaxFileSize\":0,\"ParallelLoadThreads\":0,\"ParallelLoadBufferSize\":0,\"BatchApplyEnabled\":false,\"TaskRecoveryTableEnabled\":false,\"ParallelLoadQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0},\"FullLoadSettings\":{\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"CreatePkAfterFullLoad\":false,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"MaxFullLoadSubTasks\":8,\"TransactionConsistencyTimeout\":600,\"CommitRate\":10000},\"Logging\":{\"EnableLogging\":true,\"EnableLogContext\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}],\"CloudWatchLogGroup\":\"dms-serverless-replication-ABCXXDEFGHIJJXABC4ABC2ABCABCVAB2TNQ3ABC\",\"CloudWatchLogStream\":\"dms-serverless-serv-res-id-1690783496649-cvz\"},\"ControlTablesSettings\":{\"historyTimeslotInMinutes\":5,\"ControlSchema\":\"\",\"HistoryTimeslotInMinutes\":5,\"HistoryTableEnabled\":false,\"SuspendedTablesTableEnabled\":false,\"StatusTableEnabled\":false,\"FullLoadExceptionTableEnabled\":false},\"StreamBufferSettings\":{\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8,\"CtrlStreamBufferSizeInMB\":5},\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true,\"HandleSourceTableAltered\":true},\"ErrorBehavior\":{\"DataErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorEscalationCount\":0,\"TableErrorPolicy\":\"SUSPEND_TABLE\",\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorEscalationCount\":0,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"RecoverableErrorStopRetryAfterThrottlingMax\":true,\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorFailOnTruncationDdl\":false,\"FullLoadIgnoreConflicts\":true,\"FailOnTransactionConsistencyBreached\":false,\"FailOnNoTablesCaptured\":true},\"ChangeProcessingTuning\":{\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMin\":1,\"BatchApplyTimeoutMax\":30,\"BatchApplyMemoryLimit\":500,\"BatchSplitSize\":0,\"MinTransactionSize\":1000,\"CommitTimeout\":1,\"MemoryLimitTotal\":1024,\"MemoryKeepTime\":60,\"StatementCacheSize\":50},\"PostProcessingRules\":null,\"CharacterSetSettings\":null,\"LoopbackPreventionSettings\":null,\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"TTSettings\":{\"EnableTT\":false,\"FailTaskOnTTFailure\":false,\"TTS3Settings\":null,\"TTRecordSettings\":null}}",
        "TableMappings": "{\n    \"rules\": [\n        {\n            \"rule-id\": \"783282541\",\n            \"rule-name\": \"783282541\",\n            \"rule-type\": \"selection\",\n            \"rule-action\": \"include\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\",\n                \"table-name\": \"newtable\"\n            },\n            \"filters\": []\n        },\n        {\n            \"rule-id\": 1152034124,\n            \"rule-name\": \"lkbtwrp3-a00w9q\",\n            \"rule-type\": \"transformation\",\n            \"rule-target\": \"schema\",\n            \"rule-action\": \"rename\",\n            \"object-locator\": {\n                \"schema-name\": \"mydata\"\n            },\n            \"value\": \"public\"\n        }\n    ]\n}",
        "ReplicationConfigCreateTime": "2023-07-31T15:28:56.540000+10:00",
        "ReplicationConfigUpdateTime": "2023-07-31T16:04:56.847000+10:00"
    }
}



# Delete the source endpoint
aws dms delete-endpoint \
--endpoint-arn arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F


{
    "Endpoint": 
    {
        "EndpointIdentifier": "climysqlsource",
        "EndpointType": "SOURCE",
        "EngineName": "mysql",
        "EngineDisplayName": "MySQL",
        "Username": "dbuser",
        "ServerName": "rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
        "Port": 3306,
        "TransferFiles": false,
        "ReceiveTransferredFiles": false,
        "Status": "deleting",
        "KmsKeyId": "arn:aws:kms:ap-southeast-2:123456789012:key/ab12cd3e-fg45-6hij-789k-l0l1m234n5o6",
        "EndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:ABCD4E67FGHIJJKLMNOPQRSTUUVWXY2ZABCDE3F",
        "SslMode": "none",
        "MySQLSettings": 
        {
            "Port": 3306,
            "ServerName": "rds-mysqldb-3f346432534.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
            "Username": "dbuser"
        }
    }
}



# Delete the target endpoint
aws dms delete-endpoint \
--endpoint-arn arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA


{
    "Endpoint": 
    {
        "EndpointIdentifier": "clipsqltarget",
        "EndpointType": "TARGET",
        "EngineName": "postgres",
        "EngineDisplayName": "PostgreSQL",
        "Username": "dbuser",
        "ServerName": "rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
        "Port": 5432,
        "DatabaseName": "mydata",
        "TransferFiles": false,
        "ReceiveTransferredFiles": false,
        "Status": "deleting",
        "KmsKeyId": "arn:aws:kms:ap-southeast-2:123456789012:key/ab12cd3e-fg45-6hij-789k-l0l1m234n5o6",
        "EndpointArn": "arn:aws:dms:ap-southeast-2:123456789012:endpoint:HG7FED2CB5AZ7YXWVUTSRQPO3M3LKJIHGFED4CBA",
        "SslMode": "none",
        "PostgreSQLSettings": 
        {
            "DatabaseName": "mydata",
            "Port": 5432,
            "ServerName": "rds-postgresql-cba9abc9defg.abc2ds23sf2.ap-southeast-2.rds.amazonaws.com",
            "Username": "dbuser"
        }
    }
}
Arnab Saha

Passionate AWS Solutions Architect & Database Specialist | Driving Digital Transformation with Expertise in Data & AI Technologies | AWS Certified | Thought Leader | AWS Speaker| Tech Lover | Connector | Friend |

1 å¹´

Thanks for posting

赞
回复
1 次回应
Aditi Upadhyay

STEMinst | AWS Database Migration Specialist

1 å¹´

Heaps useful mate!

赞
回复
1 次回应
查看更多评论

要查看或添加评论，请登录

社区洞察

其他会员也浏览了