From 38c968ea2d126bc7cee304300207afc863e0f32e Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 14 Dec 2020 19:59:22 -0300 Subject: [PATCH 01/44] Wait up to 3h because RDS is taking more than 2h to clone our 10TB+ databases While they don't fix it, we increase timeouts --- src/stack_mitosis/interpreter.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index 6ac7b13..d77ef8f 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -75,7 +75,7 @@ (op/completed? (describe rds new-id)))] [id #(op/completed? (describe rds id))]) started (. System (nanoTime)) - ret (wait/poll-until completed-fn {:delay 60000 :max-attempts 120}) + ret (wait/poll-until completed-fn {:delay 60000 :max-attempts 180}) msecs (/ (double (- (. System (nanoTime)) started)) 1000000.0) status (-> (describe rds result-id) :DBInstances first :DBInstanceStatus) msg (format "Completed after %.2fs with status %s" (/ msecs 1000) status)] From 706bdbea228cfa184ca94e7694ffadca4d402596 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Tue, 16 Mar 2021 18:02:45 -0300 Subject: [PATCH 02/44] Grab latest snapshot for source instance We can't create cross-vpc replicas, so we'll need to restore from snapshot --- src/stack_mitosis/cli.clj | 3 ++- src/stack_mitosis/interpreter.clj | 9 +++++++++ src/stack_mitosis/operations.clj | 5 +++++ src/stack_mitosis/planner.clj | 6 +++--- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index a966c5b..7c43587 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -53,7 +53,8 @@ instances (interpreter/databases rds)] (when (interpreter/verify-databases-exist instances [source target]) (let [tags (interpreter/list-tags rds instances target) - plan (plan/replace-tree instances source target + source-snapshot (interpreter/latest-snapshot rds source) + plan (plan/replace-tree instances source source-snapshot target :restart restart :tags tags)] (cond (:plan options) (do (println (flight-plan (interpreter/check-plan instances plan))) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index d77ef8f..1173afa 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -62,6 +62,15 @@ [db-id (:TagList (invoke-logged! rds (op/tags arn)))]))) (into {}))) +(defn latest-snapshot + "Returns the latest snapshot for an instance" + [rds target] + (->> (invoke-logged! rds (op/list-snapshots target)) + (:DBSnapshots) + (sort-by :SnapshotCreateTime) + (last) + (:DBSnapshotIdentifier))) + (defn describe [rds id] (invoke-logged! rds (op/describe id))) diff --git a/src/stack_mitosis/operations.clj b/src/stack_mitosis/operations.clj index e533386..52cfd9a 100644 --- a/src/stack_mitosis/operations.clj +++ b/src/stack_mitosis/operations.clj @@ -38,6 +38,11 @@ {:SourceDBInstanceIdentifier source :DBInstanceIdentifier replica})})) +(defn list-snapshots + ([target] + {:op :DescribeDBSnapshots + :request {:DBInstanceIdentifier target}})) + (defn promote [id] {:op :PromoteReadReplica diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index 36ecadc..ff52342 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -31,7 +31,7 @@ ;; postgres does not allow replica of replica, so need to promote before ;; replicating children (defn copy-tree - [instances source target alias-fn & {:keys [tags]}] + [instances source source-snapshot target alias-fn & {:keys [tags]}] (let [alias-tags (->> tags (map (fn [[db-id instance-tags]] [(alias-fn db-id) instance-tags])) @@ -72,12 +72,12 @@ (map op/delete))) (defn replace-tree - [instances source target & {:keys [restart tags] :or {tags {}}}] + [instances source source-snapshot target & {:keys [restart tags] :or {tags {}}}] ;; actions in copy, rename & delete change the local instances db, so use ;; predict to update that db for calculating next set of operations by ;; applying computation thus far to the initial instances ;; TODO something something sequence monad - (let [copy (copy-tree instances source target + (let [copy (copy-tree instances source source-snapshot target (partial aliased "temp") :tags tags) From 991bc797dfe56ce2fcc05f0d460bbf83b96d0e7c Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Tue, 16 Mar 2021 18:17:40 -0300 Subject: [PATCH 03/44] Map attributes for restore snapshot and the subsequent modify-instance --- src/stack_mitosis/lookup.clj | 95 ++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index e2beaea..c6707d0 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -33,6 +33,60 @@ (and (or (seq? v) (vector? v)) (empty? v)))) +(defn restore-snapshot-attributes + "Creates a list of additional attributes to clone from original instance into + the newly created replica instance. + + https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_RestoreDBInstanceFromDBSnapshot.html + has more information on these attributes." + [original tags] + (let [attributes-to-clone + [:CopyTagsToSnapshot + :PubliclyAccessible + :AutoMinorVersionUpgrade + :DBInstanceClass + ;; :DeletionProtection ; must be false for repeated invocation + ;; :KmsKeyId ;; not supported by restore or modify + ;; but is guaranteed to remain the same + ;; it can only change when we copy a snapshot + ;; :SourceRegion ; not applicable? + :ProcessorFeatures + ;; :UseDefaultProcessorFeatures ; just copy features directly? + :Iops + :StorageType + :MultiAZ] + + translated-attributes + {:Tags tags + :EnableIAMDatabaseAuthentication (:IAMDatabaseAuthenticationEnabled original) + :EnableCloudwatchLogsExports (:EnabledCloudwatchLogsExports original) + :Port (:Port (:Endpoint original)) + + ;; all active security groups ids + :VpcSecurityGroupIds + (->> original + :VpcSecurityGroups + (filter (fn [group] (= (:Status group) "active"))) + (map :VpcSecurityGroupId)) + + ;; first synchronized option group name + :OptionGroupName + (->> original + :OptionGroupMemberships + (some (fn [group] + (and (= (:Status group) "in-sync") + (:OptionGroupName group))))) + ;; TODO map for names on original + ;; :DomainMemberships -> :Domain, :DomainIAMRoleName + }] + (-> original + ;; copy as-is with no translation + (select-keys attributes-to-clone) + ;; Attributes requiring custom rules to extract from original and + ;; translate to key for clone-replica request + (merge (into {} (remove (fn [[_ v]] (nil-or-empty? v)) + translated-attributes)))))) + (defn clone-replica-attributes "Creates a list of additional attributes to clone from original instance into the newly created replica instance. @@ -122,3 +176,44 @@ ;; translate to key for modify-db request (merge (into {} (remove (fn [[_ v]] (nil-or-empty? v)) translated-attributes)))))) + +(defn post-restore-snapshot-attributes + "List of additional attributes to apply after creation. + + Some parameters are not available or applicable at time of creation, so they + need to be applied after. + + https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_ModifyDBInstance.html + has more information on these attributes." + [original] + (let [attributes-to-clone ;; attributes not supported by restore-snapshot + [:MonitoringRoleArn + :MonitoringInterval + :PerformanceInsightsKMSKeyId + ;; :KmsKeyId ;; modify_not_supported + :PerformanceInsightsRetentionPeriod] + + translated-attributes + {:EnablePerformanceInsights (:PerformanceInsightsEnabled original) ;; restore_not_supported + + ;; Triggers "The specified DB instance is already in the target DB subnet group" + ;; probably need to detect if changing? disabling for now + ;; :DBSubnetGroupName (:DBSubnetGroupName (:DBSubnetGroup original)) + ;; first synchronized db parameter group name + :DBParameterGroupName + (->> original + :DBParameterGroups + (some (fn [group] + (and (= (:ParameterApplyStatus group) "in-sync") + (:DBParameterGroupName group)))))}] + (-> original + (select-keys [:PreferredMaintenanceWindow + :PreferredBackupWindow + ;; TODO ? + ;; :AllocatedStorage ;; Tricky, only supports increase + ;; :MaxAllocatedStorage + ]) + ;; Attributes requiring custom rules to extract from original and + ;; translate to key for modify-db request + (merge (into {} (remove (fn [[_ v]] (nil-or-empty? v)) + translated-attributes)))))) From 7efd4b9c911e3e927307863cec99ae99f0ba6177 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Tue, 16 Mar 2021 18:18:48 -0300 Subject: [PATCH 04/44] Restore from snapshot when in different VPCs --- src/stack_mitosis/operations.clj | 9 +++++++++ src/stack_mitosis/planner.clj | 21 ++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/stack_mitosis/operations.clj b/src/stack_mitosis/operations.clj index 52cfd9a..249957e 100644 --- a/src/stack_mitosis/operations.clj +++ b/src/stack_mitosis/operations.clj @@ -38,6 +38,15 @@ {:SourceDBInstanceIdentifier source :DBInstanceIdentifier replica})})) +(defn restore-snapshot + ([snapshot-id source target] (restore-snapshot snapshot-id source target {})) + ([snapshot-id source target attributes] + {:op :RestoreDBInstanceFromDBSnapshot + :request (merge attributes + {:DBSnapshotIdentifier snapshot-id + :DBInstanceIdentifier target}) + :meta {:SourceDBInstanceIdentifier source}})) + (defn list-snapshots ([target] {:op :DescribeDBSnapshots diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index ff52342..3974e93 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -40,11 +40,22 @@ (partial lookup/by-id instances)) (list-tree instances target)) root-id (:DBInstanceIdentifier root) - root-attrs (lookup/clone-replica-attributes root (get alias-tags root-id))] - (into [(op/create-replica source root-id root-attrs) - (op/promote root-id) - ;; postgres only allows backups after promotion - (op/enable-backups root-id (lookup/post-create-replica-attributes root))] + + ;; can't rely on replication when target and source have different VPCs + target-vpc (->> root :DBSubnetGroup :VpcId) + source-instance (lookup/by-id instances source) + source-vpc (->> source-instance :DBSubnetGroup :VpcId) + same-vpc (= source-vpc target-vpc) + + root-attrs (lookup/clone-replica-attributes root (get alias-tags root-id)) + root-restore-attrs (lookup/restore-snapshot-attributes root (get alias-tags root-id))] + (into (if same-vpc + [(op/create-replica source root-id root-attrs) + (op/promote root-id) + ;; postgres only allows backups after promotion + (op/enable-backups root-id (lookup/post-create-replica-attributes root))] + [(op/restore-snapshot source-snapshot source-instance root-id root-restore-attrs) + (op/enable-backups root-id (lookup/post-restore-snapshot-attributes root))]) (mapcat (fn [instance] [(op/create-replica (:ReadReplicaSourceDBInstanceIdentifier instance) From fda7e9ad3ad17cc0a516100e5d7e7ec3f20cef73 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Tue, 16 Mar 2021 18:20:28 -0300 Subject: [PATCH 05/44] Add missing predict method for restore We're not doing any :pre assertions on the instance list bc we only depend on the snapshot being there. I could have added the list of all snapshots and done a pre-assertion, and this might be required if we want to add support for creating snapshots if one is missing, but I won't go there rn, looks messy. --- src/stack_mitosis/predict.clj | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index db1a50c..91d8f84 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -77,6 +77,17 @@ (update (lookup/position instances parent) detach child) (update (lookup/position instances child) promote)))))) +(defmethod predict :RestoreDBInstanceFromDBSnapshot + [instances op] + {:post [(lookup/exists? % (r/db-id op))]} + (let [source-id (->> op :meta :SourceDBInstanceIdentifier) + source-db (lookup/by-id instances source-id)] + (->> op + (:request) + (#(dissoc %2 %1) :DBSnapshotIdentifier) + (merge source-db) + (conj instances)))) + (defmethod predict :ModifyDBInstance [instances op] {:pre [(lookup/exists? instances (r/db-id op))]} From 1ae48d97c3f3c7f853ddb8a19f58b5be6892444d Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Tue, 16 Mar 2021 18:20:57 -0300 Subject: [PATCH 06/44] Mark this as a blocking operation --- src/stack_mitosis/operations.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stack_mitosis/operations.clj b/src/stack_mitosis/operations.clj index 249957e..508f461 100644 --- a/src/stack_mitosis/operations.clj +++ b/src/stack_mitosis/operations.clj @@ -82,7 +82,7 @@ (defn blocking-operation? [action] (contains? #{:CreateDBInstance :CreateDBInstanceReadReplica - :PromoteReadReplica :ModifyDBInstance} (:op action))) + :PromoteReadReplica :ModifyDBInstance :RestoreDBInstanceFromDBSnapshot} (:op action))) (defn transition-to "Maps current rds status to in-progress, failed or done From 77d4a08b3bc0c6b727ffc01d36667c31c5f8d7c9 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 18:23:12 -0300 Subject: [PATCH 07/44] DBSubnetGroup is how we specify the new VPC, gotta use it --- src/stack_mitosis/lookup.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index c6707d0..a2810db 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -61,6 +61,7 @@ :EnableIAMDatabaseAuthentication (:IAMDatabaseAuthenticationEnabled original) :EnableCloudwatchLogsExports (:EnabledCloudwatchLogsExports original) :Port (:Port (:Endpoint original)) + :DBSubnetGroupName (:DBSubnetGroupName (:DBSubnetGroup original)) ;; all active security groups ids :VpcSecurityGroupIds From 2e35b88c4277097238e0c7e4c3e57161195df09a Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 18:23:42 -0300 Subject: [PATCH 08/44] Allow skipping the restore --- src/stack_mitosis/planner.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index 3974e93..7e718ea 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -124,6 +124,9 @@ (and (= op :CreateDBInstanceReadReplica) (lookup/by-id instances (r/db-id action))) [:skip (duplicate-instance (r/db-id action))] + (and (= op :RestoreDBInstanceFromDBSnapshot) + (lookup/by-id instances (r/db-id action))) + [:skip (duplicate-instance (r/db-id action))] (and (= op :PromoteReadReplica) (not (:ReadReplicaSourceDBInstanceIdentifier (lookup/by-id instances (r/db-id action))))) [:skip (promoted-instance (r/db-id action))] From 4799d8ff1076c0cc6ea1737024f2de2989b125eb Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 18:24:41 -0300 Subject: [PATCH 09/44] Add missing properties we don't get on restore --- src/stack_mitosis/lookup.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index a2810db..3bb987e 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -190,8 +190,9 @@ (let [attributes-to-clone ;; attributes not supported by restore-snapshot [:MonitoringRoleArn :MonitoringInterval - :PerformanceInsightsKMSKeyId ;; :KmsKeyId ;; modify_not_supported + :EnhancedMonitoringResourceArn + :PerformanceInsightsKMSKeyId :PerformanceInsightsRetentionPeriod] translated-attributes From 4634dccd06e66599d00aa6257169e45e735e5e14 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 18:25:45 -0300 Subject: [PATCH 10/44] This is the full db instance data, not just the identifier --- src/stack_mitosis/operations.clj | 2 +- src/stack_mitosis/predict.clj | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stack_mitosis/operations.clj b/src/stack_mitosis/operations.clj index 508f461..c2b1b29 100644 --- a/src/stack_mitosis/operations.clj +++ b/src/stack_mitosis/operations.clj @@ -45,7 +45,7 @@ :request (merge attributes {:DBSnapshotIdentifier snapshot-id :DBInstanceIdentifier target}) - :meta {:SourceDBInstanceIdentifier source}})) + :meta {:SourceDBInstance source}})) (defn list-snapshots ([target] diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index 91d8f84..d59109e 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -81,6 +81,7 @@ [instances op] {:post [(lookup/exists? % (r/db-id op))]} (let [source-id (->> op :meta :SourceDBInstanceIdentifier) + (let [source-id (->> op :meta :SourceDBInstance :DBInstanceIdentifier) source-db (lookup/by-id instances source-id)] (->> op (:request) From b1da68593beaf34455cedc74f0b1c7a148fa26cc Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 18:28:03 -0300 Subject: [PATCH 11/44] Add a :pre hook for checking the the db instance exists I think this was necessary it to skip the restore operation --- src/stack_mitosis/predict.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index d59109e..bb0961b 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -79,8 +79,8 @@ (defmethod predict :RestoreDBInstanceFromDBSnapshot [instances op] - {:post [(lookup/exists? % (r/db-id op))]} - (let [source-id (->> op :meta :SourceDBInstanceIdentifier) + {:pre [(->> op :meta :SourceDBInstance :DBInstanceIdentifier (lookup/exists? instances))] + :post [(lookup/exists? % (r/db-id op))]} (let [source-id (->> op :meta :SourceDBInstance :DBInstanceIdentifier) source-db (lookup/by-id instances source-id)] (->> op From 6e8dad25b47520222d7b42e0229d58e1fe692b01 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 19:24:27 -0300 Subject: [PATCH 12/44] Actually make use of attributes-to-clone shall we --- src/stack_mitosis/lookup.clj | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index 3bb987e..1643c17 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -190,10 +190,14 @@ (let [attributes-to-clone ;; attributes not supported by restore-snapshot [:MonitoringRoleArn :MonitoringInterval - ;; :KmsKeyId ;; modify_not_supported - :EnhancedMonitoringResourceArn :PerformanceInsightsKMSKeyId - :PerformanceInsightsRetentionPeriod] + :PerformanceInsightsRetentionPeriod + :PreferredMaintenanceWindow + :PreferredBackupWindow + ;; TODO ? + ;; :AllocatedStorage ;; Tricky, only supports increase + ;; :MaxAllocatedStorage + ] translated-attributes {:EnablePerformanceInsights (:PerformanceInsightsEnabled original) ;; restore_not_supported @@ -209,12 +213,7 @@ (and (= (:ParameterApplyStatus group) "in-sync") (:DBParameterGroupName group)))))}] (-> original - (select-keys [:PreferredMaintenanceWindow - :PreferredBackupWindow - ;; TODO ? - ;; :AllocatedStorage ;; Tricky, only supports increase - ;; :MaxAllocatedStorage - ]) + (select-keys attributes-to-clone) ;; Attributes requiring custom rules to extract from original and ;; translate to key for modify-db request (merge (into {} (remove (fn [[_ v]] (nil-or-empty? v)) From ba8f4dbafa19540489a922b7d61b4998df2c975d Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 19:34:26 -0300 Subject: [PATCH 13/44] Tests are green again --- test/stack_mitosis/planner_test.clj | 10 ++++++---- test/stack_mitosis/policy_test.clj | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index 3dbcfb8..fb4a87b 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -21,6 +21,7 @@ :ReadReplicaSourceDBInstanceIdentifier "target"} {:DBInstanceIdentifier "b" :ReadReplicaSourceDBInstanceIdentifier "target"} {:DBInstanceIdentifier "c" :ReadReplicaSourceDBInstanceIdentifier "b"}] + snapshot-id "rds:source-snapshot-2021-03-17" tags-target [(op/kv "k" "target")] tags-b [(op/kv "k" "b")]] (is (= [(op/create-replica "source" "temp-target" {:Iops 500 :Tags tags-target}) @@ -32,7 +33,7 @@ (op/modify "temp-b" {}) (op/create-replica "temp-b" "temp-c") (op/modify "temp-c" {})] - (plan/copy-tree instances "source" "target" + (plan/copy-tree instances "source" snapshot-id "target" (partial plan/aliased "temp") :tags {"target" tags-target "b" tags-b}))))) @@ -66,6 +67,7 @@ [{:DBInstanceIdentifier "production"} {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"]} {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging"}] + snapshot-id "rds:production-2015-10-21" tags [(op/kv "Env" "Staging")]] (is (= [(op/create-replica "production" "temp-staging") (op/promote "temp-staging") @@ -78,7 +80,7 @@ (op/rename "temp-staging" "staging") (op/delete "old-staging-replica") (op/delete "old-staging")] - (plan/replace-tree instances "production" "staging"))) + (plan/replace-tree instances "production" snapshot-id "staging"))) (is (= [(op/create-replica "production" "temp-staging") (op/promote "temp-staging") @@ -100,7 +102,7 @@ :PreferredMaintenanceWindow "tue:02:00-tue:03:00"} {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" :PreferredMaintenanceWindow "tue:03:00-tue:04:00"}] - "production" "staging"))) + "production" snapshot-id "staging"))) (is (= [(op/create-replica "production" "temp-staging" {:Tags tags}) (op/promote "temp-staging") @@ -114,7 +116,7 @@ (op/shell-command "./restart.sh") (op/delete "old-staging-replica") (op/delete "old-staging")] - (plan/replace-tree instances "production" "staging" + (plan/replace-tree instances "production" snapshot-id "staging" :restart "./restart.sh" :tags {"staging" tags "staging-replica" tags}))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 819a745..ef4e868 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -95,4 +95,4 @@ (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} (sut/from-plan (example-instances) - (plan/replace-tree (example-instances) "production" "staging"))))) + (plan/replace-tree (example-instances) "production" "snapshot-id" "staging"))))) From 0d8bc85763b55543f33c481ab0199484e60d26d1 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 20:13:07 -0300 Subject: [PATCH 14/44] We don't need :pre here I explained why in "Add missing predict method for restore" but went on dismissing all I said there. --- src/stack_mitosis/predict.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index bb0961b..fe6034f 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -79,8 +79,7 @@ (defmethod predict :RestoreDBInstanceFromDBSnapshot [instances op] - {:pre [(->> op :meta :SourceDBInstance :DBInstanceIdentifier (lookup/exists? instances))] - :post [(lookup/exists? % (r/db-id op))]} + {:post [(lookup/exists? % (r/db-id op))]} (let [source-id (->> op :meta :SourceDBInstance :DBInstanceIdentifier) source-db (lookup/by-id instances source-id)] (->> op From 01b27d64b8047dba991b992a755e34f62003c30d Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 20:43:06 -0300 Subject: [PATCH 15/44] Test restore-snapshot path in clone-tree --- test/stack_mitosis/planner_test.clj | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index fb4a87b..fe17dc3 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -1,7 +1,8 @@ (ns stack-mitosis.planner-test (:require [clojure.test :refer :all] [stack-mitosis.planner :as plan] - [stack-mitosis.operations :as op])) + [stack-mitosis.operations :as op] + [stack-mitosis.lookup :as lookup])) (deftest list-tree (let [a {:DBInstanceIdentifier :a :ReadReplicaDBInstanceIdentifiers [:b]} @@ -21,6 +22,12 @@ :ReadReplicaSourceDBInstanceIdentifier "target"} {:DBInstanceIdentifier "b" :ReadReplicaSourceDBInstanceIdentifier "target"} {:DBInstanceIdentifier "c" :ReadReplicaSourceDBInstanceIdentifier "b"}] + instances-different-vpc [{:DBInstanceIdentifier "source" :Iops 1000, :DBSubnetGroup {:VpcId "v1"}} + {:DBInstanceIdentifier "target" :ReadReplicaDBInstanceIdentifiers ["a" "b"] :Iops 500} + {:DBInstanceIdentifier "a" :ReadReplicaDBInstanceIdentifiers ["c"] + :ReadReplicaSourceDBInstanceIdentifier "target"} + {:DBInstanceIdentifier "b" :ReadReplicaSourceDBInstanceIdentifier "target"} + {:DBInstanceIdentifier "c" :ReadReplicaSourceDBInstanceIdentifier "b"}] snapshot-id "rds:source-snapshot-2021-03-17" tags-target [(op/kv "k" "target")] tags-b [(op/kv "k" "b")]] @@ -34,6 +41,20 @@ (op/create-replica "temp-b" "temp-c") (op/modify "temp-c" {})] (plan/copy-tree instances "source" snapshot-id "target" + (partial plan/aliased "temp") + :tags {"target" tags-target + "b" tags-b}))) + (is (= [(op/restore-snapshot snapshot-id + (lookup/by-id instances-different-vpc "source") + "temp-target" {:Iops 500 :Tags tags-target}) + (op/enable-backups "temp-target") + (op/create-replica "temp-target" "temp-a") + (op/modify "temp-a" {:BackupRetentionPeriod 1}) + (op/create-replica "temp-target" "temp-b" {:Tags tags-b}) + (op/modify "temp-b" {}) + (op/create-replica "temp-b" "temp-c") + (op/modify "temp-c" {})] + (plan/copy-tree instances-different-vpc "source" snapshot-id "target" (partial plan/aliased "temp") :tags {"target" tags-target "b" tags-b}))))) From 1032891561068f7ecfb50e91dead1b168ed1b91a Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 20:43:36 -0300 Subject: [PATCH 16/44] Test restore-snapshot path in replace-tree --- test/stack_mitosis/planner_test.clj | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index fe17dc3..402f173 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -125,6 +125,29 @@ :PreferredMaintenanceWindow "tue:03:00-tue:04:00"}] "production" snapshot-id "staging"))) + (is (= [(op/restore-snapshot snapshot-id {:DBInstanceIdentifier "production" + :PreferredMaintenanceWindow "tue:01:00-tue:02:00" + :DBSubnetGroup {:VpcId "v1"}} "temp-staging") + (op/enable-backups "temp-staging" + {:PreferredMaintenanceWindow "tue:02:00-tue:03:00" :MonitoringInterval 60}) + (op/create-replica "temp-staging" "temp-staging-replica") + (op/modify "temp-staging-replica" + {:PreferredMaintenanceWindow "tue:03:00-tue:04:00"}) + (op/rename "staging-replica" "old-staging-replica") + (op/rename "staging" "old-staging") + (op/rename "temp-staging-replica" "staging-replica") + (op/rename "temp-staging" "staging") + (op/delete "old-staging-replica") + (op/delete "old-staging")] + (plan/replace-tree + [{:DBInstanceIdentifier "production" + :PreferredMaintenanceWindow "tue:01:00-tue:02:00" :DBSubnetGroup {:VpcId "v1"}} + {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] + :PreferredMaintenanceWindow "tue:02:00-tue:03:00" :MonitoringInterval 60} + {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" + :PreferredMaintenanceWindow "tue:03:00-tue:04:00"}] + "production" snapshot-id "staging"))) + (is (= [(op/create-replica "production" "temp-staging" {:Tags tags}) (op/promote "temp-staging") (op/enable-backups "temp-staging" {}) From 920a2500e5e80dd939824ea293fac5f3bf0c0785 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Wed, 17 Mar 2021 20:43:58 -0300 Subject: [PATCH 17/44] Test skipping restore-snapshot --- test/stack_mitosis/planner_test.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index 402f173..e95f8a8 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -174,6 +174,8 @@ (plan/attempt instances (op/create {:DBInstanceIdentifier "a"})))) (is (= [:skip (plan/duplicate-instance "b")] (plan/attempt instances (op/create-replica "a" "b")))) + (is (= [:skip (plan/duplicate-instance "b")] + (plan/attempt instances (op/restore-snapshot "c" "a" "b")))) (is (= [:skip (plan/promoted-instance "a")] (plan/attempt instances (op/promote "a")))) From c22c2ae2275d3c2d6945de6519d5e09c20a41b0c Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 18 Mar 2021 15:45:27 -0300 Subject: [PATCH 18/44] Add policy generation for restore-snapshot Add the arn to the planner test too now that we require it --- src/stack_mitosis/policy.clj | 18 ++++++++++++++++++ src/stack_mitosis/predict.clj | 23 +++++++++++++++++------ test/stack_mitosis/planner_test.clj | 3 ++- test/stack_mitosis/policy_test.clj | 9 +++++++++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 56d1fdf..81b07bb 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -36,6 +36,24 @@ [source-arn target-arn])} {:op :AddTagsToResource :arn target-arn}])) +(defmethod permissions :RestoreDBInstanceFromDBSnapshot + [instances action] + ;; For create replica, use the ARN from the source database + (let [snapshot-id (->> action :request :DBSnapshotIdentifier) + source-arn (->> action dbg :meta :SourceDBInstance :DBInstanceArn) + snapshot-arn (-> source-arn (str/split #":db:") first (str ":snapshot:" snapshot-id)) + db-id (dbg (r/db-id action)) + target-arn (:DBInstanceArn (lookup/by-id (predict/predict instances action) db-id))] + [{:op (:op action) + ;; TODO: can these permissions be more specific instead of wildcard? + ;; a) re-use the base ARN (ie region:account-id) from source + ;; b) generate named ARNs for each subtype used in source? + :arn (into [(make-arn "*" :type "og") + (make-arn "*" :type "pg") + (make-arn "*" :type "subgrp")] + [snapshot-arn target-arn])} + {:op :AddTagsToResource :arn target-arn}])) + (defmethod permissions :ModifyDBInstance [instances action] (let [db-id (r/db-id action) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index fe6034f..5a6df14 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -81,12 +81,23 @@ [instances op] {:post [(lookup/exists? % (r/db-id op))]} (let [source-id (->> op :meta :SourceDBInstance :DBInstanceIdentifier) - source-db (lookup/by-id instances source-id)] - (->> op - (:request) - (#(dissoc %2 %1) :DBSnapshotIdentifier) - (merge source-db) - (conj instances)))) + source-db (lookup/by-id instances source-id) + target-id (r/db-id op) + ;; We use the arn for some policy calculations, and we don't want it to + ;; end up being the source-arn + target-arn (-> source-db :DBInstanceArn (str/split #":db:") first (str ":db:" target-id))] + (-> op + :request + (dissoc :DBSnapshotIdentifier) + ;; This is not reeeally what happens, but replicating what happens is + ;; complicated. We're getting a DB in the new subnet, missing a few + ;; params, that are not going to be === the source params, but carry + ;; default values from RDS. + ;; + ;; Also, invert the param order so our params override source-db's + (#(merge %2 %1) source-db) + (assoc :DBInstanceArn target-arn) + (#(conj %2 %1) instances)))) (defmethod predict :ModifyDBInstance [instances op] diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index e95f8a8..72ed7be 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -126,6 +126,7 @@ "production" snapshot-id "staging"))) (is (= [(op/restore-snapshot snapshot-id {:DBInstanceIdentifier "production" + :DBInstanceArn "arn:db:production" :PreferredMaintenanceWindow "tue:01:00-tue:02:00" :DBSubnetGroup {:VpcId "v1"}} "temp-staging") (op/enable-backups "temp-staging" @@ -140,7 +141,7 @@ (op/delete "old-staging-replica") (op/delete "old-staging")] (plan/replace-tree - [{:DBInstanceIdentifier "production" + [{:DBInstanceIdentifier "production" :DBInstanceArn "arn:db:production" :PreferredMaintenanceWindow "tue:01:00-tue:02:00" :DBSubnetGroup {:VpcId "v1"}} {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] :PreferredMaintenanceWindow "tue:02:00-tue:03:00" :MonitoringInterval 60} diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index ef4e868..ebc90ce 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -47,6 +47,15 @@ {:op :AddTagsToResource :arn "arn:aws:rds:us-east-1:1234567:db:bar"}] (sut/permissions [instance] (op/create-replica "foo" "bar")))) + (is (= [{:op :RestoreDBInstanceFromDBSnapshot + :arn ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*" + "arn:aws:rds:us-east-1:1234567:snapshot:foo" + "arn:aws:rds:us-east-1:1234567:db:bar"]} + {:op :AddTagsToResource + :arn "arn:aws:rds:us-east-1:1234567:db:bar"}] + (sut/permissions [instance] (op/restore-snapshot "foo" instance "bar")))) (is (= [{:op :ModifyDBInstance :arn ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" From 369c7275ad80c1b39e9efbd26603d072d1e56655 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 18 Mar 2021 18:17:15 -0300 Subject: [PATCH 19/44] Verify a snapshot exists before proceeding and give a clearer error --- src/stack_mitosis/cli.clj | 8 +++++--- src/stack_mitosis/interpreter.clj | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index 7c43587..ae5fea7 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -50,10 +50,12 @@ (log/infof "Assuming role %s" (:role-arn role)) (sudo/sudo-provider role))) (let [rds (interpreter/client) - instances (interpreter/databases rds)] - (when (interpreter/verify-databases-exist instances [source target]) + instances (interpreter/databases rds) + source-snapshot (interpreter/latest-snapshot rds source)] + (when (and (interpreter/verify-databases-exist instances [source target]) + (interpreter/verify-snapshot-exists instances [source target] + source-snapshot)) (let [tags (interpreter/list-tags rds instances target) - source-snapshot (interpreter/latest-snapshot rds source) plan (plan/replace-tree instances source source-snapshot target :restart restart :tags tags)] (cond (:plan options) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index 1173afa..c7ac738 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -51,6 +51,22 @@ false) true))) +(defn verify-snapshot-exists + [instances identifiers snapshot] + (let [instances (map (partial lookup/by-id instances) identifiers) + vpcs (map #(->> % :DBSubnetGroup :VpcId) instances) + cross-vpc-mitosis (-> vpcs distinct count (> 1))] + (if (and cross-vpc-mitosis (not snapshot)) + (do + (log/error + (str/join "\n" ["Source database has no snapshots." "" + (str "Source and target databases are in different VPCs." + " When that happens, stack-mitosis uses " + "RestoreDBInstanceFromDBSnapshot to be able to" + " clone the source database to the target VPC.")])) + false) + true))) + (defn list-tags "Mapping of db-id to tags list for each instance in a tree." [rds instances target] From 18cf315b96f6992b183537f1ec53e6f5621888f6 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 18 Mar 2021 18:17:58 -0300 Subject: [PATCH 20/44] clj-kondo --- src/stack_mitosis/interpreter.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index c7ac738..0f57f33 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -143,7 +143,7 @@ (sudo/sudo-provider (sudo/load-role "resources/role.edn")) (def rds (client)) (-> (predict/state [] (example/create example/template)) - (plan/replace-tree "mitosis-prod" "mitosis-demo")) + (plan/replace-tree "mitosis-prod" "mitosis-demo" nil)) (interpret rds (op/shell-command "echo restart")) (evaluate-plan rds [(op/shell-command "true") (op/shell-command "false") @@ -151,7 +151,7 @@ ;; check plan (let [state (databases rds)] - (check-plan state (plan/replace-tree state "mitosis-prod" "mitosis-demo"))) + (check-plan state (plan/replace-tree state "mitosis-prod" "mitosis-demo" nil))) ;; create a copy of mitosis-prod tree (let [state (databases rds)] From 5227dc4821b84ea2d5560a7b90bc0eb1be5cb415 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 19 Mar 2021 16:36:42 -0300 Subject: [PATCH 21/44] Oops, remove dbg macro use --- src/stack_mitosis/policy.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 81b07bb..09e7bfe 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -40,9 +40,9 @@ [instances action] ;; For create replica, use the ARN from the source database (let [snapshot-id (->> action :request :DBSnapshotIdentifier) - source-arn (->> action dbg :meta :SourceDBInstance :DBInstanceArn) + source-arn (->> action :meta :SourceDBInstance :DBInstanceArn) snapshot-arn (-> source-arn (str/split #":db:") first (str ":snapshot:" snapshot-id)) - db-id (dbg (r/db-id action)) + db-id (r/db-id action) target-arn (:DBInstanceArn (lookup/by-id (predict/predict instances action) db-id))] [{:op (:op action) ;; TODO: can these permissions be more specific instead of wildcard? From 1e455271440e207f88fc3ce1be7c7acc4b8a9078 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 19 Mar 2021 17:32:15 -0300 Subject: [PATCH 22/44] We need global DescribeDBSnapshots to list them --- src/stack_mitosis/policy.clj | 2 +- test/stack_mitosis/policy_test.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 09e7bfe..39d1c5f 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -98,7 +98,7 @@ [(make-arn "mitosis-*")])) (defn globals [] - (allow [:DescribeDBInstances :ListTagsForResource] + (allow [:DescribeDBInstances :ListTagsForResource :DescribeDBSnapshots] [(make-arn "*")])) ;; TODO breakup permissions per operation type with better granularity diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index ebc90ce..f95459d 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -80,7 +80,7 @@ (deftest from-plan (is (= {:Version "2012-10-17" :Statement - [(sut/allow [:DescribeDBInstances :ListTagsForResource] + [(sut/allow [:DescribeDBInstances :ListTagsForResource :DescribeDBSnapshots] ["arn:aws:rds:*:*:db:*"]) (sut/allow [:CreateDBInstanceReadReplica] (into ["arn:aws:rds:*:*:og:*" From 29e5f8c05c3126c27e3e5ba2322c0c291f304d84 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 19 Mar 2021 17:32:52 -0300 Subject: [PATCH 23/44] We can't make it specific to one snapshot bc they're dynamically named We're always getting the latest and it's going to have the date in the name. --- src/stack_mitosis/policy.clj | 5 ++--- test/stack_mitosis/policy_test.clj | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 39d1c5f..97b2f46 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -39,9 +39,8 @@ (defmethod permissions :RestoreDBInstanceFromDBSnapshot [instances action] ;; For create replica, use the ARN from the source database - (let [snapshot-id (->> action :request :DBSnapshotIdentifier) - source-arn (->> action :meta :SourceDBInstance :DBInstanceArn) - snapshot-arn (-> source-arn (str/split #":db:") first (str ":snapshot:" snapshot-id)) + (let [source-arn (->> action :meta :SourceDBInstance :DBInstanceArn) + snapshot-arn (-> source-arn (str/split #":db:") first (str ":snapshot:*")) db-id (r/db-id action) target-arn (:DBInstanceArn (lookup/by-id (predict/predict instances action) db-id))] [{:op (:op action) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index f95459d..c561565 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -51,7 +51,7 @@ :arn ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" "arn:aws:rds:*:*:subgrp:*" - "arn:aws:rds:us-east-1:1234567:snapshot:foo" + "arn:aws:rds:us-east-1:1234567:snapshot:*" "arn:aws:rds:us-east-1:1234567:db:bar"]} {:op :AddTagsToResource :arn "arn:aws:rds:us-east-1:1234567:db:bar"}] From 525eb6b3144388ff17f66059e067eaa57f278c2e Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 22 Mar 2021 18:27:40 -0300 Subject: [PATCH 24/44] Only load snapshots when source and target live in different VPCs --- src/stack_mitosis/cli.clj | 12 +++++++++--- src/stack_mitosis/lookup.clj | 3 +++ src/stack_mitosis/planner.clj | 5 +---- src/stack_mitosis/policy.clj | 5 +++-- test/stack_mitosis/policy_test.clj | 5 +++-- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index ae5fea7..33a2e45 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -4,6 +4,7 @@ [clojure.tools.cli :as cli] [clojure.tools.logging :as log] [stack-mitosis.interpreter :as interpreter] + [stack-mitosis.lookup :as lookup] [stack-mitosis.planner :as plan] [stack-mitosis.policy :as policy] [stack-mitosis.request :as r] @@ -51,10 +52,15 @@ (sudo/sudo-provider role))) (let [rds (interpreter/client) instances (interpreter/databases rds) - source-snapshot (interpreter/latest-snapshot rds source)] + same-vpc (lookup/same-vpc? + (lookup/by-id instances source) + (lookup/by-id instances target)) + source-snapshot (when (not same-vpc) + (interpreter/latest-snapshot rds source))] (when (and (interpreter/verify-databases-exist instances [source target]) - (interpreter/verify-snapshot-exists instances [source target] - source-snapshot)) + (or same-vpc + (interpreter/verify-snapshot-exists instances [source target] + source-snapshot))) (let [tags (interpreter/list-tags rds instances target) plan (plan/replace-tree instances source source-snapshot target :restart restart :tags tags)] diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index 1643c17..aee6af1 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -33,6 +33,9 @@ (and (or (seq? v) (vector? v)) (empty? v)))) +(defn same-vpc? [db-a db-b] + (= (-> db-a :DBSubnetGroup :VpcId) (-> db-b :DBSubnetGroup :VpcId))) + (defn restore-snapshot-attributes "Creates a list of additional attributes to clone from original instance into the newly created replica instance. diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index 7e718ea..ecd9b1e 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -42,11 +42,8 @@ root-id (:DBInstanceIdentifier root) ;; can't rely on replication when target and source have different VPCs - target-vpc (->> root :DBSubnetGroup :VpcId) source-instance (lookup/by-id instances source) - source-vpc (->> source-instance :DBSubnetGroup :VpcId) - same-vpc (= source-vpc target-vpc) - + same-vpc (lookup/same-vpc? source-instance root) root-attrs (lookup/clone-replica-attributes root (get alias-tags root-id)) root-restore-attrs (lookup/restore-snapshot-attributes root (get alias-tags root-id))] (into (if same-vpc diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 97b2f46..87037a6 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -51,7 +51,8 @@ (make-arn "*" :type "pg") (make-arn "*" :type "subgrp")] [snapshot-arn target-arn])} - {:op :AddTagsToResource :arn target-arn}])) + {:op :AddTagsToResource :arn target-arn} + {:op :DescribeDBSnapshots :arn "*"}])) (defmethod permissions :ModifyDBInstance [instances action] @@ -97,7 +98,7 @@ [(make-arn "mitosis-*")])) (defn globals [] - (allow [:DescribeDBInstances :ListTagsForResource :DescribeDBSnapshots] + (allow [:DescribeDBInstances :ListTagsForResource] [(make-arn "*")])) ;; TODO breakup permissions per operation type with better granularity diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index c561565..988377d 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -54,7 +54,8 @@ "arn:aws:rds:us-east-1:1234567:snapshot:*" "arn:aws:rds:us-east-1:1234567:db:bar"]} {:op :AddTagsToResource - :arn "arn:aws:rds:us-east-1:1234567:db:bar"}] + :arn "arn:aws:rds:us-east-1:1234567:db:bar"} + {:op :DescribeDBSnapshots :arn "*"}] (sut/permissions [instance] (op/restore-snapshot "foo" instance "bar")))) (is (= [{:op :ModifyDBInstance :arn ["arn:aws:rds:*:*:og:*" @@ -80,7 +81,7 @@ (deftest from-plan (is (= {:Version "2012-10-17" :Statement - [(sut/allow [:DescribeDBInstances :ListTagsForResource :DescribeDBSnapshots] + [(sut/allow [:DescribeDBInstances :ListTagsForResource] ["arn:aws:rds:*:*:db:*"]) (sut/allow [:CreateDBInstanceReadReplica] (into ["arn:aws:rds:*:*:og:*" From 453b5b5bb0e9ffe4f1e6322c8df6c284a4c8a03d Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 22 Mar 2021 18:37:15 -0300 Subject: [PATCH 25/44] Make logs go to STDERR so we can pipe iam-policy output to other tools --- resources/log4j.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/log4j.properties b/resources/log4j.properties index a2f6e8f..e8cdd4c 100644 --- a/resources/log4j.properties +++ b/resources/log4j.properties @@ -2,3 +2,4 @@ log4j.rootLogger=DEBUG, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} | %-5p | %t | %m%n +log4j.appender.console.Target=System.err From b01c0f4d562a1d5fb1c23743be5ad7c99fe8f654 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 22 Mar 2021 18:44:18 -0300 Subject: [PATCH 26/44] Use as-> instead of #(fn %2 %1) to make pipeline args placement clearer --- src/stack_mitosis/predict.clj | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index 5a6df14..02d74a2 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -86,18 +86,16 @@ ;; We use the arn for some policy calculations, and we don't want it to ;; end up being the source-arn target-arn (-> source-db :DBInstanceArn (str/split #":db:") first (str ":db:" target-id))] - (-> op - :request - (dissoc :DBSnapshotIdentifier) - ;; This is not reeeally what happens, but replicating what happens is - ;; complicated. We're getting a DB in the new subnet, missing a few - ;; params, that are not going to be === the source params, but carry - ;; default values from RDS. - ;; - ;; Also, invert the param order so our params override source-db's - (#(merge %2 %1) source-db) - (assoc :DBInstanceArn target-arn) - (#(conj %2 %1) instances)))) + (as-> op $ + ($ :request) + (dissoc $ :DBSnapshotIdentifier) + ;; This is not reeeally what happens, but replicating what happens is + ;; complicated. We're getting a DB in the new subnet, missing a few + ;; params, that are not going to be === the source params, but carry + ;; default values from RDS. + (merge source-db $) + (assoc $ :DBInstanceArn target-arn) + (conj instances $)))) (defmethod predict :ModifyDBInstance [instances op] From 3c038b34e965f9662b6e5bd3f93b5c405d9b5db8 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 22 Mar 2021 18:47:08 -0300 Subject: [PATCH 27/44] Oops, wrong argument order --- src/stack_mitosis/predict.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index 02d74a2..158b99d 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -87,7 +87,7 @@ ;; end up being the source-arn target-arn (-> source-db :DBInstanceArn (str/split #":db:") first (str ":db:" target-id))] (as-> op $ - ($ :request) + (:request $) (dissoc $ :DBSnapshotIdentifier) ;; This is not reeeally what happens, but replicating what happens is ;; complicated. We're getting a DB in the new subnet, missing a few From 464184694c1eec9481880f8fb7d6e9f8a889746d Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 22 Mar 2021 18:55:25 -0300 Subject: [PATCH 28/44] Use get-in instead of piping to get nested data --- src/stack_mitosis/interpreter.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index 0f57f33..bc506db 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -54,7 +54,7 @@ (defn verify-snapshot-exists [instances identifiers snapshot] (let [instances (map (partial lookup/by-id instances) identifiers) - vpcs (map #(->> % :DBSubnetGroup :VpcId) instances) + vpcs (map #(get-in % [:DBSubnetGroup :VpcId]) instances) cross-vpc-mitosis (-> vpcs distinct count (> 1))] (if (and cross-vpc-mitosis (not snapshot)) (do From f8c028239dc5ec9037ec18281f4b83715c8acaa9 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Mon, 22 Mar 2021 19:16:12 -0300 Subject: [PATCH 29/44] Use get-in for same-vpc? too, and test it --- src/stack_mitosis/lookup.clj | 2 +- test/stack_mitosis/lookup_test.clj | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index aee6af1..32a932a 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -34,7 +34,7 @@ (empty? v)))) (defn same-vpc? [db-a db-b] - (= (-> db-a :DBSubnetGroup :VpcId) (-> db-b :DBSubnetGroup :VpcId))) + (= (get-in db-a [:DBSubnetGroup :VpcId]) (get-in db-b [:DBSubnetGroup :VpcId]))) (defn restore-snapshot-attributes "Creates a list of additional attributes to clone from original instance into diff --git a/test/stack_mitosis/lookup_test.clj b/test/stack_mitosis/lookup_test.clj index 3be4a14..6a9466a 100644 --- a/test/stack_mitosis/lookup_test.clj +++ b/test/stack_mitosis/lookup_test.clj @@ -55,3 +55,7 @@ ;; :DBSubnetGroupName "subnet-group" } (lookup/post-create-replica-attributes instance))))) + +(deftest same-vpc? + (is (= true (lookup/same-vpc? {:DBSubnetGroup {:VpcId "a"}} {:DBSubnetGroup {:VpcId "a"}}))) + (is (= false (lookup/same-vpc? {:DBSubnetGroup {:VpcId "a"}} {:DBSubnetGroup {:VpcId "b"}})))) \ No newline at end of file From 7f92aa521b9e5e43445eb30adf96a3aaa341a76e Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 15:53:38 -0300 Subject: [PATCH 30/44] Keep the target's DBSubnetGroupName! Turns out it's impossible to change a subnet group within the same VPC so let's ensure we keep the target's one when cloning within the same VPC. --- src/stack_mitosis/lookup.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index 32a932a..57bb29d 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -121,6 +121,7 @@ :EnablePerformanceInsights (:PerformanceInsightsEnabled original) :EnableIAMDatabaseAuthentication (:IAMDatabaseAuthenticationEnabled original) :EnableCloudwatchLogsExports (:EnabledCloudwatchLogsExports original) + :DBSubnetGroupName (:DBSubnetGroupName (:DBSubnetGroup original)) :Port (:Port (:Endpoint original)) ;; all active security groups ids From a0e09546c67a324bb8fa8589f4b80a25799bf9bd Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 16:36:17 -0300 Subject: [PATCH 31/44] Can't create a replica in a different subnet group -.- --- src/stack_mitosis/lookup.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/stack_mitosis/lookup.clj b/src/stack_mitosis/lookup.clj index 57bb29d..32a932a 100644 --- a/src/stack_mitosis/lookup.clj +++ b/src/stack_mitosis/lookup.clj @@ -121,7 +121,6 @@ :EnablePerformanceInsights (:PerformanceInsightsEnabled original) :EnableIAMDatabaseAuthentication (:IAMDatabaseAuthenticationEnabled original) :EnableCloudwatchLogsExports (:EnabledCloudwatchLogsExports original) - :DBSubnetGroupName (:DBSubnetGroupName (:DBSubnetGroup original)) :Port (:Port (:Endpoint original)) ;; all active security groups ids From be0f6da3fe666020aa56cb53193790159c4615b3 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 17:10:38 -0300 Subject: [PATCH 32/44] Move comment about promoting postgres to where promote is done --- src/stack_mitosis/planner.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index ecd9b1e..27d0bcb 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -28,8 +28,6 @@ (zipmap ids) topological-sort)) -;; postgres does not allow replica of replica, so need to promote before -;; replicating children (defn copy-tree [instances source source-snapshot target alias-fn & {:keys [tags]}] (let [alias-tags @@ -48,6 +46,8 @@ root-restore-attrs (lookup/restore-snapshot-attributes root (get alias-tags root-id))] (into (if same-vpc [(op/create-replica source root-id root-attrs) + ;; postgres does not allow replica of replica, so need to promote before + ;; replicating children (op/promote root-id) ;; postgres only allows backups after promotion (op/enable-backups root-id (lookup/post-create-replica-attributes root))] From e553e2c8543d8f8c7c196bb5fd0d870ee046f3ac Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 17:10:48 -0300 Subject: [PATCH 33/44] Stray comment from same-vpc --- src/stack_mitosis/planner.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index 27d0bcb..42e5c9a 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -39,7 +39,6 @@ (list-tree instances target)) root-id (:DBInstanceIdentifier root) - ;; can't rely on replication when target and source have different VPCs source-instance (lookup/by-id instances source) same-vpc (lookup/same-vpc? source-instance root) root-attrs (lookup/clone-replica-attributes root (get alias-tags root-id)) From d211e699dd6b069c464141ee06461182747d337a Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 18:24:02 -0300 Subject: [PATCH 34/44] Fix test to use snapshot :none --- test/stack_mitosis/planner_test.clj | 8 +++---- test/stack_mitosis/policy_test.clj | 37 ++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index 72ed7be..e683dd0 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -40,7 +40,7 @@ (op/modify "temp-b" {}) (op/create-replica "temp-b" "temp-c") (op/modify "temp-c" {})] - (plan/copy-tree instances "source" snapshot-id "target" + (plan/copy-tree instances "source" :none "target" (partial plan/aliased "temp") :tags {"target" tags-target "b" tags-b}))) @@ -101,7 +101,7 @@ (op/rename "temp-staging" "staging") (op/delete "old-staging-replica") (op/delete "old-staging")] - (plan/replace-tree instances "production" snapshot-id "staging"))) + (plan/replace-tree instances "production" :none "staging"))) (is (= [(op/create-replica "production" "temp-staging") (op/promote "temp-staging") @@ -123,7 +123,7 @@ :PreferredMaintenanceWindow "tue:02:00-tue:03:00"} {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" :PreferredMaintenanceWindow "tue:03:00-tue:04:00"}] - "production" snapshot-id "staging"))) + "production" :none "staging"))) (is (= [(op/restore-snapshot snapshot-id {:DBInstanceIdentifier "production" :DBInstanceArn "arn:db:production" @@ -161,7 +161,7 @@ (op/shell-command "./restart.sh") (op/delete "old-staging-replica") (op/delete "old-staging")] - (plan/replace-tree instances "production" snapshot-id "staging" + (plan/replace-tree instances "production" :none "staging" :restart "./restart.sh" :tags {"staging" tags "staging-replica" tags}))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 988377d..723a90f 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -7,6 +7,9 @@ (defn fake-arn [name] (str "arn:aws:rds:us-east-1:1234567:db:" name)) +(defn fake-snapshot-arn [name] + (str "arn:aws:rds:us-east-1:1234567:snapshot:" name)) + (defn example-instances [] [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] :DBInstanceArn (fake-arn "production")} @@ -105,4 +108,36 @@ (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} (sut/from-plan (example-instances) - (plan/replace-tree (example-instances) "production" "snapshot-id" "staging"))))) + (plan/replace-tree (example-instances) "production" :none "staging")))) + (is (= {:Version "2012-10-17" + :Statement + [(sut/allow [:DescribeDBInstances :ListTagsForResource] + ["arn:aws:rds:*:*:db:*"]) + (sut/allow [:RestoreDBInstanceFromDBSnapshot] + ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*" + (fake-snapshot-arn "*") + (fake-arn "temp-staging")]) + (sut/allow [:AddTagsToResource] + (mapv fake-arn ["temp-staging" "temp-staging-replica"])) + (sut/allow [:DescribeDBSnapshots] ["*"]) + (sut/allow [:ModifyDBInstance] + (into ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:secgrp:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica" + "staging-replica" "old-staging-replica" + "staging" "old-staging"]))) + (sut/allow [:RebootDBInstance] + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) + (sut/allow [:CreateDBInstanceReadReplica] + (into ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica"]))) + (sut/allow [:DeleteDBInstance] + (mapv fake-arn ["old-staging-replica" "old-staging"]))]} + (sut/from-plan (example-instances) + (dbg (plan/replace-tree (example-instances) "production" "snapshot-id" "staging")))))) From 3afba0706cdd8380a30702b45f49f80635293b8c Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 18:25:23 -0300 Subject: [PATCH 35/44] Fix bug that creates extra modify after restore when source has replicas --- src/stack_mitosis/predict.clj | 1 + test/stack_mitosis/planner_test.clj | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index 158b99d..22fbf15 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -95,6 +95,7 @@ ;; default values from RDS. (merge source-db $) (assoc $ :DBInstanceArn target-arn) + (dissoc $ :ReadReplicaDBInstanceIdentifiers) (conj instances $)))) (defmethod predict :ModifyDBInstance diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index e683dd0..ea2c7de 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -117,7 +117,9 @@ (op/delete "old-staging-replica") (op/delete "old-staging")] (plan/replace-tree - [{:DBInstanceIdentifier "production" + [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] + :PreferredMaintenanceWindow "tue:01:00-tue:02:00"} + {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" :PreferredMaintenanceWindow "tue:01:00-tue:02:00"} {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] :PreferredMaintenanceWindow "tue:02:00-tue:03:00"} @@ -128,6 +130,7 @@ (is (= [(op/restore-snapshot snapshot-id {:DBInstanceIdentifier "production" :DBInstanceArn "arn:db:production" :PreferredMaintenanceWindow "tue:01:00-tue:02:00" + :ReadReplicaDBInstanceIdentifiers ["production-replica"] :DBSubnetGroup {:VpcId "v1"}} "temp-staging") (op/enable-backups "temp-staging" {:PreferredMaintenanceWindow "tue:02:00-tue:03:00" :MonitoringInterval 60}) @@ -142,6 +145,9 @@ (op/delete "old-staging")] (plan/replace-tree [{:DBInstanceIdentifier "production" :DBInstanceArn "arn:db:production" + :ReadReplicaDBInstanceIdentifiers ["production-replica"] + :PreferredMaintenanceWindow "tue:01:00-tue:02:00" :DBSubnetGroup {:VpcId "v1"}} + {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" :PreferredMaintenanceWindow "tue:01:00-tue:02:00" :DBSubnetGroup {:VpcId "v1"}} {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] :PreferredMaintenanceWindow "tue:02:00-tue:03:00" :MonitoringInterval 60} From bbe13bfd6bf7a15855cf3e3791e0ae7cff88fd6f Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 17:02:44 -0300 Subject: [PATCH 36/44] Create an option to always use snapshot restore We need snapshot restore when DBs in the same VPC have differing DBSubnetGroupName --- src/stack_mitosis/cli.clj | 12 +++++++----- src/stack_mitosis/planner.clj | 3 +-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index 33a2e45..e24374a 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -21,6 +21,7 @@ ["-c" "--credentials FILENAME" "Credentials file in edn for iam assume-role"] ["-p" "--plan" "Display expected flightplan for operation."] ["-i" "--iam-policy" "Generate IAM policy for planned actions."] + [nil "--restore-snapshot" "Always clone using snapshot restore."] ["-h" "--help"]]) (defn parse-args [args] @@ -55,10 +56,12 @@ same-vpc (lookup/same-vpc? (lookup/by-id instances source) (lookup/by-id instances target)) - source-snapshot (when (not same-vpc) - (interpreter/latest-snapshot rds source))] + use-restore-snapshot (or (:restore-snapshot options) (not same-vpc)) + source-snapshot (if use-restore-snapshot + (interpreter/latest-snapshot rds source) + :none)] (when (and (interpreter/verify-databases-exist instances [source target]) - (or same-vpc + (or (not use-restore-snapshot) (interpreter/verify-snapshot-exists instances [source target] source-snapshot))) (let [tags (interpreter/list-tags rds instances target) @@ -79,8 +82,7 @@ (when exit-msg (println exit-msg) (System/exit (if ok 0 1))) - (System/exit (if (process options) 0 1)) - )) + (System/exit (if (process options) 0 1)))) (comment (process (parse-args ["--source" "mitosis-prod" "--target" "mitosis-demo" diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index 42e5c9a..f64a97b 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -40,10 +40,9 @@ root-id (:DBInstanceIdentifier root) source-instance (lookup/by-id instances source) - same-vpc (lookup/same-vpc? source-instance root) root-attrs (lookup/clone-replica-attributes root (get alias-tags root-id)) root-restore-attrs (lookup/restore-snapshot-attributes root (get alias-tags root-id))] - (into (if same-vpc + (into (if (= source-snapshot :none) [(op/create-replica source root-id root-attrs) ;; postgres does not allow replica of replica, so need to promote before ;; replicating children From ce779bb1037d4d711b098de3bfdc6eeb9cf52d9a Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 25 Mar 2021 18:47:11 -0300 Subject: [PATCH 37/44] Guard the same-vpc check by ensuring source and target exist first --- src/stack_mitosis/cli.clj | 49 ++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index e24374a..f802da8 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -52,30 +52,31 @@ (log/infof "Assuming role %s" (:role-arn role)) (sudo/sudo-provider role))) (let [rds (interpreter/client) - instances (interpreter/databases rds) - same-vpc (lookup/same-vpc? - (lookup/by-id instances source) - (lookup/by-id instances target)) - use-restore-snapshot (or (:restore-snapshot options) (not same-vpc)) - source-snapshot (if use-restore-snapshot - (interpreter/latest-snapshot rds source) - :none)] - (when (and (interpreter/verify-databases-exist instances [source target]) - (or (not use-restore-snapshot) - (interpreter/verify-snapshot-exists instances [source target] - source-snapshot))) - (let [tags (interpreter/list-tags rds instances target) - plan (plan/replace-tree instances source source-snapshot target - :restart restart :tags tags)] - (cond (:plan options) - (do (println (flight-plan (interpreter/check-plan instances plan))) - true) - (:iam-policy options) - (do (json/pprint (policy/from-plan instances plan)) - true) - :else - (let [last-action (interpreter/evaluate-plan rds plan)] - (not (contains? last-action :ErrorResponse)))))))) + instances (interpreter/databases rds)] + (when (interpreter/verify-databases-exist instances [source target]) + (let [same-vpc (lookup/same-vpc? + (lookup/by-id instances source) + (lookup/by-id instances target)) + use-restore-snapshot (or (:restore-snapshot options) (not same-vpc)) + source-snapshot (if use-restore-snapshot + (interpreter/latest-snapshot rds source) + :none)] + + (when (or (not use-restore-snapshot) + (interpreter/verify-snapshot-exists instances [source target] + source-snapshot)) + (let [tags (interpreter/list-tags rds instances target) + plan (plan/replace-tree instances source source-snapshot target + :restart restart :tags tags)] + (cond (:plan options) + (do (println (flight-plan (interpreter/check-plan instances plan))) + true) + (:iam-policy options) + (do (json/pprint (policy/from-plan instances plan)) + true) + :else + (let [last-action (interpreter/evaluate-plan rds plan)] + (not (contains? last-action :ErrorResponse)))))))))) (defn -main [& args] (let [{:keys [ok exit-msg] :as options} (parse-args args)] From b8f10b35bd81b1a7b5ca2c42b945dbf3e4279d9b Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 9 Apr 2021 18:14:43 -0300 Subject: [PATCH 38/44] Terraform test scenarios --- .gitignore | 3 + terraform/README.md | 30 +++++++++ terraform/main.tf | 142 +++++++++++++++++++++++++++++++++++++++++++ terraform/outputs.tf | 11 ++++ 4 files changed, 186 insertions(+) create mode 100644 terraform/README.md create mode 100644 terraform/main.tf create mode 100644 terraform/outputs.tf diff --git a/.gitignore b/.gitignore index 204ed4c..10184f3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ test-results/ /clj/ /.cpcache/ /resources/role.edn +.terraform/ +*.tfstate +*.tfstate.backup diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..2624a3f --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,30 @@ +# Test environment Terraform code + +This terraform code creates a test environment for all invariants of `stack-mitosis`: + +- MySQL + - Same VPC + - Different VPC +- Postgres + - Same VPC + - Different VPC + +## How to use + +``` +$ terraform apply +``` + +Wait some 15 minutes. + +See the output: + +``` + +``` + +Run stack-mitosis against all pairs of source and target instances: + +``` + +``` diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..7d30f06 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,142 @@ +terraform { + required_version = "~> 0.13.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } +} + +locals { + source = { + port = 5430 + backup_retention_period = 1 + backup_window = "05:30-06:20" + maintenance_window = "tue:07:02-tue:08:00" + } + target = { + port = 5431 + backup_retention_period = 1 + backup_window = "05:20-06:20" + maintenance_window = "mon:07:30-mon:08:00" + } + + mysql = { engine = "mysql" } + postgres = { engine = "postgres" } + vpc_a = { db_subnet_group_name = null } + vpc_b = { db_subnet_group_name = aws_db_subnet_group.main.name } + + sources = { + mitosis-mysql-src = merge(local.source, local.mysql, local.vpc_a), + mitosis-postgres-src = merge(local.source, local.postgres, local.vpc_a), + } + targets = { + mitosis-mysql-target-same-vpc = merge(local.target, local.mysql, local.vpc_a), + mitosis-mysql-target-different-vpc = merge(local.target, local.mysql, local.vpc_b), + mitosis-postgres-target-same-vpc = merge(local.target, local.postgres, local.vpc_a), + mitosis-postgres-target-different-vpc = merge(local.target, local.postgres, local.vpc_b), + } + dbs = merge(local.sources, local.targets) +} + +# Configure the AWS Provider +provider "aws" { + # Or whatever region you want + region = "us-west-2" +} + +# Current region +data "aws_region" "current" {} + +# Create a VPC +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" + tags = { + Name = "stack-mitosis" + Service = "Mitosis" + Env = "test" + } +} + +# Create two subnets so we can create a subnet group +resource "aws_subnet" "a" { + vpc_id = aws_vpc.main.id + cidr_block = "10.0.0.0/24" + availability_zone = "${data.aws_region.current.name}a" + + tags = { + Name = "Mitosis A" + Service = "Mitosis" + Env = "test" + } +} + +resource "aws_subnet" "b" { + vpc_id = aws_vpc.main.id + cidr_block = "10.0.1.0/24" + availability_zone = "${data.aws_region.current.name}b" + + tags = { + Name = "Mitosis B" + Service = "Mitosis" + Env = "test" + } +} + +# Subnet group for our databases +resource "aws_db_subnet_group" "main" { + name = "mitosis-test" + subnet_ids = [aws_subnet.a.id, aws_subnet.b.id] + + tags = { + Name = "Mitosis" + Service = "Mitosis" + Env = "test" + } +} + +# Random password generator for the required field `password` on the DBs +resource "random_password" "password" { + length = 16 + special = false +} + +# Create our databases +resource "aws_db_instance" "main" { + for_each = local.dbs + allocated_storage = 10 + instance_class = "db.t3.micro" + username = "foo" + skip_final_snapshot = true + + password = random_password.password.result + + identifier = each.key + engine = each.value.engine + db_subnet_group_name = each.value.db_subnet_group_name + backup_retention_period = each.value.backup_retention_period + backup_window = each.value.backup_window + maintenance_window = each.value.maintenance_window + port = each.value.port + + tags = { + Service = "Mitosis" + Env = "test" + } +} + +# And their replicas +resource "aws_db_instance" "replica" { + for_each = aws_db_instance.main + identifier = "${each.value.id}-replica" + replicate_source_db = each.value.id + engine = each.value.engine + instance_class = "db.t3.micro" + skip_final_snapshot = true + + tags = { + Service = "Mitosis" + Env = "test" + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..8d4db45 --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,11 @@ +locals { + commands = flatten([for src in keys(local.sources): + [for target in keys(local.targets): + "clj -m stack-mitosis.cli --source ${src} --target ${target}" + if aws_db_instance.main[src].engine == aws_db_instance.main[target].engine + ] + ]) +} +output "test_commands" { + value = join("\n",local.commands) +} From 22603ad5a4265f6e6c12be3539d12684913fc107 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 9 Apr 2021 18:15:38 -0300 Subject: [PATCH 39/44] Bump changelog --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0667818..029bb50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,14 @@ ### Added +## [0.6.0] + + - Added the ability to clone with source and target instances in different VPCs, by automatically falling back to `RestoreDBInstanceFromDBSnapshot`, instead of `CreateDBInstanceReadReplica` + - The restore will be done based on the latest snapshot available, which is usually at most 24h stale. + - Added `--restore-snapshot` to force even same-VPC clones to be done with `RestoreDBInstanceFromDBSnapshot` (it's currently faster) + - Added a terrafrom environment for testing + ## [0.5.0] - Added `--iam-policy` option for generating a IAM policy for a user or role to clone a replica with a minimal set of permissions. - Updated dependencies - From 501c37a6664afd30a357b9a61427647f119aeb5c Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 9 Apr 2021 18:16:32 -0300 Subject: [PATCH 40/44] Add restore-snapshot to README.md and mention it under Process --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 6d1246d..18bede2 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ Once that is complete, it's safe to rename the `temp-` prefixed clones back to ` Note that this replication graph is a simple case, but it supports replacing arbitrarily complex replication graphs on RDS and has been verified with mysql and postgres database engines. The postgres engine on RDS only allows multiple replicas of a single primary, but the Mysql engine on RDS allows cascading replicas of replicas. See the AWS documentation for [working with RDS read replicas](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html) for more information on these limitations. +In the case where source and target instances live in different VPCs, or in case `--restore-snapshot` is used, the first instance (`temp-mitosis-demo` here) is created by restoring the latest available snapshot, instead of using replication. This also skips the promote replica step. All other steps remain the same. + # Install After installing a JDK, follow the [clojure install @@ -79,6 +81,7 @@ Hopefully in the future this can be parsed directly from the `AWS_CONFIG` file. [--credentials resources/role.edn] [--plan] [--iam-policy] + [--restore-snapshot] ## Flight Plan From b3db2a33f436cf43433c981ff5e61e5ca3114511 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Fri, 9 Apr 2021 18:17:15 -0300 Subject: [PATCH 41/44] Trailing and leading whitespace --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 18bede2..c09c5f5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ data parity with production in a throw-away environment. ## Process -Suppose for testing or sales purposes it is necessary to maintain an independent application stack with it's own database, which needs to periodically refresh data from the production database. To simplify the illustration, this will focus on the changes to the [AWS RDS](https://aws.amazon.com/rds/) database replication graphs, and omit the application and other services it may depend on. +Suppose for testing or sales purposes it is necessary to maintain an independent application stack with it's own database, which needs to periodically refresh data from the production database. To simplify the illustration, this will focus on the changes to the [AWS RDS](https://aws.amazon.com/rds/) database replication graphs, and omit the application and other services it may depend on. Consider two independent application stacks, production and demo, with primary databases `mitosis-prod` and `mitosis-demo` respectively. Each stack has a replication graph where a primary database is followed by one replica, ie `mitosis-prod` replicates to `mitosis-prod-replica` and `mitosis-demo` replicates to `mitosis-demo-replica`. @@ -34,7 +34,7 @@ Once that is complete, it's safe to rename the `temp-` prefixed clones back to ` ![img](doc/img/rename-2.png) - However, as this is a DNS swap, the application is likely still connected to the original `old-mitosis-demo`. By specifying a restart script, stack-mitosis can force the demo application to restart, and connect to the newly created `mitosis-demo` with fresh data from production. Once it has restarted the application successfully, it deletes the `old-` prefixed database instances from the original demo replication graph. +However, as this is a DNS swap, the application is likely still connected to the original `old-mitosis-demo`. By specifying a restart script, stack-mitosis can force the demo application to restart, and connect to the newly created `mitosis-demo` with fresh data from production. Once it has restarted the application successfully, it deletes the `old-` prefixed database instances from the original demo replication graph. ![img](doc/img/final.png) @@ -85,7 +85,7 @@ Hopefully in the future this can be parsed directly from the `AWS_CONFIG` file. ## Flight Plan -The `--plan` flag will give a flight plan showing the expected list of API calls it's planning on executing against the Amazon API. +The `--plan` flag will give a flight plan showing the expected list of API calls it's planning on executing against the Amazon API. ``` $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --plan @@ -182,7 +182,7 @@ This ensures that a continuous integration or cronjob server like Jenkins can cl Cloudformation and Terraform are wonderful tools focused on declarative architecture transformation from one steady state to another. Stack-mitosis is focused on safely cloning the contents of a database in one environment to another without changing from one steady state to another. As example, for an environment with production and demo environments, they both exist in the correct configuration before running stack-mitosis, and then after running stack-mitosis the configuration remains the same but the demo environment has a fresh copy of the data from production. -I suspect this could also be accomplished using one of these declarative infrastructure tools by transitioning through multiple intervening states, but have not found any examples of anyone doing that. +I suspect this could also be accomplished using one of these declarative infrastructure tools by transitioning through multiple intervening states, but have not found any examples of anyone doing that. # License From d01c63e86f4d04725242e4801225c63bed5cc1a9 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 15 Apr 2021 15:02:38 -0300 Subject: [PATCH 42/44] Stray dbg call --- test/stack_mitosis/policy_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 723a90f..a55a241 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -140,4 +140,4 @@ (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} (sut/from-plan (example-instances) - (dbg (plan/replace-tree (example-instances) "production" "snapshot-id" "staging")))))) + (plan/replace-tree (example-instances) "production" "snapshot-id" "staging"))))) From 0e47cf5f8db06c63af42ec0af5579f5ed3f7c488 Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 15 Apr 2021 15:15:02 -0300 Subject: [PATCH 43/44] Replace :none with nil --- src/stack_mitosis/cli.clj | 2 +- src/stack_mitosis/planner.clj | 2 +- test/stack_mitosis/planner_test.clj | 8 ++++---- test/stack_mitosis/policy_test.clj | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index f802da8..2ebd6e6 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -60,7 +60,7 @@ use-restore-snapshot (or (:restore-snapshot options) (not same-vpc)) source-snapshot (if use-restore-snapshot (interpreter/latest-snapshot rds source) - :none)] + nil)] (when (or (not use-restore-snapshot) (interpreter/verify-snapshot-exists instances [source target] diff --git a/src/stack_mitosis/planner.clj b/src/stack_mitosis/planner.clj index f64a97b..305a4d3 100644 --- a/src/stack_mitosis/planner.clj +++ b/src/stack_mitosis/planner.clj @@ -42,7 +42,7 @@ source-instance (lookup/by-id instances source) root-attrs (lookup/clone-replica-attributes root (get alias-tags root-id)) root-restore-attrs (lookup/restore-snapshot-attributes root (get alias-tags root-id))] - (into (if (= source-snapshot :none) + (into (if (nil? source-snapshot) [(op/create-replica source root-id root-attrs) ;; postgres does not allow replica of replica, so need to promote before ;; replicating children diff --git a/test/stack_mitosis/planner_test.clj b/test/stack_mitosis/planner_test.clj index ea2c7de..09522dc 100644 --- a/test/stack_mitosis/planner_test.clj +++ b/test/stack_mitosis/planner_test.clj @@ -40,7 +40,7 @@ (op/modify "temp-b" {}) (op/create-replica "temp-b" "temp-c") (op/modify "temp-c" {})] - (plan/copy-tree instances "source" :none "target" + (plan/copy-tree instances "source" nil "target" (partial plan/aliased "temp") :tags {"target" tags-target "b" tags-b}))) @@ -101,7 +101,7 @@ (op/rename "temp-staging" "staging") (op/delete "old-staging-replica") (op/delete "old-staging")] - (plan/replace-tree instances "production" :none "staging"))) + (plan/replace-tree instances "production" nil "staging"))) (is (= [(op/create-replica "production" "temp-staging") (op/promote "temp-staging") @@ -125,7 +125,7 @@ :PreferredMaintenanceWindow "tue:02:00-tue:03:00"} {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" :PreferredMaintenanceWindow "tue:03:00-tue:04:00"}] - "production" :none "staging"))) + "production" nil "staging"))) (is (= [(op/restore-snapshot snapshot-id {:DBInstanceIdentifier "production" :DBInstanceArn "arn:db:production" @@ -167,7 +167,7 @@ (op/shell-command "./restart.sh") (op/delete "old-staging-replica") (op/delete "old-staging")] - (plan/replace-tree instances "production" :none "staging" + (plan/replace-tree instances "production" nil "staging" :restart "./restart.sh" :tags {"staging" tags "staging-replica" tags}))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index a55a241..fba49e1 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -108,7 +108,7 @@ (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} (sut/from-plan (example-instances) - (plan/replace-tree (example-instances) "production" :none "staging")))) + (plan/replace-tree (example-instances) "production" nil "staging")))) (is (= {:Version "2012-10-17" :Statement [(sut/allow [:DescribeDBInstances :ListTagsForResource] From 2e48dd64c0b03ab8b3a0a7b9e2565f21626e738a Mon Sep 17 00:00:00 2001 From: Juliano Solanho Date: Thu, 15 Apr 2021 15:29:40 -0300 Subject: [PATCH 44/44] Fix docs with example TF output --- terraform/README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index 2624a3f..c335b22 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -11,20 +11,19 @@ This terraform code creates a test environment for all invariants of `stack-mito ## How to use -``` -$ terraform apply -``` + $ terraform apply Wait some 15 minutes. -See the output: +Terraform output: -``` + Apply complete! Resources: 17 added, 0 changed, 0 destroyed. -``` + Outputs: -Run stack-mitosis against all pairs of source and target instances: + test_commands = clj -m stack-mitosis.cli --source mitosis-mysql-src --target mitosis-mysql-target-different-vpc + clj -m stack-mitosis.cli --source mitosis-mysql-src --target mitosis-mysql-target-same-vpc + clj -m stack-mitosis.cli --source mitosis-postgres-src --target mitosis-postgres-target-different-vpc + clj -m stack-mitosis.cli --source mitosis-postgres-src --target mitosis-postgres-target-same-vpc -``` - -``` +Run the test commands in the terraform output to validate stack-mitosis against all pairs of source and target instances. \ No newline at end of file