From 875fdeb4070fd898293da54cc42bee5f47d5b964 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 18 Sep 2020 20:37:06 -0500 Subject: [PATCH 01/61] Add global describe instances operation --- src/stack_mitosis/interpreter.clj | 2 +- src/stack_mitosis/operations.clj | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index 24d164d..b77438b 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -37,7 +37,7 @@ (defn databases [rds] {:post [(seq %)]} - (:DBInstances (invoke-logged! rds {:op :DescribeDBInstances}))) + (:DBInstances (invoke-logged! rds (op/describe)))) ;; TODO: verify that "old-" database copies do not exist before running (defn verify-databases-exist diff --git a/src/stack_mitosis/operations.clj b/src/stack_mitosis/operations.clj index f1bf91e..e533386 100644 --- a/src/stack_mitosis/operations.clj +++ b/src/stack_mitosis/operations.clj @@ -43,9 +43,11 @@ {:op :PromoteReadReplica :request {:DBInstanceIdentifier id}}) -(defn describe [id] - {:op :DescribeDBInstances - :request {:DBInstanceIdentifier id}}) +(defn describe + ([] {:op :DescribeDBInstances}) + ([id] + {:op :DescribeDBInstances + :request {:DBInstanceIdentifier id}})) (defn tags [db-arn] {:op :ListTagsForResource From 1ca271b142db883eaad8fc02053acb575b9baccb Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 18 Sep 2020 21:02:20 -0500 Subject: [PATCH 02/61] First pass at generating iam-policies from a list of operations --- src/stack_mitosis/example_environment.clj | 5 ++++- src/stack_mitosis/policy.clj | 20 ++++++++++++++++++++ test/stack_mitosis/policy_test.clj | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/stack_mitosis/policy.clj create mode 100644 test/stack_mitosis/policy_test.clj diff --git a/src/stack_mitosis/example_environment.clj b/src/stack_mitosis/example_environment.clj index 417d3fc..4c396e7 100644 --- a/src/stack_mitosis/example_environment.clj +++ b/src/stack_mitosis/example_environment.clj @@ -2,6 +2,7 @@ (:require [stack-mitosis.helpers :as helpers] [stack-mitosis.operations :as op] [stack-mitosis.planner :as plan] + [stack-mitosis.policy :as policy] [stack-mitosis.predict :as predict])) (def template @@ -53,4 +54,6 @@ (concat (plan/delete-tree state "mitosis-demo") (plan/delete-tree state "mitosis-prod")))) - +(comment + (policy/generate [] (create template)) + (policy/generate [] (destroy))) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj new file mode 100644 index 0000000..3a616db --- /dev/null +++ b/src/stack_mitosis/policy.clj @@ -0,0 +1,20 @@ +(ns stack-mitosis.policy + (:require [stack-mitosis.request :as r] + [stack-mitosis.lookup :as lookup])) + +(defn permissions [instances action] + (if-let [db-id (r/db-id action)] + (if-let [instance (lookup/by-id instances db-id)] + {:op (:op action) + :arn (:DBInstanceArn instance)} + {:op (:op action)}) + ;; TODO handle ResourceName for ListTagsForResource + ;; TODO Exclude shell command, and include top level describe? + {:op (:op action)})) + +(defn generate [instances operations] + (let [all-permissions (map (partial permissions instances) operations) + resources (group-by :op all-permissions)] + {:effect "Allow" + :action (keys resources) + :resource (distinct (mapcat :arn resources))})) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj new file mode 100644 index 0000000..161abcc --- /dev/null +++ b/test/stack_mitosis/policy_test.clj @@ -0,0 +1,16 @@ +(ns stack-mitosis.policy-test + (:require [clojure.test :as t :refer [deftest is]] + [stack-mitosis.operations :as op] + [stack-mitosis.policy :as sut])) + +(deftest permissions + (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn "arn:aws:rds:us-east-1"}] + (is (= {:op :DeleteDBInstance} + (sut/permissions [] (op/delete "foo"))) + "only operation if instance is not found") + (is (= {:op :DeleteDBInstance :arn "arn:aws:rds:us-east-1"} + (sut/permissions [instance] (op/delete "foo"))) + "operation and arn if instance is found") + (is (= {:op :DescribeDBInstances} + (sut/permissions [] (op/describe))) + "operation only if no database identifier in request"))) From 6ab56f5ffdfcc8e82d695747098280f5af19e46f Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Sat, 19 Sep 2020 18:43:22 -0500 Subject: [PATCH 03/61] First pass at generating an effect statement from a replace tree operation --- test/stack_mitosis/policy_test.clj | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 161abcc..07371db 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -1,7 +1,8 @@ (ns stack-mitosis.policy-test (:require [clojure.test :as t :refer [deftest is]] [stack-mitosis.operations :as op] - [stack-mitosis.policy :as sut])) + [stack-mitosis.policy :as sut] + [stack-mitosis.planner :as plan])) (deftest permissions (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn "arn:aws:rds:us-east-1"}] @@ -14,3 +15,18 @@ (is (= {:op :DescribeDBInstances} (sut/permissions [] (op/describe))) "operation only if no database identifier in request"))) + +(deftest generate + (let [instances [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] + :DBInstanceArn "production-arn"} + {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" + :DBInstanceArn "production-replica-arn"} + {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] + :DBInstanceArn "staging-arn"} + {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" + :DBInstanceArn "staging-replica-arn"}]] + (is (= {:effect "Allow" + :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] + ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete + :resource []} + (sut/generate instances (plan/replace-tree instances "production" "staging")))))) From 253e102a23a760968f337853ca15dbefe9e06656 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Tue, 29 Dec 2020 19:07:25 -0600 Subject: [PATCH 04/61] Upgrade aws and async dependencies --- deps.edn | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deps.edn b/deps.edn index 134e799..72e66ad 100644 --- a/deps.edn +++ b/deps.edn @@ -1,14 +1,14 @@ {:deps {org.clojure/clojure {:mvn/version "1.10.1"} - org.clojure/core.async {:mvn/version "1.2.603"} - com.cognitect.aws/api {:mvn/version "0.8.456"} - com.cognitect.aws/endpoints {:mvn/version "1.1.11.783"} + org.clojure/core.async {:mvn/version "1.3.610"} + com.cognitect.aws/api {:mvn/version "0.8.484"} + com.cognitect.aws/endpoints {:mvn/version "1.1.11.926"} - com.cognitect.aws/rds {:mvn/version "796.2.662.0"} + com.cognitect.aws/rds {:mvn/version "810.2.817.0"} ;; for STS refresh - com.cognitect.aws/iam {:mvn/version "796.2.654.0"} - com.cognitect.aws/sts {:mvn/version "798.2.678.0"} + com.cognitect.aws/iam {:mvn/version "801.2.704.0"} + com.cognitect.aws/sts {:mvn/version "809.2.784.0"} ;; logging org.clojure/tools.logging {:mvn/version "1.1.0"} From 34bb64369eeed341ced9217e37b87fc6a933c9a3 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Tue, 5 Jan 2021 14:00:12 -0600 Subject: [PATCH 05/61] Upgrade to latest tools-deps image for CI --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3db0bc4..22252e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/clojure:openjdk-11-tools-deps-1.10.1.483 + - image: circleci/clojure:openjdk-11-tools-deps-1.10.1.754 working_directory: ~/repo From 955cd596d4e8267c26518637f13eabc9bccf4347 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 11 Jan 2021 17:12:11 -0600 Subject: [PATCH 06/61] Upgrade aws-api to 0.8.498 --- deps.edn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps.edn b/deps.edn index 72e66ad..a13efb9 100644 --- a/deps.edn +++ b/deps.edn @@ -1,8 +1,8 @@ {:deps {org.clojure/clojure {:mvn/version "1.10.1"} org.clojure/core.async {:mvn/version "1.3.610"} - com.cognitect.aws/api {:mvn/version "0.8.484"} - com.cognitect.aws/endpoints {:mvn/version "1.1.11.926"} + com.cognitect.aws/api {:mvn/version "0.8.498"} + com.cognitect.aws/endpoints {:mvn/version "1.1.11.934"} com.cognitect.aws/rds {:mvn/version "810.2.817.0"} From 44b772eb0d1d4d6c530ec5ccce26027db9846821 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 11 Jan 2021 17:15:25 -0600 Subject: [PATCH 07/61] kaocha version 1.0.732 --- deps.edn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps.edn b/deps.edn index a13efb9..d4da3dd 100644 --- a/deps.edn +++ b/deps.edn @@ -29,9 +29,9 @@ ;; clj -A:kaocha -m kaocha.runner --config-file test/tests.edn :kaocha {:extra-paths ["test"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.0.632"} - lambdaisland/kaocha-junit-xml {:mvn/version "0.0-70"} - lambdaisland/kaocha-cloverage {:mvn/version "1.0-45"}}} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.0.732"} + lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} + lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}}} ;; clj -Aclj-kondo --lint src :clj-kondo From 4ca5ac18544408e015689337b9fdb6ef7e8e4437 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 11 Jan 2021 17:17:16 -0600 Subject: [PATCH 08/61] Remove depot/outdated as a dependency, provided by .clojure/deps.edn --- deps.edn | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deps.edn b/deps.edn index d4da3dd..ca79e03 100644 --- a/deps.edn +++ b/deps.edn @@ -23,11 +23,7 @@ } :paths ["src" "resources"] :aliases - {;; clj -Aoutdated - :outdated {:extra-deps {olical/depot {:mvn/version "RELEASE"}} - :main-opts ["-m" "depot.outdated.main"]} - - ;; clj -A:kaocha -m kaocha.runner --config-file test/tests.edn + {;; clj -A:kaocha -m kaocha.runner --config-file test/tests.edn :kaocha {:extra-paths ["test"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.0.732"} lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} From a59629f5c2be8a58cd33ea737eb621929acead2a Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 11 Jan 2021 17:18:08 -0600 Subject: [PATCH 09/61] Fix namespace issues and use -M over -A --- deps.edn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deps.edn b/deps.edn index ca79e03..0da34c5 100644 --- a/deps.edn +++ b/deps.edn @@ -29,11 +29,11 @@ lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}}} - ;; clj -Aclj-kondo --lint src + ;; clj -Mclj-kondo --lint src :clj-kondo - {:extra-deps {clj-kondo {:mvn/version "RELEASE"}} + {:extra-deps {clj-kondo/clj-kondo {:mvn/version "RELEASE"}} :main-opts ["-m" "clj-kondo.main"]} - ;; clj -Acoverage - :coverage {:extra-deps {cloverage {:mvn/version "RELEASE"}} + ;; clj -Mcoverage + :coverage {:extra-deps {cloverage/cloverage {:mvn/version "RELEASE"}} :main-opts ["-m" "cloverage.coverage" "-p" "src"]}}} From 097aebcc6b4e3dd168ed6833b61440b0f8a7ffa9 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Tue, 12 Jan 2021 03:42:17 -0600 Subject: [PATCH 10/61] Add support for predicting an ARN if instance already has one --- src/stack_mitosis/predict.clj | 11 +++++++++-- test/stack_mitosis/predict_test.clj | 8 +++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index 9f54e63..4c6dc4f 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -1,6 +1,7 @@ (ns stack-mitosis.predict (:require [stack-mitosis.lookup :as lookup] - [stack-mitosis.request :as r])) + [stack-mitosis.request :as r] + [clojure.string :as str])) (defn attach [db child-id] @@ -12,6 +13,11 @@ (update db :ReadReplicaDBInstanceIdentifiers (partial remove #(= % child-id)))) +(defn predict-arn [instance arn parent-id child-id] + (if arn + (assoc instance :DBInstanceArn (str/replace arn parent-id child-id)) + instance)) + (defmulti predict "Predict contents of instances db after applying operation to instances. @@ -48,7 +54,8 @@ (assoc :ReadReplicaSourceDBInstanceIdentifier parent) ;; remove sources replica list for new replica, and reset backup ;; retention to match what AWS does. - (dissoc :ReadReplicaDBInstanceIdentifiers :BackupRetentionPeriod))))) + (dissoc :ReadReplicaDBInstanceIdentifiers :BackupRetentionPeriod) + (predict-arn (:DBInstanceArn source) parent child))))) (defmethod predict :PromoteReadReplica [instances op] diff --git a/test/stack_mitosis/predict_test.clj b/test/stack_mitosis/predict_test.clj index 06922cb..6d711d0 100644 --- a/test/stack_mitosis/predict_test.clj +++ b/test/stack_mitosis/predict_test.clj @@ -51,11 +51,13 @@ :SourceDBInstanceIdentifier "root" :Port 123}}))) (testing "propagate only *some* instance fields to replica" - (let [root {:DBInstanceIdentifier "root" :BackupRetentionPeriod 1 + (let [root {:DBInstanceIdentifier "root" + :DBInstanceArn "arn:aws:rds:us-west-2:1234567:db:root" :ReadReplicaDBInstanceIdentifiers ["other-clone"] - :Port 123}] + :Port 123 :BackupRetentionPeriod 1}] (is (= [(update-in root [:ReadReplicaDBInstanceIdentifiers] conj "clone") - {:DBInstanceIdentifier "clone" :ReadReplicaSourceDBInstanceIdentifier "root" :Port 123}] + {:DBInstanceIdentifier "clone" :DBInstanceArn "arn:aws:rds:us-west-2:1234567:db:clone" + :ReadReplicaSourceDBInstanceIdentifier "root" :Port 123}] (p/predict [root] (op/create-replica "root" "clone")))))))) (deftest delete From 3206421efceda41fcda7086353775092470d79b9 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Tue, 12 Jan 2021 03:43:36 -0600 Subject: [PATCH 11/61] Extract example instances from policy tests --- src/stack_mitosis/policy.clj | 4 +++- test/stack_mitosis/policy_test.clj | 29 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 3a616db..6e13575 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -17,4 +17,6 @@ resources (group-by :op all-permissions)] {:effect "Allow" :action (keys resources) - :resource (distinct (mapcat :arn resources))})) + :resource (distinct (mapcat :arn resources)) + ;; :raw [all-permissions resources] + })) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 07371db..4d0a51a 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -4,6 +4,16 @@ [stack-mitosis.policy :as sut] [stack-mitosis.planner :as plan])) +(defn example-instances [] + [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] + :DBInstanceArn "production-arn"} + {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" + :DBInstanceArn "production-replica-arn"} + {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] + :DBInstanceArn "staging-arn"} + {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" + :DBInstanceArn "staging-replica-arn"}]) + (deftest permissions (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn "arn:aws:rds:us-east-1"}] (is (= {:op :DeleteDBInstance} @@ -17,16 +27,9 @@ "operation only if no database identifier in request"))) (deftest generate - (let [instances [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] - :DBInstanceArn "production-arn"} - {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" - :DBInstanceArn "production-replica-arn"} - {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] - :DBInstanceArn "staging-arn"} - {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" - :DBInstanceArn "staging-replica-arn"}]] - (is (= {:effect "Allow" - :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] - ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete - :resource []} - (sut/generate instances (plan/replace-tree instances "production" "staging")))))) + (is (= {:effect "Allow" + :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] + ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete + :resource []} + (sut/generate (example-instances) + (plan/replace-tree (example-instances) "production" "staging"))))) From 37929062cd35e2b1efd519023ea5e54d6170fb94 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Tue, 12 Jan 2021 04:20:24 -0600 Subject: [PATCH 12/61] first pass at including resources in effect statement --- src/stack_mitosis/policy.clj | 7 +++---- test/stack_mitosis/policy_test.clj | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 6e13575..c718cb4 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -7,7 +7,8 @@ (if-let [instance (lookup/by-id instances db-id)] {:op (:op action) :arn (:DBInstanceArn instance)} - {:op (:op action)}) + {:op (:op action) + :arn "fake arn"}) ;; TODO handle ResourceName for ListTagsForResource ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) @@ -17,6 +18,4 @@ resources (group-by :op all-permissions)] {:effect "Allow" :action (keys resources) - :resource (distinct (mapcat :arn resources)) - ;; :raw [all-permissions resources] - })) + :resource (distinct (map :arn (flatten (vals resources))))})) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 4d0a51a..5a5ee30 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -16,7 +16,7 @@ (deftest permissions (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn "arn:aws:rds:us-east-1"}] - (is (= {:op :DeleteDBInstance} + (is (= {:op :DeleteDBInstance :arn "fake arn"} (sut/permissions [] (op/delete "foo"))) "only operation if instance is not found") (is (= {:op :DeleteDBInstance :arn "arn:aws:rds:us-east-1"} @@ -30,6 +30,6 @@ (is (= {:effect "Allow" :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete - :resource []} + :resource ["fake arn" "staging-replica-arn" "staging-arn"]} (sut/generate (example-instances) (plan/replace-tree (example-instances) "production" "staging"))))) From dcbe20da19f5fc2a022c622da79f60a1389336c5 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 15:28:53 -0600 Subject: [PATCH 13/61] Increase fidelity of ARN generation and fallback to wildcard if missing --- src/stack_mitosis/policy.clj | 5 ++++- test/stack_mitosis/policy_test.clj | 26 +++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index c718cb4..9835e9f 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -2,13 +2,16 @@ (:require [stack-mitosis.request :as r] [stack-mitosis.lookup :as lookup])) +(defn make-wildcard-arn [db-id] + (str "arn:aws:rds:*:*:db:" db-id)) + (defn permissions [instances action] (if-let [db-id (r/db-id action)] (if-let [instance (lookup/by-id instances db-id)] {:op (:op action) :arn (:DBInstanceArn instance)} {:op (:op action) - :arn "fake arn"}) + :arn (make-wildcard-arn db-id)}) ;; TODO handle ResourceName for ListTagsForResource ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 5a5ee30..d6ea054 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -4,22 +4,25 @@ [stack-mitosis.policy :as sut] [stack-mitosis.planner :as plan])) +(defn make-arn [name] + (str "arn:aws:rds:us-east-1:1234567:db:" name)) + (defn example-instances [] [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] - :DBInstanceArn "production-arn"} + :DBInstanceArn (make-arn "production")} {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" - :DBInstanceArn "production-replica-arn"} + :DBInstanceArn (make-arn "production-replica")} {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] - :DBInstanceArn "staging-arn"} + :DBInstanceArn (make-arn "staging")} {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" - :DBInstanceArn "staging-replica-arn"}]) + :DBInstanceArn (make-arn "staging-replica")}]) (deftest permissions - (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn "arn:aws:rds:us-east-1"}] - (is (= {:op :DeleteDBInstance :arn "fake arn"} - (sut/permissions [] (op/delete "foo"))) + (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn (make-arn "foo")}] + (is (= {:op :DeleteDBInstance :arn (make-arn "foo")} + (sut/permissions [instance] (op/delete "foo"))) "only operation if instance is not found") - (is (= {:op :DeleteDBInstance :arn "arn:aws:rds:us-east-1"} + (is (= {:op :DeleteDBInstance :arn (make-arn "foo")} (sut/permissions [instance] (op/delete "foo"))) "operation and arn if instance is found") (is (= {:op :DescribeDBInstances} @@ -30,6 +33,11 @@ (is (= {:effect "Allow" :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete - :resource ["fake arn" "staging-replica-arn" "staging-arn"]} + :resource [(sut/make-wildcard-arn "temp-staging") + (sut/make-wildcard-arn "temp-staging-replica") + (make-arn "staging-replica") + (make-arn "staging") + (sut/make-wildcard-arn "old-staging-replica") + (sut/make-wildcard-arn "old-staging")]} (sut/generate (example-instances) (plan/replace-tree (example-instances) "production" "staging"))))) From 12864a23562685361c93f83899c91a949adac4bc Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 16:00:28 -0600 Subject: [PATCH 14/61] Predicted ModifyDB renames update the ARN to the new id --- src/stack_mitosis/predict.clj | 14 +++++++++++--- test/stack_mitosis/predict_test.clj | 13 +++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index 4c6dc4f..d3e2bc7 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -13,9 +13,15 @@ (update db :ReadReplicaDBInstanceIdentifiers (partial remove #(= % child-id)))) -(defn predict-arn [instance arn parent-id child-id] +(defn predict-arn + "Replace the database identifier at the end of the ARN with a new value. + + See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_Tagging.ARN.html + for documentation on RDS specific ARN generation." + [instance arn parent-id child-id] (if arn - (assoc instance :DBInstanceArn (str/replace arn parent-id child-id)) + (assoc instance :DBInstanceArn + (str/replace arn (re-pattern (str ":" parent-id "$")) (str ":" child-id))) instance)) (defmulti predict @@ -79,7 +85,9 @@ parent (lookup/parent instances current-id)] (letfn [(new-name [db] (merge (if new-id - (assoc db :DBInstanceIdentifier new-id) + (predict-arn (assoc db :DBInstanceIdentifier new-id) + (:DBInstanceArn db) + current-id new-id) db) ;; merge in everything else in request (dissoc (:request op) diff --git a/test/stack_mitosis/predict_test.clj b/test/stack_mitosis/predict_test.clj index 6d711d0..00498bc 100644 --- a/test/stack_mitosis/predict_test.clj +++ b/test/stack_mitosis/predict_test.clj @@ -13,13 +13,14 @@ (op/create {:DBInstanceIdentifier "a"}))))) (deftest modify - (let [instances [{:DBInstanceIdentifier "a"} - {:DBInstanceIdentifier "b"}]] - (is (= [{:DBInstanceIdentifier "a"} - {:DBInstanceIdentifier "new-name"}] + (let [instances [{:DBInstanceIdentifier "a" :DBInstanceArn "db:a"} + {:DBInstanceIdentifier "b" :DBInstanceArn "db:b"}]] + (is (= [{:DBInstanceIdentifier "a" :DBInstanceArn "db:a"} + {:DBInstanceIdentifier "new-name" :DBInstanceArn "db:new-name"}] (p/predict instances (op/rename "b" "new-name")))) - (is (= [{:DBInstanceIdentifier "a"} - {:DBInstanceIdentifier "b" :MultiAZ true}] + (is (= [{:DBInstanceIdentifier "a" :DBInstanceArn "db:a"} + {:DBInstanceIdentifier "b" :DBInstanceArn "db:b" + :MultiAZ true}] (p/predict instances (op/modify "b" {:MultiAZ true})))))) From e2df22553f078d53a617fcfa89d11cf03725206c Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 16:20:06 -0600 Subject: [PATCH 15/61] Predict ARNs as operations are applied for a superset of permissions --- src/stack_mitosis/policy.clj | 18 ++++++++++++++---- test/stack_mitosis/policy_test.clj | 8 ++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 9835e9f..c3d534b 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -1,6 +1,7 @@ (ns stack-mitosis.policy (:require [stack-mitosis.request :as r] - [stack-mitosis.lookup :as lookup])) + [stack-mitosis.lookup :as lookup] + [stack-mitosis.predict :as predict])) (defn make-wildcard-arn [db-id] (str "arn:aws:rds:*:*:db:" db-id)) @@ -10,14 +11,23 @@ (if-let [instance (lookup/by-id instances db-id)] {:op (:op action) :arn (:DBInstanceArn instance)} - {:op (:op action) - :arn (make-wildcard-arn db-id)}) + ;; FIXME: this is really gross + (if-let [predicted (lookup/by-id (predict/predict instances action) db-id)] + {:op (:op action) + :arn (:DBInstanceArn predicted)} + ;; fallback to wildcard if we don't recognize + {:op (:op action) + :arn (make-wildcard-arn db-id)})) ;; TODO handle ResourceName for ListTagsForResource ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) (defn generate [instances operations] - (let [all-permissions (map (partial permissions instances) operations) + (let [all-permissions + (map permissions + (reductions predict/predict instances operations) + operations) + resources (group-by :op all-permissions)] {:effect "Allow" :action (keys resources) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index d6ea054..35eed8a 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -33,11 +33,11 @@ (is (= {:effect "Allow" :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete - :resource [(sut/make-wildcard-arn "temp-staging") - (sut/make-wildcard-arn "temp-staging-replica") + :resource [(make-arn "temp-staging") + (make-arn "temp-staging-replica") (make-arn "staging-replica") (make-arn "staging") - (sut/make-wildcard-arn "old-staging-replica") - (sut/make-wildcard-arn "old-staging")]} + (make-arn "old-staging-replica") + (make-arn "old-staging")]} (sut/generate (example-instances) (plan/replace-tree (example-instances) "production" "staging"))))) From 31980f7d67a3d952ee05920ace9f380d07a193be Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 16:20:33 -0600 Subject: [PATCH 16/61] Include predicted ARN in all create-replica test cases --- test/stack_mitosis/predict_test.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/stack_mitosis/predict_test.clj b/test/stack_mitosis/predict_test.clj index 00498bc..cfb240f 100644 --- a/test/stack_mitosis/predict_test.clj +++ b/test/stack_mitosis/predict_test.clj @@ -41,10 +41,13 @@ (p/predict [] (op/promote "root")))))) (deftest create-replica - (let [instances [{:DBInstanceIdentifier "root" :MultiAZ false}]] + (let [instances [{:DBInstanceIdentifier "root" :DBInstanceArn "db:root" + :MultiAZ false}]] (is (= [{:DBInstanceIdentifier "root" :MultiAZ false + :DBInstanceArn "db:root" :ReadReplicaDBInstanceIdentifiers ["replica"]} {:DBInstanceIdentifier "replica" :MultiAZ false :Port 123 + :DBInstanceArn "db:replica" :ReadReplicaSourceDBInstanceIdentifier "root"}] (p/predict instances {:op :CreateDBInstanceReadReplica From a6f6b811fff1688f705b8c78b6453513cb9c3043 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 16:25:13 -0600 Subject: [PATCH 17/61] Update comments and leave some hooks for next steps to fix --- src/stack_mitosis/policy.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index c3d534b..fa8bec9 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -12,16 +12,23 @@ {:op (:op action) :arn (:DBInstanceArn instance)} ;; FIXME: this is really gross + ;; For create replica cases we don't yet have an instance with an ARN for + ;; the newly created id, so we use predict to look ahead a step and use + ;; the ARN from the newly created instance. (if-let [predicted (lookup/by-id (predict/predict instances action) db-id)] {:op (:op action) :arn (:DBInstanceArn predicted)} ;; fallback to wildcard if we don't recognize + ;; This probably should never happen, can we ensure this and drop this + ;; case OR should this be a nil arn case? {:op (:op action) :arn (make-wildcard-arn db-id)})) ;; TODO handle ResourceName for ListTagsForResource ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) +;; TODO breakup permissions per operation type with better granularity +;; ie Delete should only have permissions on old-, not temp- or current staging. (defn generate [instances operations] (let [all-permissions (map permissions From 0d2b47db9a969f94bdf88a7490653e22efc6fc8d Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 17:09:13 -0600 Subject: [PATCH 18/61] Use a thread-first for renaming and predicting arn for clarity --- src/stack_mitosis/predict.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index d3e2bc7..d8e1771 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -85,9 +85,9 @@ parent (lookup/parent instances current-id)] (letfn [(new-name [db] (merge (if new-id - (predict-arn (assoc db :DBInstanceIdentifier new-id) - (:DBInstanceArn db) - current-id new-id) + (-> db + (assoc :DBInstanceIdentifier new-id) + (predict-arn (:DBInstanceArn db) current-id new-id)) db) ;; merge in everything else in request (dissoc (:request op) From ea004b349ba6a3260ccb7d04172bae08f84c6f8a Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 18:13:34 -0600 Subject: [PATCH 19/61] Generate policy statements namespaced to each action --- src/stack_mitosis/policy.clj | 16 ++++++++++------ test/stack_mitosis/policy_test.clj | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index fa8bec9..e814f59 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -27,15 +27,19 @@ ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) +;; TODO possibly generate optional statement identifier? +;; TODO simplify action/resource to singular if only one value? +(defn allow [actions resources] + {:effect "Allow" + :action actions + :resource resources}) + ;; TODO breakup permissions per operation type with better granularity ;; ie Delete should only have permissions on old-, not temp- or current staging. (defn generate [instances operations] (let [all-permissions (map permissions (reductions predict/predict instances operations) - operations) - - resources (group-by :op all-permissions)] - {:effect "Allow" - :action (keys resources) - :resource (distinct (map :arn (flatten (vals resources))))})) + operations)] + (for [[op ops] (group-by :op all-permissions)] + (allow [op] (distinct (map :arn ops)))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 35eed8a..eb71866 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -29,15 +29,16 @@ (sut/permissions [] (op/describe))) "operation only if no database identifier in request"))) +;; TODO: how to incorporate permissions for ListTags and DescribeDBInstances +;; also how to create policy for example environment creation? (deftest generate - (is (= {:effect "Allow" - :action [:CreateDBInstanceReadReplica :PromoteReadReplica :ModifyDBInstance :DeleteDBInstance] - ;; FIXME: note that create db, promote, and modify may have a different set of resource permissions from delete - :resource [(make-arn "temp-staging") - (make-arn "temp-staging-replica") - (make-arn "staging-replica") - (make-arn "staging") - (make-arn "old-staging-replica") - (make-arn "old-staging")]} + (is (= [(sut/allow [:CreateDBInstanceReadReplica] + [(make-arn "temp-staging") (make-arn "temp-staging-replica")]) + (sut/allow [:PromoteReadReplica] + [(make-arn "temp-staging")]) + (sut/allow [:ModifyDBInstance] + (mapv make-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) + (sut/allow [:DeleteDBInstance] + (mapv make-arn ["old-staging-replica" "old-staging"]))] (sut/generate (example-instances) (plan/replace-tree (example-instances) "production" "staging"))))) From 0036c6fa5ca9ea8f59e4177fa3d0ca173a28d140 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 18:22:48 -0600 Subject: [PATCH 20/61] Add globals example for describe and listtags and remember reboot --- src/stack_mitosis/policy.clj | 3 +++ test/stack_mitosis/policy_test.clj | 1 + 2 files changed, 4 insertions(+) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index e814f59..2fb449b 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -34,6 +34,9 @@ :action actions :resource resources}) +(defn globals [] + (allow [:DescribeDBInstances :ListTagsForResource] ["arn:aws:rds:*"])) + ;; TODO breakup permissions per operation type with better granularity ;; ie Delete should only have permissions on old-, not temp- or current staging. (defn generate [instances operations] diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index eb71866..73e887f 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -31,6 +31,7 @@ ;; TODO: how to incorporate permissions for ListTags and DescribeDBInstances ;; also how to create policy for example environment creation? +;; TODO: handle possible need for RebootInstance if renaming with ApplyImmediatly? (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] [(make-arn "temp-staging") (make-arn "temp-staging-replica")]) From c81571ad941816e9cf752c617e019d7fa888b1e6 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 18:25:36 -0600 Subject: [PATCH 21/61] Use mapv to simplify arn generation --- test/stack_mitosis/policy_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 73e887f..f1733d8 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -34,7 +34,7 @@ ;; TODO: handle possible need for RebootInstance if renaming with ApplyImmediatly? (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] - [(make-arn "temp-staging") (make-arn "temp-staging-replica")]) + (mapv make-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(make-arn "temp-staging")]) (sut/allow [:ModifyDBInstance] From 87783e1a4ca580007fd74e002c21aa139ebbefc4 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Thu, 14 Jan 2021 18:26:38 -0600 Subject: [PATCH 22/61] Rename make-arn -> fake-arn to clarify intent --- test/stack_mitosis/policy_test.clj | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index f1733d8..434d535 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -4,25 +4,25 @@ [stack-mitosis.policy :as sut] [stack-mitosis.planner :as plan])) -(defn make-arn [name] +(defn fake-arn [name] (str "arn:aws:rds:us-east-1:1234567:db:" name)) (defn example-instances [] [{:DBInstanceIdentifier "production" :ReadReplicaDBInstanceIdentifiers ["production-replica"] - :DBInstanceArn (make-arn "production")} + :DBInstanceArn (fake-arn "production")} {:DBInstanceIdentifier "production-replica" :ReadReplicaSourceDBInstanceIdentifier "production" - :DBInstanceArn (make-arn "production-replica")} + :DBInstanceArn (fake-arn "production-replica")} {:DBInstanceIdentifier "staging" :ReadReplicaDBInstanceIdentifiers ["staging-replica"] - :DBInstanceArn (make-arn "staging")} + :DBInstanceArn (fake-arn "staging")} {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" - :DBInstanceArn (make-arn "staging-replica")}]) + :DBInstanceArn (fake-arn "staging-replica")}]) (deftest permissions - (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn (make-arn "foo")}] - (is (= {:op :DeleteDBInstance :arn (make-arn "foo")} + (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn (fake-arn "foo")}] + (is (= {:op :DeleteDBInstance :arn (fake-arn "foo")} (sut/permissions [instance] (op/delete "foo"))) "only operation if instance is not found") - (is (= {:op :DeleteDBInstance :arn (make-arn "foo")} + (is (= {:op :DeleteDBInstance :arn (fake-arn "foo")} (sut/permissions [instance] (op/delete "foo"))) "operation and arn if instance is found") (is (= {:op :DescribeDBInstances} @@ -34,12 +34,12 @@ ;; TODO: handle possible need for RebootInstance if renaming with ApplyImmediatly? (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] - (mapv make-arn ["temp-staging" "temp-staging-replica"])) + (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] - [(make-arn "temp-staging")]) + [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance] - (mapv make-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) (sut/allow [:DeleteDBInstance] - (mapv make-arn ["old-staging-replica" "old-staging"]))] + (mapv fake-arn ["old-staging-replica" "old-staging"]))] (sut/generate (example-instances) (plan/replace-tree (example-instances) "production" "staging"))))) From 7c02028075316e33178fd7f97665d59dabfbf348 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 15 Jan 2021 10:35:00 -0600 Subject: [PATCH 23/61] Prefix allow actions with rds: --- src/stack_mitosis/policy.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 2fb449b..a3218e5 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -31,7 +31,7 @@ ;; TODO simplify action/resource to singular if only one value? (defn allow [actions resources] {:effect "Allow" - :action actions + :action (mapv (partial str "rds") actions) :resource resources}) (defn globals [] From 1cf18c3e2e798b3002c7b33b9b57e882e04f089b Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 15 Jan 2021 10:54:53 -0600 Subject: [PATCH 24/61] Ensure that modify also confers reboot permissions --- src/stack_mitosis/policy.clj | 6 +++++- test/stack_mitosis/policy_test.clj | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index a3218e5..45d36e2 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -45,4 +45,8 @@ (reductions predict/predict instances operations) operations)] (for [[op ops] (group-by :op all-permissions)] - (allow [op] (distinct (map :arn ops)))))) + ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot + (cond (= op :ModifyDBInstance) + (allow [op :RebootInstance] (distinct (map :arn ops))) + :else + (allow [op] (distinct (map :arn ops))))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 434d535..127aeb3 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -31,13 +31,12 @@ ;; TODO: how to incorporate permissions for ListTags and DescribeDBInstances ;; also how to create policy for example environment creation? -;; TODO: handle possible need for RebootInstance if renaming with ApplyImmediatly? (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) - (sut/allow [:ModifyDBInstance] + (sut/allow [:ModifyDBInstance :RebootInstance] (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))] From 77fb9f63448318a478e706564936b67cc44c6ce9 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 15 Jan 2021 10:57:24 -0600 Subject: [PATCH 25/61] Incorporate permissions for example environment --- src/stack_mitosis/policy.clj | 7 ++++++- test/stack_mitosis/policy_test.clj | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 45d36e2..e1f164a 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -34,8 +34,13 @@ :action (mapv (partial str "rds") actions) :resource resources}) +(defn create-example [] + (allow [:CreateDBInstance :AddTagsToResource] + ["arn:aws:rds:*:*:db:mitosis-*"])) + (defn globals [] - (allow [:DescribeDBInstances :ListTagsForResource] ["arn:aws:rds:*"])) + (allow [:DescribeDBInstances :ListTagsForResource] + ["arn:aws:rds:*"])) ;; TODO breakup permissions per operation type with better granularity ;; ie Delete should only have permissions on old-, not temp- or current staging. diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 127aeb3..692d6e7 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -29,8 +29,6 @@ (sut/permissions [] (op/describe))) "operation only if no database identifier in request"))) -;; TODO: how to incorporate permissions for ListTags and DescribeDBInstances -;; also how to create policy for example environment creation? (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] (mapv fake-arn ["temp-staging" "temp-staging-replica"])) From 5a331f62a62b3646718c3618b2c7ff657dee27cb Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 15 Jan 2021 11:01:15 -0600 Subject: [PATCH 26/61] Tighten up wildcards --- src/stack_mitosis/policy.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index e1f164a..9602895 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -36,11 +36,11 @@ (defn create-example [] (allow [:CreateDBInstance :AddTagsToResource] - ["arn:aws:rds:*:*:db:mitosis-*"])) + [(make-wildcard-arn "mitosis-*")])) (defn globals [] (allow [:DescribeDBInstances :ListTagsForResource] - ["arn:aws:rds:*"])) + [(make-wildcard-arn "*")])) ;; TODO breakup permissions per operation type with better granularity ;; ie Delete should only have permissions on old-, not temp- or current staging. From 063f36ce34636a790c1ce0e360f98b46fa8fc9f2 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 15 Jan 2021 11:11:42 -0600 Subject: [PATCH 27/61] Generate json policy --- src/stack_mitosis/policy.clj | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 9602895..14f70c0 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -1,7 +1,8 @@ (ns stack-mitosis.policy (:require [stack-mitosis.request :as r] [stack-mitosis.lookup :as lookup] - [stack-mitosis.predict :as predict])) + [stack-mitosis.predict :as predict] + [clojure.data.json :as json])) (defn make-wildcard-arn [db-id] (str "arn:aws:rds:*:*:db:" db-id)) @@ -30,9 +31,9 @@ ;; TODO possibly generate optional statement identifier? ;; TODO simplify action/resource to singular if only one value? (defn allow [actions resources] - {:effect "Allow" - :action (mapv (partial str "rds") actions) - :resource resources}) + {:Effect "Allow" + :Action (mapv (partial str "rds") actions) + :Resource resources}) (defn create-example [] (allow [:CreateDBInstance :AddTagsToResource] @@ -55,3 +56,9 @@ (allow [op :RebootInstance] (distinct (map :arn ops))) :else (allow [op] (distinct (map :arn ops))))))) + +(defn as-json [statements] + (json/pprint {:Version "2012-10-17" :Statements statements})) + +(comment + (println (as-json [(globals) (create-example)]))) From 784897e6fdd900a98ebe197526ae41de154c17e0 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Sat, 16 Jan 2021 11:52:14 -0600 Subject: [PATCH 28/61] Add options to pprint IAM policy generated from expected plan --- src/stack_mitosis/cli.clj | 18 +++++++++++++----- src/stack_mitosis/policy.clj | 13 ++++++++----- test/stack_mitosis/policy_test.clj | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/stack_mitosis/cli.clj b/src/stack_mitosis/cli.clj index 956a634..a966c5b 100644 --- a/src/stack_mitosis/cli.clj +++ b/src/stack_mitosis/cli.clj @@ -1,11 +1,13 @@ (ns stack-mitosis.cli - (:require [clojure.tools.cli :as cli] + (:require [clojure.data.json :as json] + [clojure.string :as str] + [clojure.tools.cli :as cli] + [clojure.tools.logging :as log] [stack-mitosis.interpreter :as interpreter] [stack-mitosis.planner :as plan] + [stack-mitosis.policy :as policy] [stack-mitosis.request :as r] - [clojure.string :as str] - [stack-mitosis.sudo :as sudo] - [clojure.tools.logging :as log])) + [stack-mitosis.sudo :as sudo])) ;; TODO: add max-timeout for actions ;; TODO: show attempt info like skipped steps in flight plan? @@ -16,7 +18,8 @@ ["-t" "--target DST" "Root identifier of database tree to copy over"] [nil "--restart CMD" "Blocking script to restart application."] ["-c" "--credentials FILENAME" "Credentials file in edn for iam assume-role"] - ["-p" "--plan", "Display expected flightplan for operation."] + ["-p" "--plan" "Display expected flightplan for operation."] + ["-i" "--iam-policy" "Generate IAM policy for planned actions."] ["-h" "--help"]]) (defn parse-args [args] @@ -55,6 +58,9 @@ (cond (:plan options) (do (println (flight-plan (interpreter/check-plan instances plan))) true) + (:iam-policy options) + (do (json/pprint (policy/from-plan instances plan)) + true) :else (let [last-action (interpreter/evaluate-plan rds plan)] (not (contains? last-action :ErrorResponse)))))))) @@ -72,4 +78,6 @@ "--plan" "--restart" "'./service-restart.sh'"])) (process (parse-args ["--source" "mitosis-prod" "--target" "mitosis-demo" "--plan" "--credentials" "resources/role.edn"])) + (process (parse-args ["--source" "mitosis-prod" "--target" "mitosis-demo" + "--iam-policy"])) (process (parse-args ["--source" "mitosis-prod" "--target" "mitosis-demo"]))) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 14f70c0..c608876 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -1,8 +1,7 @@ (ns stack-mitosis.policy (:require [stack-mitosis.request :as r] [stack-mitosis.lookup :as lookup] - [stack-mitosis.predict :as predict] - [clojure.data.json :as json])) + [stack-mitosis.predict :as predict])) (defn make-wildcard-arn [db-id] (str "arn:aws:rds:*:*:db:" db-id)) @@ -57,8 +56,12 @@ :else (allow [op] (distinct (map :arn ops))))))) -(defn as-json [statements] - (json/pprint {:Version "2012-10-17" :Statements statements})) +(defn policy [statements] + {:Version "2012-10-17" :Statements statements}) + +(defn from-plan [instances operations] + (policy (concat [(globals)] (generate instances operations)))) (comment - (println (as-json [(globals) (create-example)]))) + (require '(clojure.data.json :as json)) + (json/pprint (policy [(globals) (create-example)]))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 692d6e7..c088d86 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -40,3 +40,19 @@ (mapv fake-arn ["old-staging-replica" "old-staging"]))] (sut/generate (example-instances) (plan/replace-tree (example-instances) "production" "staging"))))) + +(deftest from-plan + (is (= {:Version "2012-10-17" + :Statements + [(sut/allow [:DescribeDBInstances :ListTagsForResource] + ["arn:aws:rds:*:*:db:*"]) + (sut/allow [:CreateDBInstanceReadReplica] + (mapv fake-arn ["temp-staging" "temp-staging-replica"])) + (sut/allow [:PromoteReadReplica] + [(fake-arn "temp-staging")]) + (sut/allow [:ModifyDBInstance :RebootInstance] + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) + (sut/allow [:DeleteDBInstance] + (mapv fake-arn ["old-staging-replica" "old-staging"]))]} + (sut/from-plan (example-instances) + (plan/replace-tree (example-instances) "production" "staging"))))) From a183bb50e0c151a670a5b5802a8035abc7a64bba Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 16:22:20 -0600 Subject: [PATCH 29/61] Statement not Statements in policy definition --- src/stack_mitosis/policy.clj | 2 +- test/stack_mitosis/policy_test.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index c608876..75e22ed 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -57,7 +57,7 @@ (allow [op] (distinct (map :arn ops))))))) (defn policy [statements] - {:Version "2012-10-17" :Statements statements}) + {:Version "2012-10-17" :Statement statements}) (defn from-plan [instances operations] (policy (concat [(globals)] (generate instances operations)))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index c088d86..3e13dbd 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -43,7 +43,7 @@ (deftest from-plan (is (= {:Version "2012-10-17" - :Statements + :Statement [(sut/allow [:DescribeDBInstances :ListTagsForResource] ["arn:aws:rds:*:*:db:*"]) (sut/allow [:CreateDBInstanceReadReplica] From 7a84c0b7ff6be7671ec7990b6a279c7c1e72c335 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 16:22:34 -0600 Subject: [PATCH 30/61] Some minor changes to make it easier to test --- src/stack_mitosis/example_environment.clj | 4 ++-- src/stack_mitosis/interpreter.clj | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/example_environment.clj b/src/stack_mitosis/example_environment.clj index 4c396e7..4fa37d9 100644 --- a/src/stack_mitosis/example_environment.clj +++ b/src/stack_mitosis/example_environment.clj @@ -6,9 +6,9 @@ [stack-mitosis.predict :as predict])) (def template - {:DBInstanceClass "db.t3.micro" + {:DBInstanceClass "db.t2.micro" :Engine "postgres" ;;"mysql" - :StorageType "gp2" + :StorageType "gp2" ;; ie ssd storage :AllocatedStorage 5 :PubliclyAccessible false :MasterUsername "root"}) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index b77438b..bb40ae8 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -181,6 +181,7 @@ (list-tags rds instances "mitosis-demo") + (lookup/by-id (databases rds) "mitosis-prod") (let [instances (databases rds)] (clojure.data/diff (lookup/by-id instances "mitosis-prod") From 7684fb1c49cf0e18f1c9ed75cb889fc597d7a5e5 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 16:23:15 -0600 Subject: [PATCH 31/61] Use wildcards by not specifying region or account-id --- src/stack_mitosis/policy.clj | 2 +- test/stack_mitosis/policy_test.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 75e22ed..b6a7373 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -4,7 +4,7 @@ [stack-mitosis.predict :as predict])) (defn make-wildcard-arn [db-id] - (str "arn:aws:rds:*:*:db:" db-id)) + (str "arn:aws:rds:::db:" db-id)) (defn permissions [instances action] (if-let [db-id (r/db-id action)] diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 3e13dbd..6b5cc73 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -45,7 +45,7 @@ (is (= {:Version "2012-10-17" :Statement [(sut/allow [:DescribeDBInstances :ListTagsForResource] - ["arn:aws:rds:*:*:db:*"]) + ["arn:aws:rds:::db:*"]) (sut/allow [:CreateDBInstanceReadReplica] (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] From b3219bf1a262cdd7e6806269047e10af62f76721 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 16:26:37 -0600 Subject: [PATCH 32/61] Disable empty databases post-condition as it won't work on a bare account --- src/stack_mitosis/interpreter.clj | 4 +++- test/stack_mitosis/interpreter_test.clj | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index bb40ae8..9add41e 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -36,7 +36,9 @@ (defn databases [rds] - {:post [(seq %)]} + ;; FIXME: if account has *no* databases this fails too, need a more nuanced + ;; postcondition here + ;; {:post [(seq %)]} (:DBInstances (invoke-logged! rds (op/describe)))) ;; TODO: verify that "old-" database copies do not exist before running diff --git a/test/stack_mitosis/interpreter_test.clj b/test/stack_mitosis/interpreter_test.clj index 62d2ad3..e2ef6b8 100644 --- a/test/stack_mitosis/interpreter_test.clj +++ b/test/stack_mitosis/interpreter_test.clj @@ -23,9 +23,10 @@ (with-redefs [aws/invoke (mock-invoke [{:DBInstanceIdentifier "a"}])] (is (= [{:DBInstanceIdentifier "a"}] (sut/databases identity)))) - (with-redefs [aws/invoke (mock-invoke [])] - (is (thrown-with-msg? java.lang.AssertionError #"Assert failed" - (sut/databases identity))))) + ;; (with-redefs [aws/invoke (mock-invoke [])] + ;; (is (thrown-with-msg? java.lang.AssertionError #"Assert failed" + ;; (sut/databases identity)))) + ) (deftest evaluate-plan (with-redefs [aws/invoke (mock-invoke [{:DBInstanceIdentifier "a"}])] From dc22a371ac2fdcb6e0a8f3312c59d6ef0128effd Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 16:37:07 -0600 Subject: [PATCH 33/61] Update LICENSE years and add to base of README --- LICENSE | 2 +- README.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 93ae5f0..9188c40 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2019, Charles L.G. Comstock +Copyright (c) 2019-2021, Charles L.G. Comstock All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 020628b..8b9ad01 100644 --- a/README.md +++ b/README.md @@ -116,3 +116,9 @@ Note that for many cases, even if the clone process is interrupted, the flight p bin/kaocha # basic unit tests bin/kaocha --plugin cloverage # with coverage output + +# License + +Copyright © 2019-2021 Charles L.G. Comstock + +Distributed under the BSD-3 Clause License (see LICENSE file) From d14822f50600dee928dc716947626c7864e010d7 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 16:52:29 -0600 Subject: [PATCH 34/61] Explain why not cloudformation/terraform --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 8b9ad01..ed37d96 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,14 @@ Note that for many cases, even if the clone process is interrupted, the flight p bin/kaocha # basic unit tests bin/kaocha --plugin cloverage # with coverage output +# Frequently Asked Questions + +## Why not use Cloudformation/Terraform + +Cloudformation and Terraform are wonderful tools focused on declarative architecture transformation from one steady state to another. Stack-mitosis is focused on safely cloning the contents of a database in one environment to another without changing from one steady state to another. As example, for an environment with production and demo environments, they both exist in the correct configuration before running stack-mitosis, and then after running stack-mitosis the configuration remains the same but the demo environment has a fresh copy of the data from production. + +I suspect this could also be accomplished using one of these declarative infrastructure tools by transitioning through multiple intervening states, but have not found any examples of anyone doing that. + # License Copyright © 2019-2021 Charles L.G. Comstock From 3cf7b8fcc15cb3700a9c3b870b6626f4a6ba51e1 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:05:23 -0600 Subject: [PATCH 35/61] Document IAM policy generation --- README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/README.md b/README.md index ed37d96..9cd4903 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ Hopefully in the future this can be parsed directly from the `AWS_CONFIG` file. --restart "./restart-service.sh" --credentials resources/role.edn [--plan] + [--iam-policy] ## Flight Plan @@ -112,6 +113,42 @@ Flight plan: Note that for many cases, even if the clone process is interrupted, the flight plan will show steps it will try to execute again, and steps it will skip because it has detected that the instance has already been created or modified to the right attribute values. In other words, it tries to pickup where it left-off if there is a failure. +## IAM Policy Generation + +Stack-mitosis can also generate an IAM policy for an automated user to update a particular environment. The policy uses the database names from the planned changeset to calculate these minimal permissions. While they have been elided from the example below, the ARNs are locked to the account & region. + +``` +$ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-policy +{"Version":"2012-10-17", + "Statement": + [{"Effect":"Allow", + "Action":["rds:DescribeDBInstances", "rds:ListTagsForResource"], + "Resource":["arn:aws:rds:::db:*"]}, + {"Effect":"Allow", + "Action":["rds:CreateDBInstanceReadReplica"], + "Resource": + ["arn:aws:rds:::db:temp-mitosis-demo", + "arn:aws:rds:::db:temp-mitosis-demo-replica"]}, + {"Effect":"Allow", + "Action":["rds:PromoteReadReplica"], + "Resource": + ["arn:aws:rds:::db:temp-mitosis-demo"]}, + {"Effect":"Allow", + "Action":["rds:ModifyDBInstance", "rds:RebootInstance"], + "Resource": + ["arn:aws:rds:::db:temp-mitosis-demo", + "arn:aws:rds:::db:temp-mitosis-demo-replica", + "arn:aws:rds:::db:mitosis-demo-replica", + "arn:aws:rds:::db:mitosis-demo"]}, + {"Effect":"Allow", + "Action":["rds:DeleteDBInstance"], + "Resource": + ["arn:aws:rds:::db:old-mitosis-demo-replica", + "arn:aws:rds:::db:old-mitosis-demo"]}]} +``` + +This ensures that a continuous integration or cronjob server like Jenkins can clone production to demo environments on a weekly basis restricted to the minimal permissions necessary. If a user needs to run stack-mitosis for multiple environments (demo, staging, random developer test environment), then a policy can be attached for each environment. + # Testing bin/kaocha # basic unit tests From e1d9cee1526425c4942e4b90c6b64935f4f712c6 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:16:58 -0600 Subject: [PATCH 36/61] *:* is in fact required permissions to describe all --- README.md | 20 ++++++++++---------- src/stack_mitosis/policy.clj | 2 +- test/stack_mitosis/policy_test.clj | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9cd4903..79b465e 100644 --- a/README.md +++ b/README.md @@ -123,28 +123,28 @@ $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-pol "Statement": [{"Effect":"Allow", "Action":["rds:DescribeDBInstances", "rds:ListTagsForResource"], - "Resource":["arn:aws:rds:::db:*"]}, + "Resource":["arn:aws:rds:*:*:db:*"]}, {"Effect":"Allow", "Action":["rds:CreateDBInstanceReadReplica"], "Resource": - ["arn:aws:rds:::db:temp-mitosis-demo", - "arn:aws:rds:::db:temp-mitosis-demo-replica"]}, + ["arn:aws:rds:*:*:db:temp-mitosis-demo", + "arn:aws:rds:*:*:db:temp-mitosis-demo-replica"]}, {"Effect":"Allow", "Action":["rds:PromoteReadReplica"], "Resource": - ["arn:aws:rds:::db:temp-mitosis-demo"]}, + ["arn:aws:rds:*:*:db:temp-mitosis-demo"]}, {"Effect":"Allow", "Action":["rds:ModifyDBInstance", "rds:RebootInstance"], "Resource": - ["arn:aws:rds:::db:temp-mitosis-demo", - "arn:aws:rds:::db:temp-mitosis-demo-replica", - "arn:aws:rds:::db:mitosis-demo-replica", - "arn:aws:rds:::db:mitosis-demo"]}, + ["arn:aws:rds:*:*:db:temp-mitosis-demo", + "arn:aws:rds:*:*:db:temp-mitosis-demo-replica", + "arn:aws:rds:*:*:db:mitosis-demo-replica", + "arn:aws:rds:*:*:db:mitosis-demo"]}, {"Effect":"Allow", "Action":["rds:DeleteDBInstance"], "Resource": - ["arn:aws:rds:::db:old-mitosis-demo-replica", - "arn:aws:rds:::db:old-mitosis-demo"]}]} + ["arn:aws:rds:*:*:db:old-mitosis-demo-replica", + "arn:aws:rds:*:*:db:old-mitosis-demo"]}]} ``` This ensures that a continuous integration or cronjob server like Jenkins can clone production to demo environments on a weekly basis restricted to the minimal permissions necessary. If a user needs to run stack-mitosis for multiple environments (demo, staging, random developer test environment), then a policy can be attached for each environment. diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index b6a7373..75e22ed 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -4,7 +4,7 @@ [stack-mitosis.predict :as predict])) (defn make-wildcard-arn [db-id] - (str "arn:aws:rds:::db:" db-id)) + (str "arn:aws:rds:*:*:db:" db-id)) (defn permissions [instances action] (if-let [db-id (r/db-id action)] diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 6b5cc73..3e13dbd 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -45,7 +45,7 @@ (is (= {:Version "2012-10-17" :Statement [(sut/allow [:DescribeDBInstances :ListTagsForResource] - ["arn:aws:rds:::db:*"]) + ["arn:aws:rds:*:*:db:*"]) (sut/allow [:CreateDBInstanceReadReplica] (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] From 8728dee2430346015c83e8f49ce56c5157776cd9 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:19:33 -0600 Subject: [PATCH 37/61] Fix typo and rename rds:RebootInstance to rds:RebootDBInstance --- README.md | 2 +- src/stack_mitosis/policy.clj | 2 +- test/stack_mitosis/policy_test.clj | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 79b465e..68e3f75 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-pol "Resource": ["arn:aws:rds:*:*:db:temp-mitosis-demo"]}, {"Effect":"Allow", - "Action":["rds:ModifyDBInstance", "rds:RebootInstance"], + "Action":["rds:ModifyDBInstance", "rds:RebootDBInstance"], "Resource": ["arn:aws:rds:*:*:db:temp-mitosis-demo", "arn:aws:rds:*:*:db:temp-mitosis-demo-replica", diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 75e22ed..f74602a 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -52,7 +52,7 @@ (for [[op ops] (group-by :op all-permissions)] ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot (cond (= op :ModifyDBInstance) - (allow [op :RebootInstance] (distinct (map :arn ops))) + (allow [op :RebootDBInstance] (distinct (map :arn ops))) :else (allow [op] (distinct (map :arn ops))))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 3e13dbd..577a3a6 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -34,7 +34,7 @@ (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) - (sut/allow [:ModifyDBInstance :RebootInstance] + (sut/allow [:ModifyDBInstance :RebootDBInstance] (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))] @@ -50,7 +50,7 @@ (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) - (sut/allow [:ModifyDBInstance :RebootInstance] + (sut/allow [:ModifyDBInstance :RebootDBInstance] (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} From c8d11027c4d78680ab20b2410df83f8e9f044504 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:30:25 -0600 Subject: [PATCH 38/61] Extend make-wildcard-arn to include types like db, subgrp, etc --- src/stack_mitosis/policy.clj | 7 ++++--- test/stack_mitosis/policy_test.clj | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index f74602a..c6694db 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -1,10 +1,11 @@ (ns stack-mitosis.policy (:require [stack-mitosis.request :as r] [stack-mitosis.lookup :as lookup] - [stack-mitosis.predict :as predict])) + [stack-mitosis.predict :as predict] + [clojure.string :as str])) -(defn make-wildcard-arn [db-id] - (str "arn:aws:rds:*:*:db:" db-id)) +(defn make-wildcard-arn [db-id & {:keys [type] :or {type "db"}}] + (str/join ":" ["arn:aws:rds:*:*" type db-id])) (defn permissions [instances action] (if-let [db-id (r/db-id action)] diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 577a3a6..a6a2f4e 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -17,6 +17,12 @@ {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" :DBInstanceArn (fake-arn "staging-replica")}]) +(deftest make-wildcard-arn + (is (= "arn:aws:rds:*:*:db:production" + (sut/make-wildcard-arn "production"))) + (is (= "arn:aws:rds:*:*:og:*" + (sut/make-wildcard-arn "*" :type "og")))) + (deftest permissions (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn (fake-arn "foo")}] (is (= {:op :DeleteDBInstance :arn (fake-arn "foo")} From 76c41bd89eac4d8c769841b8ea346fecfacf350d Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:36:59 -0600 Subject: [PATCH 39/61] Rename wildcard to make-arn and extend to cover account-id and region --- src/stack_mitosis/policy.clj | 12 +++++++----- test/stack_mitosis/policy_test.clj | 10 +++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index c6694db..fe14a7b 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -4,8 +4,10 @@ [stack-mitosis.predict :as predict] [clojure.string :as str])) -(defn make-wildcard-arn [db-id & {:keys [type] :or {type "db"}}] - (str/join ":" ["arn:aws:rds:*:*" type db-id])) +(defn make-arn + [db-id & {:keys [account-id region type] + :or {account-id "*" region "*" type "db"}}] + (str/join ":" ["arn:aws:rds" region account-id type db-id])) (defn permissions [instances action] (if-let [db-id (r/db-id action)] @@ -23,7 +25,7 @@ ;; This probably should never happen, can we ensure this and drop this ;; case OR should this be a nil arn case? {:op (:op action) - :arn (make-wildcard-arn db-id)})) + :arn (make-arn db-id)})) ;; TODO handle ResourceName for ListTagsForResource ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) @@ -37,11 +39,11 @@ (defn create-example [] (allow [:CreateDBInstance :AddTagsToResource] - [(make-wildcard-arn "mitosis-*")])) + [(make-arn "mitosis-*")])) (defn globals [] (allow [:DescribeDBInstances :ListTagsForResource] - [(make-wildcard-arn "*")])) + [(make-arn "*")])) ;; TODO breakup permissions per operation type with better granularity ;; ie Delete should only have permissions on old-, not temp- or current staging. diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index a6a2f4e..6381fef 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -17,11 +17,15 @@ {:DBInstanceIdentifier "staging-replica" :ReadReplicaSourceDBInstanceIdentifier "staging" :DBInstanceArn (fake-arn "staging-replica")}]) -(deftest make-wildcard-arn +(deftest make-arn (is (= "arn:aws:rds:*:*:db:production" - (sut/make-wildcard-arn "production"))) + (sut/make-arn "production"))) (is (= "arn:aws:rds:*:*:og:*" - (sut/make-wildcard-arn "*" :type "og")))) + (sut/make-arn "*" :type "og"))) + (is (= "arn:aws:rds:us-east-1:*:db:*" + (sut/make-arn "*" :region "us-east-1"))) + (is (= "arn:aws:rds:*:1234:db:*" + (sut/make-arn "*" :account-id "1234")))) (deftest permissions (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn (fake-arn "foo")}] From 28ea314433d43db161bb549ccb0f6572d2c628e9 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:37:21 -0600 Subject: [PATCH 40/61] Extract common arns listing --- src/stack_mitosis/policy.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index fe14a7b..21eb575 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -52,12 +52,13 @@ (map permissions (reductions predict/predict instances operations) operations)] - (for [[op ops] (group-by :op all-permissions)] + (for [[op ops] (group-by :op all-permissions) + :let [arns (distinct (map :arn ops))]] ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot (cond (= op :ModifyDBInstance) - (allow [op :RebootDBInstance] (distinct (map :arn ops))) + (allow [op :RebootDBInstance] arns) :else - (allow [op] (distinct (map :arn ops))))))) + (allow [op] arns))))) (defn policy [statements] {:Version "2012-10-17" :Statement statements}) From 1a04a5d85c026e07b33e22b951c3f966e30193bd Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 17:53:28 -0600 Subject: [PATCH 41/61] Incorporate wildcard option group, parameter group, and subnet group permissions --- README.md | 7 +++++-- src/stack_mitosis/policy.clj | 17 ++++++++++++++--- test/stack_mitosis/policy_test.clj | 18 ++++++++++++++---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 68e3f75..163a4b6 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,9 @@ $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-pol {"Effect":"Allow", "Action":["rds:CreateDBInstanceReadReplica"], "Resource": - ["arn:aws:rds:*:*:db:temp-mitosis-demo", + ["arn:aws:rds:*:*:og:*", "arn:aws:rds:*:*:pg:*", + "arn:aws:rds:*:*:subgrp:*", + "arn:aws:rds:*:*:db:temp-mitosis-demo", "arn:aws:rds:*:*:db:temp-mitosis-demo-replica"]}, {"Effect":"Allow", "Action":["rds:PromoteReadReplica"], @@ -136,7 +138,8 @@ $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-pol {"Effect":"Allow", "Action":["rds:ModifyDBInstance", "rds:RebootDBInstance"], "Resource": - ["arn:aws:rds:*:*:db:temp-mitosis-demo", + ["arn:aws:rds:*:*:og:*", "arn:aws:rds:*:*:subgrp:*", + "arn:aws:rds:*:*:db:temp-mitosis-demo", "arn:aws:rds:*:*:db:temp-mitosis-demo-replica", "arn:aws:rds:*:*:db:mitosis-demo-replica", "arn:aws:rds:*:*:db:mitosis-demo"]}, diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 21eb575..159aff4 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -47,6 +47,8 @@ ;; TODO breakup permissions per operation type with better granularity ;; ie Delete should only have permissions on old-, not temp- or current staging. +;; TODO tighten restrictions on DB OptionGroup (og), DB ParameterGroup (pg) and DB Subnet Group (subgrp) +;; these were listed as warnings in the policy editor so leaving wildcard for now (defn generate [instances operations] (let [all-permissions (map permissions @@ -54,9 +56,18 @@ operations)] (for [[op ops] (group-by :op all-permissions) :let [arns (distinct (map :arn ops))]] - ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot - (cond (= op :ModifyDBInstance) - (allow [op :RebootDBInstance] arns) + (cond (= op :CreateDBInstanceReadReplica) + (allow [op] + (into [(make-arn "*" :type "og") + (make-arn "*" :type "pg") + (make-arn "*" :type "subgrp")] + arns)) + (= op :ModifyDBInstance) + ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot + (allow [op :RebootDBInstance] + (into [(make-arn "*" :type "og") + (make-arn "*" :type "subgrp")] + arns)) :else (allow [op] arns))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 6381fef..11928dd 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -41,11 +41,16 @@ (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] - (mapv fake-arn ["temp-staging" "temp-staging-replica"])) + (into ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica"]))) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] - (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) + (into ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"]))) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))] (sut/generate (example-instances) @@ -57,11 +62,16 @@ [(sut/allow [:DescribeDBInstances :ListTagsForResource] ["arn:aws:rds:*:*:db:*"]) (sut/allow [:CreateDBInstanceReadReplica] - (mapv fake-arn ["temp-staging" "temp-staging-replica"])) + (into ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica"]))) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] - (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) + (into ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"]))) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} (sut/from-plan (example-instances) From 0a71064e7f54cc364ae8a438db42c107579ce460 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 18:13:45 -0600 Subject: [PATCH 42/61] create-replica requires permissions for the source ARN not the target --- src/stack_mitosis/policy.clj | 22 +++++++++++----------- test/stack_mitosis/policy_test.clj | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 159aff4..80fdb7c 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -15,17 +15,17 @@ {:op (:op action) :arn (:DBInstanceArn instance)} ;; FIXME: this is really gross - ;; For create replica cases we don't yet have an instance with an ARN for - ;; the newly created id, so we use predict to look ahead a step and use - ;; the ARN from the newly created instance. - (if-let [predicted (lookup/by-id (predict/predict instances action) db-id)] - {:op (:op action) - :arn (:DBInstanceArn predicted)} - ;; fallback to wildcard if we don't recognize - ;; This probably should never happen, can we ensure this and drop this - ;; case OR should this be a nil arn case? - {:op (:op action) - :arn (make-arn db-id)})) + ;; For create replica, use the ARN from the source database + (let [source-id (r/source-id action) + source (lookup/by-id instances source-id)] + (if (and source-id source) + {:op (:op action) + :arn (:DBInstanceArn source)} + ;; fallback to wildcard if we don't recognize + ;; This probably should never happen, can we ensure this and drop this + ;; case OR should this be a nil arn case? + {:op (:op action) + :arn (make-arn db-id)}))) ;; TODO handle ResourceName for ListTagsForResource ;; TODO Exclude shell command, and include top level describe? {:op (:op action)})) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 11928dd..65e6529 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -44,7 +44,7 @@ (into ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" "arn:aws:rds:*:*:subgrp:*"] - (mapv fake-arn ["temp-staging" "temp-staging-replica"]))) + (mapv fake-arn ["production" "temp-staging"]))) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] @@ -65,7 +65,7 @@ (into ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" "arn:aws:rds:*:*:subgrp:*"] - (mapv fake-arn ["temp-staging" "temp-staging-replica"]))) + (mapv fake-arn ["production" "temp-staging"]))) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] From c300a4af5dc020b14565f4fa98c3aef34c96f2fc Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 18:20:39 -0600 Subject: [PATCH 43/61] Remember to include permissions for add tags to resource --- src/stack_mitosis/policy.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 80fdb7c..1a4ff89 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -57,6 +57,7 @@ (for [[op ops] (group-by :op all-permissions) :let [arns (distinct (map :arn ops))]] (cond (= op :CreateDBInstanceReadReplica) + ;; TODO account for AddTagsToResource on created instances (allow [op] (into [(make-arn "*" :type "og") (make-arn "*" :type "pg") From 7011eaa572c2d4b85ededdfb81cab33e2779dbe5 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 18:37:54 -0600 Subject: [PATCH 44/61] Expand policy/permissions to use a defmulti --- src/stack_mitosis/policy.clj | 51 +++++++++++++++++------------- test/stack_mitosis/policy_test.clj | 6 ++-- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 1a4ff89..5788c40 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -9,26 +9,32 @@ :or {account-id "*" region "*" type "db"}}] (str/join ":" ["arn:aws:rds" region account-id type db-id])) -(defn permissions [instances action] - (if-let [db-id (r/db-id action)] +(defmulti permissions + "Calculate permissions required for a given operation." + (fn [_ action] (get action :op))) + +(defmethod permissions :shell-command + [_ _] + []) + +(defmethod permissions :CreateDBInstanceReadReplica + [instances action] + ;; For create replica, use the ARN from the source database + ;; TODO: include target ARN + ;; TODO: include AddTagsToResource permissions + (let [source-id (r/source-id action) + source (lookup/by-id instances source-id)] + [{:op (:op action) + :arn (:DBInstanceArn source)}])) + +(defmethod permissions :default + [instances action] + (let [db-id (r/db-id action)] (if-let [instance (lookup/by-id instances db-id)] - {:op (:op action) - :arn (:DBInstanceArn instance)} - ;; FIXME: this is really gross - ;; For create replica, use the ARN from the source database - (let [source-id (r/source-id action) - source (lookup/by-id instances source-id)] - (if (and source-id source) - {:op (:op action) - :arn (:DBInstanceArn source)} - ;; fallback to wildcard if we don't recognize - ;; This probably should never happen, can we ensure this and drop this - ;; case OR should this be a nil arn case? - {:op (:op action) - :arn (make-arn db-id)}))) - ;; TODO handle ResourceName for ListTagsForResource - ;; TODO Exclude shell command, and include top level describe? - {:op (:op action)})) + [{:op (:op action) + :arn (:DBInstanceArn instance)}] + ;; TODO handle ResourceName for ListTagsForResource + [{:op (:op action)}]))) ;; TODO possibly generate optional statement identifier? ;; TODO simplify action/resource to singular if only one value? @@ -51,9 +57,10 @@ ;; these were listed as warnings in the policy editor so leaving wildcard for now (defn generate [instances operations] (let [all-permissions - (map permissions - (reductions predict/predict instances operations) - operations)] + (flatten + (map permissions + (reductions predict/predict instances operations) + operations))] (for [[op ops] (group-by :op all-permissions) :let [arns (distinct (map :arn ops))]] (cond (= op :CreateDBInstanceReadReplica) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 65e6529..ed28f52 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -29,13 +29,13 @@ (deftest permissions (let [instance {:DBInstanceIdentifier "foo" :DBInstanceArn (fake-arn "foo")}] - (is (= {:op :DeleteDBInstance :arn (fake-arn "foo")} + (is (= [{:op :DeleteDBInstance :arn (fake-arn "foo")}] (sut/permissions [instance] (op/delete "foo"))) "only operation if instance is not found") - (is (= {:op :DeleteDBInstance :arn (fake-arn "foo")} + (is (= [{:op :DeleteDBInstance :arn (fake-arn "foo")}] (sut/permissions [instance] (op/delete "foo"))) "operation and arn if instance is found") - (is (= {:op :DescribeDBInstances} + (is (= [{:op :DescribeDBInstances}] (sut/permissions [] (op/describe))) "operation only if no database identifier in request"))) From 66640765fe921ba5c4ee0e71d2c2e4b109f46de9 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 18:39:49 -0600 Subject: [PATCH 45/61] Clarify intended result from a call to permissions --- src/stack_mitosis/policy.clj | 4 +++- src/stack_mitosis/predict.clj | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 5788c40..d33df86 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -10,7 +10,9 @@ (str/join ":" ["arn:aws:rds" region account-id type db-id])) (defmulti permissions - "Calculate permissions required for a given operation." + "Calculate permissions required for a given operation. + + (permissions [instances op]) => [(allow ops arns) ...]" (fn [_ action] (get action :op))) (defmethod permissions :shell-command diff --git a/src/stack_mitosis/predict.clj b/src/stack_mitosis/predict.clj index d8e1771..db1a50c 100644 --- a/src/stack_mitosis/predict.clj +++ b/src/stack_mitosis/predict.clj @@ -27,7 +27,7 @@ (defmulti predict "Predict contents of instances db after applying operation to instances. - (predict [instances op] _) => instances' + (predict [instances op]) => instances' " (fn [_ op] (get op :op))) From 5e38cc14ecb2deb0b16319f877ab6494d97a46fe Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 18:49:40 -0600 Subject: [PATCH 46/61] Include permissions of target and source + add tags for create replica --- src/stack_mitosis/policy.clj | 13 +++++++------ test/stack_mitosis/policy_test.clj | 8 ++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index d33df86..8279912 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -12,7 +12,7 @@ (defmulti permissions "Calculate permissions required for a given operation. - (permissions [instances op]) => [(allow ops arns) ...]" + (permissions [instances action]) => [{:op _ :arn _} ...]" (fn [_ action] (get action :op))) (defmethod permissions :shell-command @@ -22,12 +22,13 @@ (defmethod permissions :CreateDBInstanceReadReplica [instances action] ;; For create replica, use the ARN from the source database - ;; TODO: include target ARN - ;; TODO: include AddTagsToResource permissions (let [source-id (r/source-id action) - source (lookup/by-id instances source-id)] - [{:op (:op action) - :arn (:DBInstanceArn source)}])) + db-id (r/db-id action) + source-arn (:DBInstanceArn (lookup/by-id instances source-id)) + target-arn (:DBInstanceArn (lookup/by-id (predict/predict instances action) db-id))] + [{:op (:op action) :arn source-arn} + {:op (:op action) :arn target-arn} + {:op :AddTagsToResource :arn target-arn}])) (defmethod permissions :default [instances action] diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index ed28f52..c51790e 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -44,7 +44,9 @@ (into ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" "arn:aws:rds:*:*:subgrp:*"] - (mapv fake-arn ["production" "temp-staging"]))) + (mapv fake-arn ["production" "temp-staging" "temp-staging-replica"]))) + (sut/allow [:AddTagsToResource] + (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] @@ -65,7 +67,9 @@ (into ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" "arn:aws:rds:*:*:subgrp:*"] - (mapv fake-arn ["production" "temp-staging"]))) + (mapv fake-arn ["production" "temp-staging" "temp-staging-replica"]))) + (sut/allow [:AddTagsToResource] + (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] From ddc0e81bedcfc903957c1c8728d8431cdb78be67 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 18:56:04 -0600 Subject: [PATCH 47/61] Include recommended secgrp and pg for modifydbinstance --- src/stack_mitosis/policy.clj | 4 +++- test/stack_mitosis/policy_test.clj | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 8279912..6a367fd 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -77,7 +77,9 @@ ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot (allow [op :RebootDBInstance] (into [(make-arn "*" :type "og") - (make-arn "*" :type "subgrp")] + (make-arn "*" :type "pg") + (make-arn "*" :type "subgrp") + (make-arn "*" :type "secgrp")] arns)) :else (allow [op] arns))))) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index c51790e..ba42144 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -51,7 +51,9 @@ [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] (into ["arn:aws:rds:*:*:og:*" - "arn:aws:rds:*:*:subgrp:*"] + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*" + "arn:aws:rds:*:*:secgrp:*"] (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"]))) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))] @@ -74,7 +76,9 @@ [(fake-arn "temp-staging")]) (sut/allow [:ModifyDBInstance :RebootDBInstance] (into ["arn:aws:rds:*:*:og:*" - "arn:aws:rds:*:*:subgrp:*"] + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*" + "arn:aws:rds:*:*:secgrp:*"] (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"]))) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} From 7e9ec63ade72473a1f8c9d44b311e5795bdae643 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:02:13 -0600 Subject: [PATCH 48/61] Clarify policy is not working for create/destroy of example --- src/stack_mitosis/example_environment.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stack_mitosis/example_environment.clj b/src/stack_mitosis/example_environment.clj index 4fa37d9..19c46ac 100644 --- a/src/stack_mitosis/example_environment.clj +++ b/src/stack_mitosis/example_environment.clj @@ -55,5 +55,7 @@ (plan/delete-tree state "mitosis-prod")))) (comment + ;; Add fake arn to create template? (policy/generate [] (create template)) - (policy/generate [] (destroy))) + ;; Fix this to actual return a functioning delete policy? + (policy/generate (predict/state [] (create template)) (destroy))) From 65514b9e1ecc58896e3b71c35f5b3c5ee83b53aa Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:03:50 -0600 Subject: [PATCH 49/61] Create example policy is missing delete and the various og,pg,etc --- src/stack_mitosis/policy.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 6a367fd..2566c7b 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -47,7 +47,8 @@ :Resource resources}) (defn create-example [] - (allow [:CreateDBInstance :AddTagsToResource] + ;; TODO handle og, pg, subgrp, secgrp permissions? + (allow [:CreateDBInstance :AddTagsToResource :DeleteDBInstance] [(make-arn "mitosis-*")])) (defn globals [] From b9369e00dea12b2fb3d1ced3d85792392abd9617 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:11:38 -0600 Subject: [PATCH 50/61] Move custom policy generation for create-replica to permissions defmulti --- src/stack_mitosis/policy.clj | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index 2566c7b..a918a08 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -26,8 +26,14 @@ db-id (r/db-id action) source-arn (:DBInstanceArn (lookup/by-id instances source-id)) target-arn (:DBInstanceArn (lookup/by-id (predict/predict instances action) db-id))] - [{:op (:op action) :arn source-arn} - {:op (:op action) :arn target-arn} + [{:op (:op action) + ;; TODO: can these permissions be more specific instead of wildcard? + ;; a) re-use the base ARN (ie region:account-id) from source + ;; b) generate named ARNs for each subtype used in source? + :arn (into [(make-arn "*" :type "og") + (make-arn "*" :type "pg") + (make-arn "*" :type "subgrp")] + [source-arn target-arn])} {:op :AddTagsToResource :arn target-arn}])) (defmethod permissions :default @@ -66,24 +72,18 @@ (reductions predict/predict instances operations) operations))] (for [[op ops] (group-by :op all-permissions) - :let [arns (distinct (map :arn ops))]] - (cond (= op :CreateDBInstanceReadReplica) - ;; TODO account for AddTagsToResource on created instances - (allow [op] - (into [(make-arn "*" :type "og") - (make-arn "*" :type "pg") - (make-arn "*" :type "subgrp")] - arns)) - (= op :ModifyDBInstance) - ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot - (allow [op :RebootDBInstance] - (into [(make-arn "*" :type "og") - (make-arn "*" :type "pg") - (make-arn "*" :type "subgrp") - (make-arn "*" :type "secgrp")] - arns)) - :else - (allow [op] arns))))) + :let [arns (distinct (flatten (map :arn ops)))]] + (cond (= op :ModifyDBInstance) + ;; TODO: move this to permissions defmulti + ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot + (allow [op :RebootDBInstance] + (into [(make-arn "*" :type "og") + (make-arn "*" :type "pg") + (make-arn "*" :type "subgrp") + (make-arn "*" :type "secgrp")] + arns)) + :else + (allow [op] arns))))) (defn policy [statements] {:Version "2012-10-17" :Statement statements}) From e1cb2afaefd2b3073aef4c4b5028565b72c37b5a Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:30:20 -0600 Subject: [PATCH 51/61] Add modify to permissions defmulti and clarify new-id vs reboot arns --- src/stack_mitosis/policy.clj | 44 ++++++++++++++++++------------ test/stack_mitosis/policy_test.clj | 24 ++++++++++------ 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/stack_mitosis/policy.clj b/src/stack_mitosis/policy.clj index a918a08..56d1fdf 100644 --- a/src/stack_mitosis/policy.clj +++ b/src/stack_mitosis/policy.clj @@ -36,6 +36,28 @@ [source-arn target-arn])} {:op :AddTagsToResource :arn target-arn}])) +(defmethod permissions :ModifyDBInstance + [instances action] + (let [db-id (r/db-id action) + arn (:DBInstanceArn (lookup/by-id instances db-id))] + (keep identity + [{:op (:op action) + ;; TODO: see above about tightening permissions + :arn [(make-arn "*" :type "og") + (make-arn "*" :type "pg") + (make-arn "*" :type "secgrp") + (make-arn "*" :type "subgrp") + arn]} + ;; If renaming we need the new arn too + (when-let [new-arn + (:DBInstanceArn (lookup/by-id (predict/predict instances action) + (r/new-id action)))] + {:op (:op action) + :arn new-arn}) + ;; RebootDBInstance is necessary for :ApplyImmediately true + {:op :RebootDBInstance + :arn arn}]))) + (defmethod permissions :default [instances action] (let [db-id (r/db-id action)] @@ -67,23 +89,11 @@ ;; these were listed as warnings in the policy editor so leaving wildcard for now (defn generate [instances operations] (let [all-permissions - (flatten - (map permissions - (reductions predict/predict instances operations) - operations))] - (for [[op ops] (group-by :op all-permissions) - :let [arns (distinct (flatten (map :arn ops)))]] - (cond (= op :ModifyDBInstance) - ;; TODO: move this to permissions defmulti - ;; Give RebootInstance if apply ModifyDBInstance so that ApplyImmediately can reboot - (allow [op :RebootDBInstance] - (into [(make-arn "*" :type "og") - (make-arn "*" :type "pg") - (make-arn "*" :type "subgrp") - (make-arn "*" :type "secgrp")] - arns)) - :else - (allow [op] arns))))) + (mapcat permissions + (reductions predict/predict instances operations) + operations)] + (for [[op ops] (group-by :op all-permissions)] + (allow [op] (distinct (flatten (map :arn ops))))))) (defn policy [statements] {:Version "2012-10-17" :Statement statements}) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index ba42144..f0f6aa5 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -49,12 +49,16 @@ (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) - (sut/allow [:ModifyDBInstance :RebootDBInstance] + (sut/allow [:ModifyDBInstance] (into ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" - "arn:aws:rds:*:*:subgrp:*" - "arn:aws:rds:*:*:secgrp:*"] - (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"]))) + "arn:aws:rds:*:*:secgrp:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica" + "staging-replica" "old-staging-replica" + "staging" "old-staging"]))) + (sut/allow [:RebootDBInstance] + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))] (sut/generate (example-instances) @@ -74,12 +78,16 @@ (mapv fake-arn ["temp-staging" "temp-staging-replica"])) (sut/allow [:PromoteReadReplica] [(fake-arn "temp-staging")]) - (sut/allow [:ModifyDBInstance :RebootDBInstance] + (sut/allow [:ModifyDBInstance] (into ["arn:aws:rds:*:*:og:*" "arn:aws:rds:*:*:pg:*" - "arn:aws:rds:*:*:subgrp:*" - "arn:aws:rds:*:*:secgrp:*"] - (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"]))) + "arn:aws:rds:*:*:secgrp:*" + "arn:aws:rds:*:*:subgrp:*"] + (mapv fake-arn ["temp-staging" "temp-staging-replica" + "staging-replica" "old-staging-replica" + "staging" "old-staging"]))) + (sut/allow [:RebootDBInstance] + (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) (sut/allow [:DeleteDBInstance] (mapv fake-arn ["old-staging-replica" "old-staging"]))]} (sut/from-plan (example-instances) From 97702014df130a90fa6c3d0d045e42f2240e5a2e Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:36:11 -0600 Subject: [PATCH 52/61] Add permissions cases for create-replica and modify/rename --- test/stack_mitosis/policy_test.clj | 31 +++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index f0f6aa5..35aed0c 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -37,7 +37,36 @@ "operation and arn if instance is found") (is (= [{:op :DescribeDBInstances}] (sut/permissions [] (op/describe))) - "operation only if no database identifier in request"))) + "operation only if no database identifier in request") + (is (= [{:op :CreateDBInstanceReadReplica + :arn ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:subgrp:*" + "arn:aws:rds:us-east-1:1234567:db:foo" + "arn:aws:rds:us-east-1:1234567:db:bar"]} + {:op :AddTagsToResource + :arn "arn:aws:rds:us-east-1:1234567:db:bar"}] + (sut/permissions [instance] (op/create-replica "foo" "bar")))) + (is (= [{:op :ModifyDBInstance + :arn ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:secgrp:*" + "arn:aws:rds:*:*:subgrp:*" + "arn:aws:rds:us-east-1:1234567:db:foo"]} + {:op :ModifyDBInstance + :arn "arn:aws:rds:us-east-1:1234567:db:bar"} + {:op :RebootDBInstance + :arn "arn:aws:rds:us-east-1:1234567:db:foo"}] + (sut/permissions [instance] (op/rename "foo" "bar")))) + (is (= [{:op :ModifyDBInstance + :arn ["arn:aws:rds:*:*:og:*" + "arn:aws:rds:*:*:pg:*" + "arn:aws:rds:*:*:secgrp:*" + "arn:aws:rds:*:*:subgrp:*" + "arn:aws:rds:us-east-1:1234567:db:foo"]} + {:op :RebootDBInstance + :arn "arn:aws:rds:us-east-1:1234567:db:foo"}] + (sut/permissions [instance] (op/modify "foo" {})))))) (deftest generate (is (= [(sut/allow [:CreateDBInstanceReadReplica] From 4f9a73d910cee93fc8ad5f4211f716868d609657 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:37:15 -0600 Subject: [PATCH 53/61] deprecate specs for generate as they are inside the from-plan test --- test/stack_mitosis/policy_test.clj | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/test/stack_mitosis/policy_test.clj b/test/stack_mitosis/policy_test.clj index 35aed0c..819a745 100644 --- a/test/stack_mitosis/policy_test.clj +++ b/test/stack_mitosis/policy_test.clj @@ -68,31 +68,6 @@ :arn "arn:aws:rds:us-east-1:1234567:db:foo"}] (sut/permissions [instance] (op/modify "foo" {})))))) -(deftest generate - (is (= [(sut/allow [:CreateDBInstanceReadReplica] - (into ["arn:aws:rds:*:*:og:*" - "arn:aws:rds:*:*:pg:*" - "arn:aws:rds:*:*:subgrp:*"] - (mapv fake-arn ["production" "temp-staging" "temp-staging-replica"]))) - (sut/allow [:AddTagsToResource] - (mapv fake-arn ["temp-staging" "temp-staging-replica"])) - (sut/allow [:PromoteReadReplica] - [(fake-arn "temp-staging")]) - (sut/allow [:ModifyDBInstance] - (into ["arn:aws:rds:*:*:og:*" - "arn:aws:rds:*:*:pg:*" - "arn:aws:rds:*:*:secgrp:*" - "arn:aws:rds:*:*:subgrp:*"] - (mapv fake-arn ["temp-staging" "temp-staging-replica" - "staging-replica" "old-staging-replica" - "staging" "old-staging"]))) - (sut/allow [:RebootDBInstance] - (mapv fake-arn ["temp-staging" "temp-staging-replica" "staging-replica" "staging"])) - (sut/allow [:DeleteDBInstance] - (mapv fake-arn ["old-staging-replica" "old-staging"]))] - (sut/generate (example-instances) - (plan/replace-tree (example-instances) "production" "staging"))))) - (deftest from-plan (is (= {:Version "2012-10-17" :Statement From 7a9e097ee158711a697108b33e21c5cbc65bf892 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 19:47:40 -0600 Subject: [PATCH 54/61] Update README permissions to match revisions --- README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 163a4b6..7b580b4 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Note that for many cases, even if the clone process is interrupted, the flight p ## IAM Policy Generation -Stack-mitosis can also generate an IAM policy for an automated user to update a particular environment. The policy uses the database names from the planned changeset to calculate these minimal permissions. While they have been elided from the example below, the ARNs are locked to the account & region. +Stack-mitosis can also generate an IAM policy for an automated user to update a particular environment. The policy uses the database names from the planned changeset to calculate these minimal permissions. While they have been elided from the example below, the ARNs are locked to the specific account & region used. ``` $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-policy @@ -129,17 +129,33 @@ $ clj -m stack-mitosis.cli --source mitosis-prod --target mitosis-demo --iam-pol "Resource": ["arn:aws:rds:*:*:og:*", "arn:aws:rds:*:*:pg:*", "arn:aws:rds:*:*:subgrp:*", + "arn:aws:rds:*:*:db:mitosis-prod", "arn:aws:rds:*:*:db:temp-mitosis-demo", "arn:aws:rds:*:*:db:temp-mitosis-demo-replica"]}, + {"Effect":"Allow", + "Action":["rds:AddTagsToResource"], + "Resource": + ["arn:aws:rds:*:*:db:temp-mitosis-demo", + "arn:aws:rds:*:*:db:temp-mitosis-demo-replica"]}, {"Effect":"Allow", "Action":["rds:PromoteReadReplica"], "Resource": ["arn:aws:rds:*:*:db:temp-mitosis-demo"]}, {"Effect":"Allow", - "Action":["rds:ModifyDBInstance", "rds:RebootDBInstance"], + "Action":["rds:ModifyDBInstance"], "Resource": - ["arn:aws:rds:*:*:og:*", "arn:aws:rds:*:*:subgrp:*", + ["arn:aws:rds:*:*:og:*", "arn:aws:rds:*:*:pg:*", + "arn:aws:rds:*:*:secgrp:*", "arn:aws:rds:*:*:subgrp:*", "arn:aws:rds:*:*:db:temp-mitosis-demo", + "arn:aws:rds:*:*:db:temp-mitosis-demo-replica", + "arn:aws:rds:*:*:db:mitosis-demo-replica", + "arn:aws:rds:*:*:db:old-mitosis-demo-replica", + "arn:aws:rds:*:*:db:mitosis-demo", + "arn:aws:rds:*:*:db:old-mitosis-demo"]}, + {"Effect":"Allow", + "Action":["rds:RebootDBInstance"], + "Resource": + ["arn:aws:rds:*:*:db:temp-mitosis-demo", "arn:aws:rds:*:*:db:temp-mitosis-demo-replica", "arn:aws:rds:*:*:db:mitosis-demo-replica", "arn:aws:rds:*:*:db:mitosis-demo"]}, From 97fe3f18efd97d615a84d2316b164763c5f699ce Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 22:15:38 -0600 Subject: [PATCH 55/61] Remove warnings about -M from tools.deps --- bin/ci | 2 +- bin/kaocha | 2 +- deps.edn | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ci b/bin/ci index 8f24b9b..cbdb5f9 100755 --- a/bin/ci +++ b/bin/ci @@ -8,4 +8,4 @@ bin/kaocha --plugin cloverage \ --plugin kaocha.plugin/junit-xml \ --junit-xml-file test-results/kaocha/results.xml -clojure -Aclj-kondo --lint src +clojure -Aclj-kondo -M --lint src diff --git a/bin/kaocha b/bin/kaocha index faf68eb..055a31a 100755 --- a/bin/kaocha +++ b/bin/kaocha @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clojure -A:kaocha -m kaocha.runner --config-file test/tests.edn "$@" +clojure -Mkaocha -m kaocha.runner --config-file test/tests.edn "$@" diff --git a/deps.edn b/deps.edn index 0da34c5..81f47c1 100644 --- a/deps.edn +++ b/deps.edn @@ -23,7 +23,7 @@ } :paths ["src" "resources"] :aliases - {;; clj -A:kaocha -m kaocha.runner --config-file test/tests.edn + {;; clj -Mkaocha -m kaocha.runner --config-file test/tests.edn :kaocha {:extra-paths ["test"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.0.732"} lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} From dcb7ee976a994cfe3d5ec36517135f23d5cdccbc Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 22:18:27 -0600 Subject: [PATCH 56/61] Inline -m kaocha.runner into alias definition --- bin/kaocha | 2 +- deps.edn | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/kaocha b/bin/kaocha index 055a31a..af779b7 100755 --- a/bin/kaocha +++ b/bin/kaocha @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clojure -Mkaocha -m kaocha.runner --config-file test/tests.edn "$@" +clojure -Mkaocha --config-file test/tests.edn "$@" diff --git a/deps.edn b/deps.edn index 81f47c1..7df89d5 100644 --- a/deps.edn +++ b/deps.edn @@ -23,11 +23,12 @@ } :paths ["src" "resources"] :aliases - {;; clj -Mkaocha -m kaocha.runner --config-file test/tests.edn + {;; clj -Mkaocha --config-file test/tests.edn :kaocha {:extra-paths ["test"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.0.732"} lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} - lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}}} + lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}} + :main-opts ["-m" "kaocha.runner"]} ;; clj -Mclj-kondo --lint src :clj-kondo From 94cdfa5d5e2832814db97425e5af9951a6416506 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Mon, 18 Jan 2021 22:31:34 -0600 Subject: [PATCH 57/61] Credentials are optional --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b580b4..6d1246d 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Hopefully in the future this can be parsed directly from the `AWS_CONFIG` file. clj -m stack-mitosis.cli \ --source mitosis-production --target mitosis-staging \ --restart "./restart-service.sh" - --credentials resources/role.edn + [--credentials resources/role.edn] [--plan] [--iam-policy] From e326ccfcbb0a9e2e78ab297cbea6ffcfdb6900a3 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 22 Jan 2021 15:11:19 -0600 Subject: [PATCH 58/61] Can't use main-opts by default as it needs to include for cider boot --- bin/kaocha | 2 +- deps.edn | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/kaocha b/bin/kaocha index af779b7..055a31a 100755 --- a/bin/kaocha +++ b/bin/kaocha @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clojure -Mkaocha --config-file test/tests.edn "$@" +clojure -Mkaocha -m kaocha.runner --config-file test/tests.edn "$@" diff --git a/deps.edn b/deps.edn index 7df89d5..81f47c1 100644 --- a/deps.edn +++ b/deps.edn @@ -23,12 +23,11 @@ } :paths ["src" "resources"] :aliases - {;; clj -Mkaocha --config-file test/tests.edn + {;; clj -Mkaocha -m kaocha.runner --config-file test/tests.edn :kaocha {:extra-paths ["test"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.0.732"} lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} - lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}} - :main-opts ["-m" "kaocha.runner"]} + lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}}} ;; clj -Mclj-kondo --lint src :clj-kondo From f20a144c920cc514446d3c275bd15fddad505f09 Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 22 Jan 2021 18:57:34 -0600 Subject: [PATCH 59/61] Add the basics of a changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0667818 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## Unreleased + +### Added + +## [0.5.0] + + - Added `--iam-policy` option for generating a IAM policy for a user or role to clone a replica with a minimal set of permissions. + - Updated dependencies + From 4ee85424ead03f5d376d44f53796ab5e062d015b Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 22 Jan 2021 19:09:12 -0600 Subject: [PATCH 60/61] Narrow post-condition for databases to account for no databases --- src/stack_mitosis/interpreter.clj | 5 ++--- test/stack_mitosis/interpreter_test.clj | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index 9add41e..d05c8b7 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -36,9 +36,8 @@ (defn databases [rds] - ;; FIXME: if account has *no* databases this fails too, need a more nuanced - ;; postcondition here - ;; {:post [(seq %)]} + ;; exclude nil, but fresh account might return empty list + {:post [(sequential? %)]} (:DBInstances (invoke-logged! rds (op/describe)))) ;; TODO: verify that "old-" database copies do not exist before running diff --git a/test/stack_mitosis/interpreter_test.clj b/test/stack_mitosis/interpreter_test.clj index e2ef6b8..b49b7d5 100644 --- a/test/stack_mitosis/interpreter_test.clj +++ b/test/stack_mitosis/interpreter_test.clj @@ -23,9 +23,11 @@ (with-redefs [aws/invoke (mock-invoke [{:DBInstanceIdentifier "a"}])] (is (= [{:DBInstanceIdentifier "a"}] (sut/databases identity)))) - ;; (with-redefs [aws/invoke (mock-invoke [])] - ;; (is (thrown-with-msg? java.lang.AssertionError #"Assert failed" - ;; (sut/databases identity)))) + (with-redefs [aws/invoke (mock-invoke [])] + (is (= [] (sut/databases identity)))) + (with-redefs [aws/invoke (fn [& _] {})] + (is (thrown-with-msg? java.lang.AssertionError #"Assert failed" + (sut/databases identity)))) ) (deftest evaluate-plan From e7638faf54ab5db2e1417bbb584714c1705aadbd Mon Sep 17 00:00:00 2001 From: Charles Comstock Date: Fri, 22 Jan 2021 19:14:36 -0600 Subject: [PATCH 61/61] Remove spurious extra comment --- src/stack_mitosis/interpreter.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/stack_mitosis/interpreter.clj b/src/stack_mitosis/interpreter.clj index d05c8b7..6ac7b13 100644 --- a/src/stack_mitosis/interpreter.clj +++ b/src/stack_mitosis/interpreter.clj @@ -182,7 +182,6 @@ (list-tags rds instances "mitosis-demo") - (lookup/by-id (databases rds) "mitosis-prod") (let [instances (databases rds)] (clojure.data/diff (lookup/by-id instances "mitosis-prod")