Support adding & removing master keys, fixes #26

README.rst

@@ -71,7 +71,7 @@ Your AWS credentials must be present in `~/.aws/credentials`. sops uses boto3.
 
 .. code::
 
-	$ cat ~/.aws/credentials 
+	$ cat ~/.aws/credentials
 	[default]
 	aws_access_key_id = AKI.....
 	aws_secret_access_key = mw......
@@ -156,15 +156,33 @@ steps, apart from the actual editing, are transparent to the user.
 Adding and removing keys
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-When creating a new files, `sops` uses the PGP and KMS defined in the command
+When creating new files, `sops` uses the PGP and KMS defined in the command
 line arguments `--kms` and `--pgp`, or from the environment variables
 `SOPS_KMS_ARN` and `SOPS_PGP_FP`. That information is stored in the file under
-the `sops` section. When editing a file, it is trivial to add or remove keys:
-invoke `sops` with the flag **-s** to display the master keys while editing, and
-add or remove kms or pgp keys under the sops section.
+the `sops` section, such that decrypting files does not require providing those
+parameters again.
 
-For example, to add a KMS master key to a file, we would add the following
-entry:
+Master PGP and KMS keys can be added and removed from a `sops` file in one of
+two ways: by using command line flag, or by editing the file directly.
+
+Command line flag `--add-kms`, `--add-pgp`, `--rm-kms` and `--rm-pgp` can be
+used to add and remove keys from a file. These flags use the comma separated
+syntax as the `--kms` and `--pgp` arguments when creating new files.
+
+.. code:: bash
+
+	# add a new pgp key to the file while editing
+	$ sops --add-pgp 85D77543B3D624B63CEA9E6DBC17301B491B3F21 example.yaml
+
+	# remove a pgp key from the file while editing
+	$ sops --rm-pgp 85D77543B3D624B63CEA9E6DBC17301B491B3F21 example.yaml
+
+Alternatively, invoking `sops` with the flag **-s** will display the master keys
+while editing. This method can be used to add or remove kms or pgp keys under the
+sops section.
+
+For example, to add a KMS master key to a file, add the following entry while
+editing:
 
 .. code:: yaml
 
@@ -323,10 +341,10 @@ In-place encryption/decryption also works on binary files.
 	$ sha512sum /tmp/somerandom
 	9589bb20280e9d381f7a192000498c994e921b3cdb11d2ef5a986578dc2239a340b25ef30691bac72bdb14028270828dad7e8bd31e274af9828c40d216e60cbe /tmp/somerandom
 
-	$ sops -e -i /tmp/somerandom 
+	$ sops -e -i /tmp/somerandom
 	please wait while a data encryption key is being generated and stored securely
 
-	$ sops -d -i /tmp/somerandom 
+	$ sops -d -i /tmp/somerandom
 
 	$ sha512sum /tmp/somerandom
 	9589bb20280e9d381f7a192000498c994e921b3cdb11d2ef5a986578dc2239a340b25ef30691bac72bdb14028270828dad7e8bd31e274af9828c40d216e60cbe /tmp/somerandom
@@ -550,7 +568,7 @@ systems. Not unlike many other organizations that operate sufficiently complex
 automation, we found this to be a hard problem with a number of prerequisites:
 
 1. Secrets must be stored in YAML files for easy integration into hiera
-	    
+
 2. Secrets must be stored in GIT, and when a new CloudFormation stack is
    built, the current HEAD is pinned to the stack. (This allows secrets to
    be changed in GIT without impacting the current stack that may
@@ -611,7 +629,7 @@ The security of the data stored using sops is as strong as the weakest
 cryptographic mechanism. Values are encrypted using AES256_GCM which is the
 strongest symetric encryption algorithm known today. Data keys are encrypted
 in either KMS, which also uses AES256_GCM, or PGP which uses either RSA or
-ECDSA keys. 
+ECDSA keys.
 
 Going from the most likely to the least likely, the threats are as follows:
 

example.yaml

@@ -22,8 +22,8 @@ this:
             nested:
                 value: ENC[AES256_GCM,data:TzfuYK7BOwJlmlxydTmtPKlfIvSxoaIMiqrt,iv:q+YKcwFOImx8VX4Ti1ECjBWLz32gtkxzBDq12uOsmvk=,tag:GXz+BkXKbblwfEc/dZLgzg==,type:str]
 sops:
-    mac: ENC[AES256_GCM,data:gj86GUajphvwhmUS5Z+1nK+yxqleOTOSj71WVl48K4P6R3o/K9rE+doLFl1z2xAw0TxSFnNAR5L/fpvR/3uZCQRfXb9yLTZNOAOtc0JYh3B9Epsa78uznGXnQwuuiX4rXprsrLZ0d07cEuCkhFJww7v27C6zlP8MpthpYTWMXfM=,iv:NlJqEdMb5Y6V54hbzTAwDZ067xFUvxobX05Xa2PuwZs=,tag:Wk9061XNFX8Xpp7duxE+tg==,type:str]
-    version: 0.90000000000000002
+    mac: ENC[AES256_GCM,data:svdUk+7ahpTaWBUdXqgEy5+K6uMm210Jrm3fPvsx2VaCiONv5QIDQbUipRFOpGKubKfhJk9XPcr+4MaE6oUxW8snxkN0p1BMAqpZhQ31xdwila318TckJltgPQQfAl59CNsLf1EgweBTWhvZL5sWGOEMXfMAHuHWzN4v1CmAU3w=,iv:vyFzhy4LwFQ6pNJulze9BBt9sfIfLwhhmlrIAroO+JE=,tag:AihY+oO3J7mon003SHYrfQ==,type:str]
+    version: 1.0
     kms:
     -   created_at: '2015-10-25T12:52:27Z'
         enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAykG26ZbESEOy9KtoQCARCAO4cK6asAUiZBDmIgWk98BTvxUkvUmXYF2dxkP+Pr6F+r2oO7jhyB/FqyV5WAHCmdljs6DzBvB0FSKgdL
@@ -46,26 +46,26 @@ sops:
             =YXAh
             -----END PGP MESSAGE-----
     -   fp: 85D77543B3D624B63CEA9E6DBC17301B491B3F21
-        created_at: '2015-10-25T12:52:27Z'
+        created_at: '2015-11-24T14:19:08Z'
         enc: |
             -----BEGIN PGP MESSAGE-----
             Version: GnuPG v1
 
-            hQIMA0t4uZHfl9qgARAAjQrSvwlR66cwzHM9HkzvKcfXxy71mBwCjVYR/dazz+Bg
-            WWyAOsaZ9lPnR1K7ANaiKPtsF6+drEWokUsHdDc4waYMYX4Ha7kjXr9CfbrlhM6Y
-            gI0PrRI17Un85HjeHQYp/Vndw8c4ZKV0tOKKGGiWA+GAXiM+fDrSJBSt8wy6SJtY
-            t+T1Wl/VEmyvLGM9VGK+MI6Htyy2FCH0kWQ8wBA9iJv59MvBTR2s3FhdGovosk7k
-            1PIRV3M5A7yjOMgHkvdC149BfqBLGcUYM+1xwXLJOGX04eCD8Y/XT41NRMH44rfq
-            ev6LEVJlqi50DhagkBdPp/FTpFLhhTRfkIISz236XPzZC/zDXLBSQt5DbwqymsVy
-            WTavSqDmOQFX2Zir+nlZcKwwCsY3funZm9jVefuQmphN+yXRM2VkK67goH5ZMeGI
-            uFU3xzuhyibYH/YgJT8g0fTYeiaKzcIicwN4klkhpnckrHSa8brMTYK1eZ+olB29
-            XnbCM6unDnaKDJGarD9reDQt91lRsENUkj83mOrHdtGQigV2fbv3+KtfXvW4Q8Fq
-            lcq5oLRTch6RAcSbQfTL9fK6AjPbWZX97JjAZbeSY+HI6YRGL+Iaf26sHbZQTcuS
-            rrx0vX2rvkQSwdZ+ZfKC9az2/9hPjWkDLyhu5WE3KIq/SlsDl8pTLXzarGeEPN7S
-            XgHWXXf18MxU482uhGAysV50jpmnJXQk4SCM8QHZMqgKIDmJD4E6hq6WqEi1AR2C
-            8HwuI2CMQ9skRKtoQJUV1gdSXuLYWzfJKCv0nrLk6Ot94QQV9RsxMeKaqf2V47o=
-            =ca1h
+            hQIMA0t4uZHfl9qgAQ/+LGpXd7Vn5RYK52Kpp1FPqUHiu3Yt1XylFSXT4BzHSxa9
+            PR4sz83rDkeRwfitFf4ll1MB7zhCiekTUvBWUkdSZHLj9o+XX/7E3OvZ+B7HFG90
+            +MLb7h2Cp8KRgEHBppxtbkNOmzgbDZ0s9vqxm+JU3IJzqmq1Roc2P4FVYtUgQxlY
+            spYxWzhizLxO/TJlERA8YV921vbKHhIP4I4KoWUk8gYR31b1kakrRcKI8/SDmbe8
+            6TlaPIZDxuSzY+toaesClJSkv7pMCByzyVgXbdgtHMReU8y4MSEjvhBcxrsZ3X8m
+            awJpw45DuZl+xhPGgFik/ERHewxMnoRUjmxHgfWLVkR8uP+FHXjbMiIAk0wMekGC
+            UbzbpESh5zLa2zlCgPshgYnEJobua0BrC+x3pVV8RFRrkKJXL1NuZ1gX4oFHePuV
+            O9UiVrsi5wpL2+jkAKqs/buDeOH4piWFoD03NAXX1SbHOHg6W/ji5C5Hen1WloXN
+            +NRffmmujiPFM2FzZWKiWDZgP+VopA3IHFUYv/TeepZUa3ldsaYh8tr1UAZswvG2
+            lG5HGs7yMw1IPATWfpxhe0f2vzSKGc13Y+y0YEWqd6FH6ixo97XOMvVmRvl550o/
+            iDN8v5xHUG7tKTNAB5aU7IFldUjwUcCqky7e4twNRJkcid6oXWBpBcgbcJLPvVnS
+            XgHYdVzf3Rvb+Jb5UlNs33wH5j1EHp7MVEIs1Gp8fySlW8m7Ui6lagPfyqb0n3nD
+            OujQzMRAaTP2iWEAQICXW1gnnDDyg2o5+WnnVVzX+YVYxCTLq66wnvB2M/yAcuM=
+            =ulPs
             -----END PGP MESSAGE-----
-    lastmodified: '2015-10-26T18:15:24Z'
+    lastmodified: '2015-11-24T14:19:08Z'
     attention: This section contains key material that should only be modified with
         extra care. See `sops -h`.

sops/__init__.py

@@ -136,6 +136,21 @@ def main():
                            dest='show_master_keys',
                            help="display master encryption keys in the file "
                                 "during editing (off by default).")
+    argparser.add_argument('--add-kms', dest='add_kms',
+                           help="Add the given comma separated KMS ARNs to the"
+                                " list of master keys on an existing file.")
+    argparser.add_argument('--rm-kms', dest='rm_kms',
+                           help="Remove the given comma separated KMS ARNs "
+                                "from the list of master keys on an existing "
+                                "file.")
+    argparser.add_argument('--add-pgp', dest='add_pgp',
+                           help="Add the given comma separated PGP fingerprint"
+                                " to the list of master keys on an existing "
+                                "file.")
+    argparser.add_argument('--rm-pgp', dest='rm_pgp',
+                           help="Remove the given comma separated PGP "
+                                "fingerprint from the list of master keys on "
+                                "an existing file.")
     argparser.add_argument('--ignore-mac', action='store_true',
                            dest='ignore_mac',
                            help="ignore Message Authentication Code "
@@ -175,6 +190,11 @@ def main():
                                                     kms_arns=kms_arns,
                                                     pgp_fps=pgp_fps)
     if not existing_file:
+        if len(args.add_kms) > 0 or len(args.add_pgp) > 0 \
+           or len(args.rm_kms) > 0 or len(args.rm_pgp) > 0:
+            panic("cannot add or remove keys on non-existent files, use "
+                  "`--kms` and `--pgp` instead.", error_code=49)
+        # encrypt and decrypt modes are not available on non-existent files
         if (args.encrypt or args.decrypt):
             panic("cannot operate on non-existent file", error_code=100)
         else:
@@ -278,6 +298,8 @@ def main():
               error_code=200)
 
     tree = walk_and_encrypt(tree, key, stash=stash)
+    tree = add_new_master_keys(tree, args.add_kms, args.add_pgp)
+    tree = remove_master_keys(tree, args.rm_kms, args.rm_pgp)
     tree = update_master_keys(tree, key)
     os.remove(tmppath)
 
@@ -499,6 +521,76 @@ def check_master_keys(tree):
     return False
 
 
+def add_new_master_keys(tree, new_kms, new_pgp):
+    """ Add new master keys by creating a new tree and updating
+        the main tree with them
+    """
+    if new_kms and len(new_kms) > 0:
+        newtree = {}
+        newtree['sops'] = {}
+        newtree, throwaway = parse_kms_arn(newtree, new_kms)
+        if 'kms' in newtree['sops']:
+            for newentry in newtree['sops']['kms']:
+                if 'kms' not in tree['sops']:
+                    tree['sops']['kms'] = [newentry]
+                    continue
+                shouldadd = True
+                for entry in tree['sops']['kms']:
+                    if newentry['arn'] == entry['arn']:
+                        # arn already present, don't re-add it
+                        shouldadd = False
+                        break
+                if shouldadd:
+                    tree['sops']['kms'].append(newentry)
+    if new_pgp and len(new_pgp) > 0:
+        newtree = {}
+        newtree['sops'] = {}
+        newtree, throwaway = parse_pgp_fp(newtree, new_pgp)
+        if 'pgp' in newtree['sops']:
+            for newentry in newtree['sops']['pgp']:
+                if 'pgp' not in tree['sops']:
+                    tree['sops']['pgp'] = [newentry]
+                    continue
+                shouldadd = True
+                for entry in tree['sops']['pgp']:
+                    if newentry['fp'] == entry['fp']:
+                        # arn already present, don't re-add it
+                        shouldadd = False
+                        break
+                if shouldadd:
+                    tree['sops']['pgp'].append(newentry)
+    return tree
+
+
+def remove_master_keys(tree, rm_kms, rm_pgp):
+    """ remove master keys by creating a new tree and removing
+        the master keys present in the new tree from the old tree
+    """
+    if rm_kms and len(rm_kms) > 0:
+        newtree = {}
+        newtree['sops'] = {}
+        newtree, throwaway = parse_kms_arn(newtree, rm_kms)
+        if 'kms' in newtree['sops'] and 'kms' in tree['sops']:
+            for rmentry in newtree['sops']['kms']:
+                i = 0
+                for entry in tree['sops']['kms']:
+                    if rmentry['arn'] == entry['arn']:
+                        del tree['sops']['kms'][i]
+                    i += 1
+    if rm_pgp and len(rm_pgp) > 0:
+        newtree = {}
+        newtree['sops'] = {}
+        newtree, throwaway = parse_pgp_fp(newtree, rm_pgp)
+        if 'pgp' in newtree['sops'] and 'pgp' in tree['sops']:
+            for rmentry in newtree['sops']['pgp']:
+                i = 0
+                for entry in tree['sops']['pgp']:
+                    if rmentry['fp'] == entry['fp']:
+                        del tree['sops']['pgp'][i]
+                    i += 1
+    return tree
+
+
 def walk_and_decrypt(branch, key, aad=b'', stash=None, digest=None,
                      isRoot=True, ignoreMac=False):
     """Walk the branch recursively and decrypt leaves."""

tests/test_sops.py

@@ -197,6 +197,92 @@ def test_encrypt_decrypt(self):
         clearstr = sops.decrypt(sops.encrypt(origin, key, aad=aad), key, aad=aad)
         assert clearstr == origin
 
+    def test_add_kms_master_keys(self):
+        """ test adding a kms master key to an existing tree """
+        tree = {'sops': { 'kms': [ {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' } ] } }
+        newkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz'
+        assert len(tree['sops']['kms']) == 1
+        tree = sops.add_new_master_keys(tree, newkms, '')
+        assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e'
+        assert tree['sops']['kms'][1]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac'
+        assert tree['sops']['kms'][1]['role'] == 'arn:aws:iam::927034868273:role/sops-dev-xyz'
+
+    def test_add_pgp_master_keys_where_none_existed(self):
+        """ test adding a pgp master key to an existing tree
+            that does not have any pgp master key yet
+        """
+        tree = {'sops': { 'kms': [ {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' } ] } }
+        newpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F'
+        tree = sops.add_new_master_keys(tree, '', newpgp)
+        assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e'
+        assert tree['sops']['pgp'][0]['fp'] == 'E60892BB9BD89A69F759A1A0A3D652173B763E8F'
+
+    def test_add_pgp_master_keys(self):
+        """ test adding a pgp master key to an existing tree """
+        tree = {'sops': { 'pgp': [ {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' } ] } }
+        newpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F'
+        assert len(tree['sops']['pgp']) == 1
+        tree = sops.add_new_master_keys(tree, '', newpgp)
+        assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A'
+        assert tree['sops']['pgp'][1]['fp'] == 'E60892BB9BD89A69F759A1A0A3D652173B763E8F'
+
+    def test_add_kms_master_keys_where_none_existed(self):
+        """ test adding a kms master key to an existing tree
+            that does not have any kms master key yet
+        """
+        tree = {'sops': { 'pgp': [ {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' } ] } }
+        newkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz'
+        tree = sops.add_new_master_keys(tree, newkms, '')
+        assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A'
+        assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac'
+        assert tree['sops']['kms'][0]['role'] == 'arn:aws:iam::927034868273:role/sops-dev-xyz'
+
+    def test_rm_kms_master_keys(self):
+        """ test removing a kms master key to an existing tree """
+        tree = {'sops': { 'kms': [
+                    {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' },
+                    {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac' }
+                ] } }
+        rmkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz'
+        assert len(tree['sops']['kms']) == 2
+        tree = sops.remove_master_keys(tree, rmkms, '')
+        assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e'
+        assert len(tree['sops']['kms']) == 1
+
+    def test_rm_pgp_master_keys_where_none_existed(self):
+        """ test removing a pgp master key to an existing tree
+            that does not have any pgp master key yet
+        """
+        tree = {'sops': { 'kms': [ {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' } ] } }
+        rmpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F'
+        tree = sops.remove_master_keys(tree, '', rmpgp)
+        assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e'
+        assert len(tree['sops']['kms']) == 1
+        assert 'pgp' not in tree['sops']
+
+    def test_rm_pgp_master_keys(self):
+        """ test removing a pgp master key to an existing tree """
+        tree = {'sops': { 'pgp': [
+                    {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' },
+                    {'fp': 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' }
+                ] } }
+        rmpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F'
+        assert len(tree['sops']['pgp']) == 2
+        tree = sops.remove_master_keys(tree, '', rmpgp)
+        assert len(tree['sops']['pgp']) == 1
+        assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A'
+
+    def test_rm_kms_master_keys_where_none_existed(self):
+        """ test removing a kms master key to an existing tree
+            that does not have any kms master key yet
+        """
+        tree = {'sops': { 'pgp': [ {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' } ] } }
+        rmkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz'
+        tree = sops.remove_master_keys(tree, rmkms, '')
+        assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A'
+        assert len(tree['sops']['pgp']) == 1
+        assert 'kms' not in tree['sops']
+
     # Test keys management
     def test_get_key(self):
         """Test we obtain a 256 bits symetric key."""