remove dots from the start or end of a key (#1650)

* remove dots from the start or end of a key

* updating the comment text

* re-envisioned code per @pwnbus' suggestion

* addressing review comments

* adds logic to check for null keys

mq/plugins/remove_dots.py

@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+
+class message(object):
+
+    def __init__(self):
+        '''
+        takes an incoming message and checks for dots at the
+        start or end of the key and removes them
+        '''
+
+        self.registration = ['cloudtrail']
+        self.priority = 5
+
+    def onMessage(self, message, metadata):
+        def renameKeysToRemoveDots(message):
+            if isinstance(message, dict):
+                message_keys = list(message.keys())
+                for key in message_keys:
+                    if len(key) > 0 and (key[0] == '.' or key[-1] == '.'):
+                        new_key = key.replace(".", "")
+                        if new_key != key:
+                            message[new_key] = message.pop(key)
+                    if isinstance(message.get(key), (dict, list)):
+                        message[key] = renameKeysToRemoveDots(message[key])
+            elif isinstance(message, list):
+                for item in message:
+                    item = renameKeysToRemoveDots(item)
+            return message
+
+        message = renameKeysToRemoveDots(message)
+        return (message, metadata)

tests/mq/plugins/test_remove_dots.py

@@ -0,0 +1,103 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+from mq.plugins.remove_dots import message
+
+
+class TestRemoveDotsPlugin():
+    def setup(self):
+        self.plugin = message()
+
+    def test_first_dot_removal(self):
+        msg = {
+            'source': 'cloudtrail',
+            'details': {
+                'responseelements': {
+                    'fileuploadurls': {
+                        '.ds_store': 'astringvalue',
+                        'js/app.js': 'astringvalue'
+                    }
+                }
+            }
+        }
+        (retmessage, retmeta) = self.plugin.onMessage(msg, {})
+
+        expected_message = {
+            'source': 'cloudtrail',
+            'details': {
+                'responseelements': {
+                    'fileuploadurls': {
+                        'ds_store': 'astringvalue',
+                        'js/app.js': 'astringvalue'
+                    }
+                }
+            }
+        }
+        assert retmessage == expected_message
+        assert retmeta == {}
+
+    def test_last_dot_removal(self):
+        msg = {
+            'source': 'cloudtrail',
+            'details': {
+                'responseelements': {
+                    'fileuploadurls': {
+                        'ds_store.': 'astringvalue',
+                        'js/app.js': 'astringvalue'
+                    }
+                }
+            }
+        }
+        (retmessage, retmeta) = self.plugin.onMessage(msg, {})
+
+        expected_message = {
+            'source': 'cloudtrail',
+            'details': {
+                'responseelements': {
+                    'fileuploadurls': {
+                        'ds_store': 'astringvalue',
+                        'js/app.js': 'astringvalue'
+                    }
+                }
+            }
+        }
+        assert retmessage == expected_message
+        assert retmeta == {}
+
+    def test_list_dot_removal(self):
+        msg = {
+            'source': 'cloudtrail',
+            'details': {
+                'responseelements': {
+                    'fileuploadurls': {
+                        'ds_store.': 'astringvalue',
+                        'js/app.js': 'astringvalue'
+                    },
+                    '.randomlist.': [
+                        'weird_case',
+                        'random_case',
+                    ],
+                }
+            }
+        }
+        (retmessage, retmeta) = self.plugin.onMessage(msg, {})
+
+        expected_message = {
+            'source': 'cloudtrail',
+            'details': {
+                'responseelements': {
+                    'fileuploadurls': {
+                        'ds_store': 'astringvalue',
+                        'js/app.js': 'astringvalue'
+                    },
+                    'randomlist': [
+                        'weird_case',
+                        'random_case',
+                    ],
+                }
+            }
+        }
+        assert retmessage == expected_message
+        assert retmeta == {}