truncates htmlpart to something ES can handle and adds authparameters…

… to modify_keys (#1661)

* truncates htmlpart to something ES can handle

* adding comments to code and checking if values exist per @arcrose's suggestion

* removing whitespace

* adds a constnat and adds description for the ES_FIELD_LIMIT_VALUE constant.

mq/plugins/cloudtrail.py

@@ -24,6 +24,7 @@ def __init__(self):
             'details.apiversion',
             'details.serviceeventdetails',
             'details.requestparameters.attribute',
+            'details.requestparameters.authparameters',
             'details.requestparameters.bucketpolicy.statement.principal.service',
             'details.requestparameters.bucketpolicy.statement.principal.aws',
             'details.requestparameters.callerreference',
@@ -111,12 +112,24 @@ def convert_key_raw_str(self, needle, haystack):
                 return haystack
 
     def onMessage(self, message, metadata):
+        '''
+        Check if source is in the message, if not then add cloudtrail as the source.
+        '''
         if 'source' not in message:
             return (message, metadata)
 
         if not message['source'] == 'cloudtrail':
             return (message, metadata)
 
+        '''
+        Check if details.requestparameters.htmlpart exists, if it does it's generally longer than 32000 bytes,
+        so we'll truncate it to 4096 characters using the constant ES_FIELD_VALUE_LIMIT so that ES will ingest it,
+        leaving us with knowledge of what the field contains without the overkill of storing the entire page.
+        '''
+        ES_FIELD_VALUE_LIMIT = 4095
+        if message.get('details', {}).get('requestparameters', {}).get('htmlpart') is not None:
+                message['details']['requestparameters']['htmlpart'] = message['details']['requestparameters']['htmlpart'][0:ES_FIELD_VALUE_LIMIT]
+
         for modified_key in self.modify_keys:
             if key_exists(modified_key, message):
                 message = self.convert_key_raw_str(modified_key, message)

tests/mq/plugins/test_cloudtrail.py

@@ -418,3 +418,25 @@ def test_unusual(self):
         }
         assert retmessage == expected_message
         assert retmeta == {}
+
+    def test_length_truncate(self):
+        msg = {
+            'source': 'cloudtrail',
+            'details': {
+                'requestparameters': {
+                    'htmlpart': 'phdolIXPdTsHKfdoadpPfWXBkiSDWKGORPaHYlwvkuYHjPyeJDoaTswKpHSNqMMDQwHkEuCfKGqBZhZmjVEflKLRCwWcOErRISwyScnrhOAMoEjQsdbiNWqIZRwlxEqWvzxUugJJqTImGxsfysJcDGEnCamwkbyILnBChhsTFzvgUEKhiKPHdVNdjysJYLbgCIHCagKyfuXswFzmJHOwrEtMwIYqCnOteWBRwefYICdGbIWhlxzMwlIWGpMWgRZWzpoaqMQvTQDNJqTvkjxaUMIQbdmKDPjbZoFmaIjbDyDTwCkGCLzROyMHXdAqSRtBttUCvJtuyxvjlBXJdMzUDBtIPDjadBrqLDbKDdeFznFafjUhioHhcGgkdfIpoPGxWjrcZdWjJeQcRDwhXtekbNzFQbTtFJStNBXuVyZfKYYARtSiSMtLPEfqSOBrpyJlotaAcVutGgVhrVeVOaTBidaXwDjRUnfNOVgGiCkycpWsSCSCfozOwEiTunVdfPElCFyZXuOIRKFvEiTyuPExUcDVTNFsLvYmlOUqTvlMaWnXNpLQGpOTHSGxFPqlqsCeslnOKxvaXDKnQYufXatLmqFLygOsfbUsbbUWPEqyjUrtaEfXpxDwljeSYOTsgJllZlfWRjqtOKGwpKpyKewHNUWyJestNIbsWSvVQXMETeDacivvIbuqtIZbBXSQkYJNjmmYqnJWRXtLyqDKkzGegfHXWmIVfuXhHCqzfNfXNsRpZNeXhdEQqlcromCrPmYKkOQvrqqpXxaNDEoAkMKSVOHPcQEMVKjXQhMJGfVnxCbesFaMBHvBaPPZbaZoZCGgrJgBdCqGrtGMdmtJBRwSTZvUmjFLOWhhkALLEbcAVxwFDJCkvxsUdaGwkCXhxIFhaAcDffAmYLJsBlXCDFxDBUeGxTyRLSNeJclZUONvaFbSnMvYbKbrWTKmERYSivGoFwspIsjBHHksbIYLNGQRPxdVwzrkbLUMWjbXGolXMKUFZmHButvJZOlduxlraoRxydpOmgLKWtmplljItNNxzGzqbHhTXyPAiNCGpJKgLkKDJVPdkXaISEZGmkLSNSnOEKNXLRFpCHWQvbCGzuYLDPJnQymlVKfYUykucojsgSyckydidGSdjnXiioXGFseUPzTJkVuVyXBeYygarJUIitpJEYuBXxtGnMwZHWsCiPjDdHnMMvtgVjjpXgCoLWJTMMAfwmnImAwSLLwVJzYOcIQTapuxziKoWPDDBmhSfYROFosGxmEtSSDUYETwUMcnZLCAvqrMBWZootAkbSAeRrRrxUXkuegVdvlJjGfUupDNTqKSWmpKkHmQBIvRFCmvLlsSuRmYTenQosDYhPdjxLyKmtIBbiKuuTalIxoCpdHlhsavlRDXYsxBlUArwPsGPJGsEnEwYspAhwZmyKXUHXIlzKgLiCtDtbCNuHnCzveRJoswWzrUHvNnLJSwIRyVENfJNBMzdRoRLmSMkDqfseqTadFbYYFnnElrTjBYlJVHlwcSsrOYMpmXdyLMVtUpLzMzzqtUzEcsZsUPPUJeBBCdVXzyIGJXDxaXaFEsLuKPSOnLcKHucteEGutYsqhsgwQisIiKeSRfwjVtZYxPbdqxzttnMFRnheiMvqModwLIkpVEzzVPbADRgTfZXNpCGoorMwZikiQfKXGkdbOSOtZAwjrskEspBgeXRLniFEDOWoNlqqOZyrpQovlxaNFDBCJsjrysxMRfeEJMrJRkcGqjfSUUDLKnwcadJlsoPBEUsDJIOhrEMggHREPAOWrePhwEgxlvxHgQrCVoGXXwdESsBUSWTSkqwkcRKyFQvmbqAUjLaFsELMEVPNbRciUcSueLILHIJQHlWrvqPnPledJNVCmwTHfBToeFnaFndWrCXfKpuTjMAwgQWhMVmamBcETQBgLHAylOrZrfDbAFxzxkkAebQqnaLiOVJxdZtrzSxPZfYYBpYHxpbaHGVqSgPPhTDqMCbiMTKLFUWGpuYUhGTOtbiltQEjwuWKfXhiRdRZutLfesRuqhHTEzLTiugoFbxTgFfoTGLoHiJquJvFFXDDLYwIPjGQSZmcaVECduqvVCkdWXNTOoLOcHPmTavmeLldTbjIUXYAUTmMwpyqeVblDYaYWpdwuNhJrgDYQiYuZnyYfkvplFbnDkunVuFSrbXKxxMAkpxbxqJipPrgPxUTCjyNwpjezruOnKlSmFMlFbUIzMypluiLMiZYNuyVTzJRTZHyBYHwGNJzBaNhyDjLBLyRpVhNRPBbUcHJjhcoxzmrglayUlpeUaarBjKogpNskrcsRKqRUdozlCliXucWeJDFkoMNrDKpGQFyzdkiIxKeOyVoNYdwknviWLgPilaqGtsbLccKqIVuNUmhvKTZGjqDdoocqcexxSpbZJrGvXmmmmMrzmKgGQBNBSiuAUneqJgNvsblzuhMDhTGzpIDksJPLvTMMhSWIYZdNxDqAvrldBzTbsHEYFxSDWURsSADQdqsplnbmgTtJfMMpCQQMCyGPMdWwpLjnBeedyjcdDOZWLODtzTyrbRbXwPZOplZtexKmTeavTycDOEOUorwnaLfloSzNxqRZhDkpldvihtPnFcVEvObIOvXPSTUJTHewYEaXiYgNlXilgImichPahKESYveAZRxDUCtiMyStQeaoRMJevbVVzWzHSBZrYsvlocgKuAgCPZblpGTyVGmPvJjAaszosjgRwGmdUFLqOPzkzuXymWXAtKfbYJRWTfiyVaKrbjrVWPtsNlJiHepLJmskGMkXInnCHifEHjcnDJNjTjJefQZbmDcAECUoNFbtBdJluZFUBvGrzOYgJiCpUIAHzgAEtojIbqBRSvGDudQIuJnjokzYRTWvinLKHNgnxTfkKgIxBqdoAsgGCuGeknbMRqvJJCQOicQslJIuEruGNxdegjHRLJdoHfxcTeduKJuFoHDWrOxpYinwaOwyJmJPcbRVIGCiKQGegHKZzVHHCpYDfuAFPzvfyZxodikbbkLaDaAkkHUIvokQzqJltIcguObiltOTbMvjlnozYzMSoYtjFSjICmKGyGNplVHLLCtepjbXsTVKMDCxfInKZoYKyfSTFrvGbWnyyWOBnAiKeOZWHMgykQUVWyxOLfKijidYxHmPnPQjiQQLteUwTPqOSJwcEWQKiYTuERlsfapHJPVXdLVZUufNUDbGQwnZgmKxhoUMuhzRmDAQlDqRtTOdgAbomwfFiBwCoIMpdLUhVHucAEjhBmResdYCsptHJpJyrZcCilZvsWjfBFNVSDxrtgRmTAOKaFkAqYpAGKyezofzEZbwdgZndnkKfBnwbvPBaKCvOVbPGDDnkvARJJtqJrVMoUbCWNPIOYuKCihfYqXYyZHpWJErcDqrEhEpkfaqrMPOxmjAVWIQnoNXLdcGSJLehlPvmKBHGisEkigSXVZigxZZSPLdzzsBOoeULKPfLVWDGixOrIwfoYFNWetqcEYswPycRSOCzxzHPVREKSLTkSbJHyYCUqepAIDdCdMGukXINiDAdLtYYGKjFxVHEoiEVPsmsnZjAYIMEZKmjBxNSEGdWsnNNgoRKUHAYZiAnnNZMxEcrRlMdyfsCKppLNROmRCqlaxvbHBsGGiWxstfFGCgcRPNumnByFUdsirIgJiEXmPSIQuWYyjqselIqsingwPmazhRkPagxTdzVewNOMKvUlKdHwyWzCzSInMLhuWRGLrPTgtItOqSSaAuNLOhdgMdFeBPciyVzYTHtUXtwZwmVGFNWuIUjjXOJgmCDtLoZTrxEixDwAgutkNkuweHptnbpzPeMUgymRoTNzFYFAazauukrinTphAPcHSDiGHHMeahHuYkwFhDYJQxluoTPHPxYZrHvicOnRUBgejGsxsVTwPGBrCSeeiBzfPtvukyLWNxdNvZEmrDuGwemTXxhrJHpbqDYFWzSjOWpHxTeCVMrCFLquQBUsqBoRfCaDtGXcKrXUmttnGaSrbEQGQghioHauENjmDGSUTKylhrcWCQiaGocLLKphAmpOrKiZeAuQsRqkcDNYEyogAlJMIQbnFoRTRUQViyseoUuQvjttEFyljDOodihensleMhcQwlzfWrbhbwhqRhrsLArDSdgOjpBBmhwSuikYtPvFdawmiQmtecXdyFMmHIKXDrXtNuDnuqzPpTrImGIxTkdgsHxXgBYQkBRDqZHtiklgeAPZaMlBxKhAixLzqKBnKgRuafUAnFvstEUhxlUIdZhCOHYfKbCoxUaWpZVTfMQcXkfNPbqwqoLdTzfdhHhBhkPAcKfVyLkzKXjRJTUnJHrmpyXMRsKYpokmZzmRIKzDquVtNNilNZaQARKpGYJdfYlwTqzXdrFInQsPQVNcUuZBwQqtirZLHqDcBZfljwPTPWgKmOaZsJGtDQLoIytEMaOrwGDDSotRwApvhrvVkEoXphHTBEJcdZozToGYUNzpyuJocIZvygtAjwINNBrEuOBWuNjlEhyWhntFQFzDthwCfCKpRlpPDnaPuoKKfbyRrIPYKEaFnLZmFBmppPXBgwpyAxwnWWFKPLRMDowpQuTweqAYMnqZPLyJJpsIWnGjTBbzVewlEGtIiwRoEFsYDJdkatlUqlUldWkypHAjcDGOBylHQaSCmmGXrVuIsBtfkTHZdSXfiOQEGzUZPqxnbwoZXbAeROOehRXKgwkWUqKuACkaNlwbwESOoyCyTpWKigAInnaUfXvHERtfFrtftKenryCSCuVuIbWjKnMWVLzptodNrpGtjmOjKeJyLRxQbEEQZcPcUxKjfNPbsCxNDhpCljcIeOEQreSeFQBFflliXRCWtbQTaxuYbiYYYAdicUhucBLRMPhGZXHoaDdpZmQulubBhLaPkZxuWylSYomUZOngPpyYQfkvqmLPpbJYvvlkroVScXkiBezGuGPvlFXbknKSxAXhiFhKacnxICGhIpLLasBQHXSNKkSwwBbGAFetgMTxAuAqwnDUVsn'
+                }
+            }
+        }
+        (retmessage, retmeta) = self.plugin.onMessage(msg, {})
+
+        expected_message = {
+            'source': 'cloudtrail',
+            'details': {
+                'requestparameters': {
+                    'htmlpart': 'phdolIXPdTsHKfdoadpPfWXBkiSDWKGORPaHYlwvkuYHjPyeJDoaTswKpHSNqMMDQwHkEuCfKGqBZhZmjVEflKLRCwWcOErRISwyScnrhOAMoEjQsdbiNWqIZRwlxEqWvzxUugJJqTImGxsfysJcDGEnCamwkbyILnBChhsTFzvgUEKhiKPHdVNdjysJYLbgCIHCagKyfuXswFzmJHOwrEtMwIYqCnOteWBRwefYICdGbIWhlxzMwlIWGpMWgRZWzpoaqMQvTQDNJqTvkjxaUMIQbdmKDPjbZoFmaIjbDyDTwCkGCLzROyMHXdAqSRtBttUCvJtuyxvjlBXJdMzUDBtIPDjadBrqLDbKDdeFznFafjUhioHhcGgkdfIpoPGxWjrcZdWjJeQcRDwhXtekbNzFQbTtFJStNBXuVyZfKYYARtSiSMtLPEfqSOBrpyJlotaAcVutGgVhrVeVOaTBidaXwDjRUnfNOVgGiCkycpWsSCSCfozOwEiTunVdfPElCFyZXuOIRKFvEiTyuPExUcDVTNFsLvYmlOUqTvlMaWnXNpLQGpOTHSGxFPqlqsCeslnOKxvaXDKnQYufXatLmqFLygOsfbUsbbUWPEqyjUrtaEfXpxDwljeSYOTsgJllZlfWRjqtOKGwpKpyKewHNUWyJestNIbsWSvVQXMETeDacivvIbuqtIZbBXSQkYJNjmmYqnJWRXtLyqDKkzGegfHXWmIVfuXhHCqzfNfXNsRpZNeXhdEQqlcromCrPmYKkOQvrqqpXxaNDEoAkMKSVOHPcQEMVKjXQhMJGfVnxCbesFaMBHvBaPPZbaZoZCGgrJgBdCqGrtGMdmtJBRwSTZvUmjFLOWhhkALLEbcAVxwFDJCkvxsUdaGwkCXhxIFhaAcDffAmYLJsBlXCDFxDBUeGxTyRLSNeJclZUONvaFbSnMvYbKbrWTKmERYSivGoFwspIsjBHHksbIYLNGQRPxdVwzrkbLUMWjbXGolXMKUFZmHButvJZOlduxlraoRxydpOmgLKWtmplljItNNxzGzqbHhTXyPAiNCGpJKgLkKDJVPdkXaISEZGmkLSNSnOEKNXLRFpCHWQvbCGzuYLDPJnQymlVKfYUykucojsgSyckydidGSdjnXiioXGFseUPzTJkVuVyXBeYygarJUIitpJEYuBXxtGnMwZHWsCiPjDdHnMMvtgVjjpXgCoLWJTMMAfwmnImAwSLLwVJzYOcIQTapuxziKoWPDDBmhSfYROFosGxmEtSSDUYETwUMcnZLCAvqrMBWZootAkbSAeRrRrxUXkuegVdvlJjGfUupDNTqKSWmpKkHmQBIvRFCmvLlsSuRmYTenQosDYhPdjxLyKmtIBbiKuuTalIxoCpdHlhsavlRDXYsxBlUArwPsGPJGsEnEwYspAhwZmyKXUHXIlzKgLiCtDtbCNuHnCzveRJoswWzrUHvNnLJSwIRyVENfJNBMzdRoRLmSMkDqfseqTadFbYYFnnElrTjBYlJVHlwcSsrOYMpmXdyLMVtUpLzMzzqtUzEcsZsUPPUJeBBCdVXzyIGJXDxaXaFEsLuKPSOnLcKHucteEGutYsqhsgwQisIiKeSRfwjVtZYxPbdqxzttnMFRnheiMvqModwLIkpVEzzVPbADRgTfZXNpCGoorMwZikiQfKXGkdbOSOtZAwjrskEspBgeXRLniFEDOWoNlqqOZyrpQovlxaNFDBCJsjrysxMRfeEJMrJRkcGqjfSUUDLKnwcadJlsoPBEUsDJIOhrEMggHREPAOWrePhwEgxlvxHgQrCVoGXXwdESsBUSWTSkqwkcRKyFQvmbqAUjLaFsELMEVPNbRciUcSueLILHIJQHlWrvqPnPledJNVCmwTHfBToeFnaFndWrCXfKpuTjMAwgQWhMVmamBcETQBgLHAylOrZrfDbAFxzxkkAebQqnaLiOVJxdZtrzSxPZfYYBpYHxpbaHGVqSgPPhTDqMCbiMTKLFUWGpuYUhGTOtbiltQEjwuWKfXhiRdRZutLfesRuqhHTEzLTiugoFbxTgFfoTGLoHiJquJvFFXDDLYwIPjGQSZmcaVECduqvVCkdWXNTOoLOcHPmTavmeLldTbjIUXYAUTmMwpyqeVblDYaYWpdwuNhJrgDYQiYuZnyYfkvplFbnDkunVuFSrbXKxxMAkpxbxqJipPrgPxUTCjyNwpjezruOnKlSmFMlFbUIzMypluiLMiZYNuyVTzJRTZHyBYHwGNJzBaNhyDjLBLyRpVhNRPBbUcHJjhcoxzmrglayUlpeUaarBjKogpNskrcsRKqRUdozlCliXucWeJDFkoMNrDKpGQFyzdkiIxKeOyVoNYdwknviWLgPilaqGtsbLccKqIVuNUmhvKTZGjqDdoocqcexxSpbZJrGvXmmmmMrzmKgGQBNBSiuAUneqJgNvsblzuhMDhTGzpIDksJPLvTMMhSWIYZdNxDqAvrldBzTbsHEYFxSDWURsSADQdqsplnbmgTtJfMMpCQQMCyGPMdWwpLjnBeedyjcdDOZWLODtzTyrbRbXwPZOplZtexKmTeavTycDOEOUorwnaLfloSzNxqRZhDkpldvihtPnFcVEvObIOvXPSTUJTHewYEaXiYgNlXilgImichPahKESYveAZRxDUCtiMyStQeaoRMJevbVVzWzHSBZrYsvlocgKuAgCPZblpGTyVGmPvJjAaszosjgRwGmdUFLqOPzkzuXymWXAtKfbYJRWTfiyVaKrbjrVWPtsNlJiHepLJmskGMkXInnCHifEHjcnDJNjTjJefQZbmDcAECUoNFbtBdJluZFUBvGrzOYgJiCpUIAHzgAEtojIbqBRSvGDudQIuJnjokzYRTWvinLKHNgnxTfkKgIxBqdoAsgGCuGeknbMRqvJJCQOicQslJIuEruGNxdegjHRLJdoHfxcTeduKJuFoHDWrOxpYinwaOwyJmJPcbRVIGCiKQGegHKZzVHHCpYDfuAFPzvfyZxodikbbkLaDaAkkHUIvokQzqJltIcguObiltOTbMvjlnozYzMSoYtjFSjICmKGyGNplVHLLCtepjbXsTVKMDCxfInKZoYKyfSTFrvGbWnyyWOBnAiKeOZWHMgykQUVWyxOLfKijidYxHmPnPQjiQQLteUwTPqOSJwcEWQKiYTuERlsfapHJPVXdLVZUufNUDbGQwnZgmKxhoUMuhzRmDAQlDqRtTOdgAbomwfFiBwCoIMpdLUhVHucAEjhBmResdYCsptHJpJyrZcCilZvsWjfBFNVSDxrtgRmTAOKaFkAqYpAGKyezofzEZbwdgZndnkKfBnwbvPBaKCvOVbPGDDnkvARJJtqJrVMoUbCWNPIOYuKCihfYqXYyZHpWJErcDqrEhEpkfaqrMPOxmjAVWIQnoNXLdcGSJLehlPvmKBHGisEkigSXVZigxZZSPLdzzsBOoeULKPfLVWDGixOrIwfoYFNWetqcEYswPycRSOCzxzHPVREKSLTkSbJHyYCUqepAIDdCdMGukXINiDAdLtYYGKjFxVHEoiEVPsmsnZjAYIMEZKmjBxNSEGdWsnNNgoRKUHAYZiAnnNZMxEcrRlMdyfsCKppLNROmRCqlaxvbHBsGGiWxstfFGCgcRPNumnByFUdsirIgJiEXmPSIQuWYyjqselIqsingwPmazhRkPagxTdzVewNOMKvUlKdHwyWzCzSInMLhuWRGLrPTgtItOqSSaAuNLOhdgMdFeBPciyVzYTHtUXtwZwmVGFNWuIUjjXOJgmCDtLoZTrxEixDwAgutkNkuweHptnbpzPeMUgymRoTNzFYFAazauukrinTphAPcHSDiGHHMeahHuYkwFhDYJQxluoTPHPxYZrHvicOnRUBgejGsxsVTwPGBrCSeeiBzfPtvukyLWNxdNvZEmrDuGwemTXxhrJHpbqDYFWzSjOWpHxTeCVMrCFLquQBUsqBoRfCaDtGXcKrXUmttnGaSrbEQGQghioHauENjmDGSUTKylhrcWCQiaGocLLKphAmpOrKiZeAuQsRqkcDNYEyogAlJMIQbnFoRTRUQViyseoUuQvjttEFyljD'
+                }
+            }
+        }
+        assert retmessage == expected_message
+        assert retmeta == {}