-
Notifications
You must be signed in to change notification settings - Fork 97
/
query.py
429 lines (412 loc) · 19.2 KB
/
query.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
from __future__ import print_function
import json
import sys
import contextlib
from collections import defaultdict
from datetime import datetime
from functools import partial
from multiprocessing.pool import ThreadPool
from random import shuffle
from time import time
from traceback import print_exc
from .introspection import get_listing_operations, get_regions_for_service
from .listing import Listing
RESULT_NOTHING = '---'
RESULT_SOMETHING = '+++'
RESULT_ERROR = '!!!'
RESULT_NO_ACCESS = '>:|'
# List of requests with legitimate, persistent errors that indicate that no listable resources are present.
#
# If the request would never return listable resources, it should not be done and be listed in one of the lists
# in introspection.py.
#
# TODO: If the error just indicates that the current user does not have the permissions to list resources,
# the user of this tool should probably be warned.
RESULT_IGNORE_ERRORS = {
'auditmanager': {
# Needs enabling
'GetDelegations': 'AccessDeniedException',
'GetInsights': 'AccessDeniedException',
'GetOrganizationAdminAccount': 'AccessDeniedException',
'GetServicesInScope': 'AccessDeniedException',
'ListAssessmentReports': 'AccessDeniedException',
'ListAssessments': 'AccessDeniedException',
'ListControlDomainInsights': 'AccessDeniedException',
'ListNotifications': 'AccessDeniedException',
},
'apigateway': {
# apigateway<->vpc links not supported in all regions
'GetVpcLinks': 'vpc link not supported for region',
},
'autoscaling-plans': {
# autoscaling-plans service not available in all advertised regions
'DescribeScalingPlans': 'AccessDeniedException',
},
'backup': {
'GetSupportedResourceTypes': 'AccessDeniedException',
},
'cloud9': {
# Cloud9 has DNS entries for endpoints in some regions, but does not serve the right certificate
'ListEnvironments': 'SSLError',
'DescribeEnvironmentMemberships': 'SSLError',
},
'cloudformation': {
# DescribePublisher only works if the account itself is a publisher
'DescribePublisher': 'CFNRegistryException',
},
'cloudhsm': {
'ListHapgs': 'This service is unavailable.',
'ListLunaClients': 'This service is unavailable.',
'ListHsms': 'This service is unavailable.',
},
'compute-optimizer': {
'DescribeRecommendationExportJobs':
'Aws account is not registered for recommendation.',
'GetAutoScalingGroupRecommendations':
'Aws account is not registered for recommendation.',
'GetEBSVolumeRecommendations':
'Aws account is not registered for recommendation.',
'GetEC2InstanceRecommendations':
'Aws account is not registered for recommendation.',
'GetEnrollmentStatusesForOrganization':
"You can't get enrollment statuses for member accounts of an organization.",
'GetLambdaFunctionRecommendations':
'Aws account is not registered for recommendation.',
'GetRecommendationSummaries':
'Aws account is not registered for recommendation.',
},
'config': {
# config service not available in all advertised regions
'DescribeConfigRules': 'AccessDeniedException',
},
'connect': {
# needs allowlisting
'ListTrafficDistributionGroups': 'AccessDeniedException',
},
'cur': {
# Linked accounts are not authorized to describe report definitions
'DescribeReportDefinitions': 'is not authorized to callDescribeReportDefinitions',
},
'devops-guru': {
'DescribeEventSourcesConfig': 'Please onboard to DevOps Guru',
},
'directconnect': {
'DescribeInterconnects': 'not an authorized Direct Connect partner.',
},
'discovery': {
'DescribeExportConfigurations': 'Home Region is not set',
},
'drs': {
'DescribeJobs': 'Account not initialized',
'DescribeRecoveryInstances': 'Account not initialized',
'DescribeReplicationConfigurationTemplates': 'Account not initialized',
'DescribeSourceServers': 'Account not initialized',
'ListStagingAccounts': 'Account not initialized',
},
'dynamodb': {
# dynamodb Backups not available in all advertised regions
'ListBackups': 'UnknownOperationException',
# dynamodb Global Tables not available in all advertised regions
'ListGlobalTables': 'UnknownOperationException',
},
'ec2': {
# ec2 FPGAs not available in all advertised regions
'DescribeFpgaImages':
'not valid for this web service',
# Need to register as a seller to get this listing
'DescribeReservedInstancesListings':
'not authorized to use the requested product. Please complete the seller registration',
# This seems to be the error if no ClientVpnEndpoints are available in the region
'DescribeClientVpnEndpoints':
'InternalError',
# The following API needs explicit whitelisting
'DescribeTrunkInterfaceAssociations':
'OperationNotPermitted'
},
'fms': {
'ListMemberAccounts': 'not currently delegated by AWS FM',
'ListPolicies': 'not currently delegated by AWS FM',
'ListAppsLists': 'not currently delegated by AWS FM',
'ListProtocolsLists': 'not currently delegated by AWS FM',
'ListResourceSets': 'not currently delegated by AWS FM',
},
'finspace-data': {
'GetWorkingLocation': 'Failed to retrieve environment',
'ListDatasets': 'Failed to retrieve environment',
},
'iot': {
# full iot service not available in all advertised regions
'DescribeAccountAuditConfiguration': ['An error occurred', 'No listing'],
'ListActiveViolations': 'An error occurred',
'ListIndices': 'An error occurred',
'ListJobs': 'An error occurred',
'ListOTAUpdates': 'An error occurred',
'ListScheduledAudits': 'An error occurred',
'ListSecurityProfiles': 'An error occurred',
'ListStreams': 'An error occurred',
},
'iotanalytics': {
'DescribeLoggingOptions': 'An error occurred',
},
'license-manager': {
'GetServiceSettings': 'Service role not found',
'ListDistributedGrants': 'Service role not found',
'ListLicenseConfigurations': 'Service role not found',
'ListLicenseConversionTasks': 'Service role not found',
'ListLicenseManagerReportGenerators': 'Service role not found',
'ListLicenses': 'Service role not found',
'ListReceivedGrants': 'Service role not found',
'ListReceivedLicenses': 'Service role not found',
'ListResourceInventory': 'Service role not found',
'ListTokens': 'Service role not found',
},
'license-manager-user-subscriptions': {
'ListIdentityProviders': 'Service Linked role is not present',
'ListInstances': 'Service Linked role is not present',
},
'lightsail': {
# lightsail GetDomains only available in us-east-1
'GetDomains': 'only available in the us-east-1',
},
'machinelearning': {
'DescribeBatchPredictions': 'AmazonML is no longer available to new customers.',
'DescribeDataSources': 'AmazonML is no longer available to new customers.',
'DescribeEvaluations': 'AmazonML is no longer available to new customers.',
'DescribeMLModels': 'AmazonML is no longer available to new customers.',
},
'macie': {
'ListMemberAccounts': 'Macie is not enabled',
'ListS3Resources': 'Macie is not enabled',
},
'macie2': {
'DescribeBuckets': 'Macie is not enabled',
'DescribeOrganizationConfiguration': 'you must be the Macie administrator',
'GetAdministratorAccount': 'Macie is not enabled',
'GetBucketStatistics': 'Macie is not enabled',
'GetFindingsPublicationConfiguration': 'Macie is not enabled',
'GetMacieSession': 'Macie is not enabled',
'GetMasterAccount': 'Macie is not enabled',
'GetRevealConfiguration': 'you have to enable Macie for your account',
'ListAllowLists': 'Macie isn’t enabled',
'ListClassificationJobs': 'Macie is not enabled',
'ListCustomDataIdentifiers': 'Macie is not enabled',
'ListFindings': 'Macie is not enabled',
'ListFindingsFilters': 'Macie is not enabled',
'ListManagedDataIdentifiers': 'Macie is not enabled',
'ListMembers': 'Macie is not enabled',
},
'mgn': {
'DescribeJobs': 'Account not initialized',
'DescribeLaunchConfigurationTemplates': 'Account not initialized',
'DescribeReplicationConfigurationTemplates': 'Account not initialized',
'DescribeSourceServers': 'Account not initialized',
'DescribeVcenterClients': 'Account not initialized',
},
'mturk': {
'GetAccountBalance': 'Your AWS account must be linked to your Amazon Mechanical Turk Account',
'ListBonusPayments': 'Your AWS account must be linked to your Amazon Mechanical Turk Account',
'ListHITs': 'Your AWS account must be linked to your Amazon Mechanical Turk Account',
'ListQualificationRequests': 'Your AWS account must be linked to your Amazon Mechanical Turk Account',
'ListReviewableHITs': 'Your AWS account must be linked to your Amazon Mechanical Turk Account',
'ListWorkerBlocks': 'Your AWS account must be linked to your Amazon Mechanical Turk Account',
},
'organizations': {
'DescribeOrganization': 'AccessDeniedException',
'ListAWSServiceAccessForOrganization': 'AccessDeniedException',
'ListAccounts': 'AccessDeniedException',
'ListCreateAccountStatus': 'AccessDeniedException',
'ListHandshakesForOrganization': 'AccessDeniedException',
'ListRoots': 'AccessDeniedException',
},
'rds': {
'DescribeGlobalClusters': 'Access Denied to API Version',
},
'rekognition': {
# rekognition stream processors not available in all advertised regions
'ListStreamProcessors': 'AccessDeniedException',
},
'resource-explorer-2': {
'GetDefaultView': 'No Resource Explorer index exists in this region.',
'GetIndex': 'No index found for the account.',
},
'robomaker': {
# ForbiddenException is raised if robomaker is not available in a region
'ListDeploymentJobs': 'ForbiddenException',
'ListFleets': 'ForbiddenException',
'ListRobotApplications': 'ForbiddenException',
'ListRobots': 'ForbiddenException',
'ListSimulationApplications': 'ForbiddenException',
'ListSimulationJobs': 'ForbiddenException',
},
'service-quotas': {
'GetAssociationForServiceQuotaTemplate': 'TemplatesNotAvailableInRegionException',
'ListServiceQuotaIncreaseRequestsInTemplate': 'TemplatesNotAvailableInRegionException',
},
'servicecatalog': {
'GetAWSOrganizationsAccessStatus': 'AccessDeniedException',
},
'ses': {
'DescribeActiveReceiptRuleSet': 'Service returned the HTTP status code: 404',
'ListReceiptFilters': 'Service returned the HTTP status code: 404',
'ListReceiptRuleSets': 'Service returned the HTTP status code: 404',
},
'sesv2': {
'ListImportJobs': 'Your account is still in the sandbox.',
'ListRecommendations': 'To use this feature you must enable',
},
'shield': {
'DescribeDRTAccess': 'An error occurred',
'DescribeEmergencyContactSettings': 'An error occurred',
'ListProtections': 'ResourceNotFoundException',
'ListProtectionGroups': 'ResourceNotFoundException',
},
'snowball': {
'ListCompatibleImages': 'An error occurred',
},
'snow-device-management': {
'ListDevices': 'No devices found in this region.',
'ListTasks': 'No devices found in this region.',
},
'sso-admin': {
'ListInstances': 'is not authorized to perform: sso:ListInstances',
},
'storagegateway': {
# The storagegateway advertised but not available in some regions
'DescribeTapeArchives': 'InvalidGatewayRequestException',
'ListTapes': 'InvalidGatewayRequestException',
},
'support': {
'DescribeCases': 'Amazon Web Services Premium Support Subscription is required',
'DescribeServices': 'Amazon Web Services Premium Support Subscription is required',
'DescribeSeverityLevels': 'Amazon Web Services Premium Support Subscription is required',
},
}
NOT_AVAILABLE_FOR_REGION_STRINGS = [
'is not supported in this region',
'is not available in this region',
'not supported in the called region.',
'Operation not available in this region',
'Credential should be scoped to a valid region,',
'The security token included in the request is invalid.',
'AWS was not able to validate the provided access credentials',
'InvalidAction',
]
NOT_AVAILABLE_FOR_ACCOUNT_STRINGS = [
'This request has been administratively disabled',
'Your account isn\'t authorized to call this operation.',
'AWS Premium Support Subscription is required',
'not subscribed to AWS Security Hub',
'is not authorized to use this service',
'Account not whitelisted',
]
NOT_AVAILABLE_STRINGS = NOT_AVAILABLE_FOR_REGION_STRINGS + NOT_AVAILABLE_FOR_ACCOUNT_STRINGS
def do_query(services, selected_regions=(), selected_operations=(), verbose=0, parallel=32, selected_profile=None):
"""For the given services, execute all selected operations (default: all) in selected regions
(default: all)"""
to_run = []
print('Building set of queries to execute...')
for service in services:
for region in get_regions_for_service(service, selected_regions):
for operation in get_listing_operations(service, region, selected_operations, selected_profile):
if verbose > 0:
region_name = region or 'n/a'
print('Service: {: <28} | Region: {:<15} | Operation: {}'.format(service, region_name, operation))
to_run.append([service, region, operation, selected_profile])
shuffle(to_run) # Distribute requests across endpoints
results_by_type = defaultdict(list)
print('...done. Executing queries...')
# the `with` block is a workaround for a bug: https://bugs.python.org/issue35629
with contextlib.closing(ThreadPool(parallel)) as pool:
for result in pool.imap_unordered(partial(acquire_listing, verbose), to_run):
results_by_type[result[0]].append(result)
if verbose > 1:
print('ExecutedQueryResult: {}'.format(result))
else:
print(result[0][-1], end='')
sys.stdout.flush()
print('...done')
for result_type in (RESULT_NOTHING, RESULT_SOMETHING, RESULT_NO_ACCESS, RESULT_ERROR):
for result in sorted(results_by_type[result_type]):
print(*result)
def acquire_listing(verbose, what):
"""Given a service, region and operation execute the operation, serialize and save the result and
return a tuple of strings describing the result."""
service, region, operation, profile = what
start_time = time()
try:
if verbose > 1:
print(what, 'starting request...')
listing = Listing.acquire(service, region, operation, profile)
duration = time() - start_time
if verbose > 1:
print(what, '...request successful')
print("timing [success]:", duration, what)
if listing.resource_total_count > 0:
with open('{}_{}_{}_{}.json'.format(service, operation, region, profile), 'w') as jsonfile:
json.dump(listing.to_json(), jsonfile, default=datetime.isoformat)
return (RESULT_SOMETHING, service, region, operation, profile, ', '.join(listing.resource_types))
else:
return (RESULT_NOTHING, service, region, operation, profile, ', '.join(listing.resource_types))
except Exception as exc: # pylint:disable=broad-except
duration = time() - start_time
if verbose > 1:
print(what, '...exception:', exc)
print("timing [failure]:", duration, what)
if verbose > 2:
print_exc()
result_type = RESULT_NO_ACCESS if 'AccessDeniedException' in str(exc) else RESULT_ERROR
ignored_err = RESULT_IGNORE_ERRORS.get(service, {}).get(operation)
if ignored_err is not None:
if not isinstance(ignored_err, list):
ignored_err = list(ignored_err)
for ignored_str_err in ignored_err:
if ignored_str_err in str(exc):
result_type = RESULT_NOTHING
for not_available_string in NOT_AVAILABLE_STRINGS:
if not_available_string in str(exc):
result_type = RESULT_NOTHING
return (result_type, service, region, operation, profile, repr(exc))
def do_list_files(filenames, verbose=0):
"""Print out a rudimentary summary of the Listing objects contained in the given files"""
for listing_filename in filenames:
listing = Listing.from_json(json.load(open(listing_filename, 'rb')))
resources = listing.resources
truncated = False
if 'truncated' in resources:
truncated = resources['truncated']
del resources['truncated']
for resource_type, value in resources.items():
len_string = '> {}'.format(len(value)) if truncated else str(len(value))
print(listing.service, listing.region, listing.operation, resource_type, len_string)
if verbose > 0:
for item in value:
idkey = None
if isinstance(item, dict):
guesses = [resource_type[:-1] + "Id", "id", "SerialNumber"]
# Find the last uppercase word in the resource_type and construct some guesses from that
uppercase_indices = [i for (i, c) in enumerate(resource_type) if c.isupper()]
if uppercase_indices:
last_word_in_resource_type = resource_type[uppercase_indices[-1]:]
guesses.append(last_word_in_resource_type[:-1] + "Id")
guesses.append(last_word_in_resource_type + "Id")
for guess in guesses:
if guess in item:
idkey = guess
break
if idkey is None:
for heuristic in [
lambda x: x.endswith('Id'),
lambda x: x.endswith('Name'),
]:
idkeys = [k for k in item.keys() if heuristic(k)]
if idkeys:
# Heuristic: Shortest ID is probably the Resource ID
idkeys.sort(key=len)
idkey = idkeys[0]
break
if idkey:
print(' - ', item.get(idkey, ', '.join(item.keys())))
else:
print(' - ', item)
if truncated:
print(' - ... (more items, query truncated)')