Flag support. (#30) 🎌🏁⛳🏴🏳️🚩🏳️‍🌈

* Initial commit of s3 flag support.

* Adding flag registry

* Support for multiple return values.

* Adding FlagRegistry to IAM Role

* Fixing naming for the FlagRegistry class.

* Adding flag feature to IAM user

* Adding flags to gcp.serviceaccount

* Documenting the flags.

* Bumping version to 1.2.0

* Making S3 GRANTS, GRANT_REFERENCES, and OWNER flags separate values.

* Adding Flag class.  Adding BASE flag to all technologies. Removing bunch from requirements

* Updating Flag section in README's

* Updating FlagRegistry.build_out to only call a method if one of the return values is required.

* whitespace

* Removing FLAGS.CREATED_DATE from default S3 flags in get_bucket(). Correcting include_created behavior for backwards compatibility.

* Removing include_created flag from GCS Bucket get_bucket method

README.md

@@ -24,12 +24,14 @@ AWS:
  - rate limit handling, with exponential backoff.
  - multi-account sts:assumerole abstraction.
  - orchestrates all the calls required to fully describe an item.
+ - control which attributes are returned with flags.
 
 GCP:
  - choosing the best client based on service
  - client caching
  - general caching and stats decorators available
  - basic support for non-specified discovery-API services
+ - control which attributes are returned with flags.
 
 ## Orchestration Supported Technologies
 
@@ -129,7 +131,7 @@ GCP:
 
 ### AWS IAM Role
 
-    from cloudaux.orchestration.aws.iam.role import get_role
+    from cloudaux.orchestration.aws.iam.role import get_role, FLAGS
 
     # account_number may be extracted from the ARN of the role passed to get_role
     # if not included in conn.
@@ -142,8 +144,15 @@ GCP:
     role = get_role(
         dict(arn='arn:aws:iam::000000000000:role/myRole', role_name='myRole'),
         output='camelized',  # optional: {camelized underscored}
+        flags=FLAGS.ALL,  # optional
         **conn)
 
+    # The flags parameter is optional but allows the user to indicate that 
+    # only a subset of the full item description is required.
+    # IAM Role Flag Options:
+    #   BASE, MANAGED_POLICIES, INLINE_POLICIES, INSTANCE_PROFILES, ALL (default)
+    # For instance: flags=FLAGS.MANAGED_POLICIES | FLAGS.INSTANCE_PROFILES
+
     # cloudaux makes a number of calls to obtain a full description of the role
     print(json.dumps(role, indent=4, sort_keys=True))
 
@@ -162,11 +171,13 @@ GCP:
 
 ### GCP IAM Service Account
 
-    from cloudaux.orchestration.gcp.iam.serviceaccount import get_serviceaccount_complete
+    from cloudaux.orchestration.gcp.iam.serviceaccount import get_serviceaccount_complete, FLAGS
     sa_name = 'projects/my-project-one/serviceAccounts/service-account-key@my-project-one.iam.gserviceaccount.com'
-    sa = get_serviceaccount_complete(sa_name, **conn_details)
+    sa = get_serviceaccount_complete(sa_name, flags=FLAGS.ALL, **conn_details)
     print(json.dumps(sa, indent=4, sort_keys=True))
 
+    # Flag options for Service Accounts are BASE, KEYS, POLICY, ALL (default).
+
     {
       "DisplayName": "service-account", 
       "Email": "service-account@my-project-one.iam.gserviceaccount.com", 

cloudaux/__about__.py

@@ -13,7 +13,7 @@
 __summary__ = 'Cloud Auxiliary is a python wrapper and orchestration module for interacting with cloud providers'
 __uri__ = 'https://github.com/Netflix-Skunkworks/cloudaux'
 
-__version__ = '1.1.9'
+__version__ = '1.2.0'
 
 __author__ = 'Patrick Kelley'
 __email__ = 'patrick@netflix.com'

cloudaux/aws/README.md

@@ -15,6 +15,7 @@ Cloud Auxiliary has support for Amazon Web Services.
  - rate limit handling, with exponential backoff.
  - multi-account sts:assumerole abstraction.
  - orchestrates all the calls required to fully describe an item.
+ - control which attributes are returned flags.
 
 ## Orchestration Supported Technologies
 
@@ -72,7 +73,7 @@ Cloud Auxiliary has support for Amazon Web Services.
 
 ### Role
 
-    from cloudaux.orchestration.aws.iam.role import get_role
+    from cloudaux.orchestration.aws.iam.role import get_role, FLAGS
 
     # account_number may be extracted from the ARN of the role passed to get_role
     # if not included in conn.
@@ -85,7 +86,14 @@ Cloud Auxiliary has support for Amazon Web Services.
     role = get_role(
         dict(arn='arn:aws:iam::000000000000:role/myRole', role_name='myRole'),
         output='camelized',  # optional: {camelized underscored}
+        flags=FLAGS.ALL,  # optional
         **conn)
+        
+    # The flags parameter is optional but allows the user to indicate that 
+    # only a subset of the full item description is required.
+    # IAM Role Flag Options:
+    #   BASE, MANAGED_POLICIES, INLINE_POLICIES, INSTANCE_PROFILES, ALL (default)
+    # For instance: flags=FLAGS.MANAGED_POLICIES | FLAGS.INSTANCE_PROFILES
 
     # cloudaux makes a number of calls to obtain a full description of the role
     print(json.dumps(role, indent=4, sort_keys=True))
@@ -105,11 +113,19 @@ Cloud Auxiliary has support for Amazon Web Services.
 
 ### User    
 
-    from cloudaux.orchestration.aws.iam.user import get_user
+    from cloudaux.orchestration.aws.iam.user import get_user, FLAGS
 
     user = get_user(
         dict(arn='arn:aws:iam::000000000000:user/myUser', role_name='myUser'),
+        flags=FLAGS.ALL,
         **conn)
+
+    # The flags parameter is optional but allows the user to indicate that 
+    # only a subset of the full item description is required.
+    # IAM User Flag Options:
+    #   BASE, ACCESS_KEYS, INLINE_POLICIES, MANAGED_POLICIES       
+    #   MFA_DEVICES, LOGIN_PROFILE, SIGNING_CERTIFICATES, ALL (default)
+    # For instance: flags=FLAGS.ACCESS_KEYS | FLAGS.MFA_DEVICES | FLAGS.LOGIN_PROFILE
 
     print(json.dumps(user, indent=2, sort_keys=True))
 
@@ -141,13 +157,21 @@ Cloud Auxiliary has support for Amazon Web Services.
 
 ### S3
 
-    from cloudaux.orchestration.aws.s3 import get_bucket
+    from cloudaux.orchestration.aws.s3 import get_bucket, FLAGS
 
     conn = dict(
         account_number='000000000000',
         assume_role='SecurityMonkey')
 
-    bucket = get_bucket('MyS3Bucket', **conn)
+    bucket = get_bucket('MyS3Bucket', flags=FLAGS.ALL, **conn)
+
+    # The flags parameter is optional but allows the user to indicate that 
+    # only a subset of the full item description is required.
+    # S3 Flag Options are:
+    #   BASE, GRANTS, GRANT_REFERENCES, OWNER, LIFECYCLE, LOGGING, POLICY, TAGS
+    #   VERSIONING, WEBSITE, CORS, NOTIFICATIONS, ACCELERATION, REPLICATION
+    #   ANALYTICS, METRICS, INVENTORY, CREATED_DATE, ALL (default)
+    # For instance: flags=FLAGS.WEBSITE | FLAGS.CORS | FLAGS.POLICY
 
     print(json.dumps(bucket, indent=2, sort_keys=True))
 

cloudaux/gcp/README.md

@@ -14,6 +14,7 @@ Cloud Auxiliary has support for Google Cloud Platform.
  - client caching
  - general caching and stats decorators available
  - basic support for non-specified discovery-API services
+ - control which attributes are returned with flags.
 
 ## Orchestration Supported Technologies
 
@@ -66,10 +67,12 @@ Cloud Auxiliary has support for Google Cloud Platform.
 
 ### IAM Service Account
 
-    from cloudaux.orchestration.gcp.iam.serviceaccount import get_serviceaccount_complete
+    from cloudaux.orchestration.gcp.iam.serviceaccount import get_serviceaccount_complete, FLAGS
     sa_name = 'projects/my-project-one/serviceAccounts/service-account-key@my-project-one.iam.gserviceaccount.com'
-    sa = get_serviceaccount_complete(sa_name, **conn_details)
+    sa = get_serviceaccount_complete(sa_name, flags=FLAGS.ALL, **conn_details)
     print(json.dumps(sa, indent=4, sort_keys=True))
+
+    # Flag options for Service Accounts are BASE, KEYS, POLICY, ALL (default).
 
     {
       "DisplayName": "service-account", 
@@ -103,6 +106,9 @@ Cloud Auxiliary has support for Google Cloud Platform.
     net_subnet = get_network_and_subnetworks(network=NETWORK, **conn_details)
     print(json.dumps(net_subnet, indent=4, sort_keys=True))
 
+    # Flag options for Service Accounts are BASE, ALL (default).
+    # BASE and ALL are equivelant for this service.
+
     {
       "AutoCreateSubnetworks": true, 
       "CreationTimestamp": "2016-05-09T11:15:47.434-07:00", 
@@ -132,6 +138,9 @@ Cloud Auxiliary has support for Google Cloud Platform.
     b = get_bucket(bucket_name=BUCKET, **conn_details)
     print(json.dumps(b, indent=4, sort_keys=True))
 
+    # Flag options for Service Accounts are BASE, ALL (default).
+    # BASE and ALL are equivelant for this service.
+
     {
 	"Acl": [
           {

cloudaux/orchestration/aws/iam/role.py

@@ -2,8 +2,33 @@
 from cloudaux.aws.iam import get_role_managed_policies, get_role_inline_policies, get_role_instance_profiles
 from cloudaux.orchestration.aws import _get_name_from_structure, _conn_from_args
 from cloudaux.orchestration import modify
+from cloudaux.orchestration.flag_registry import FlagRegistry, Flags
 
 
+class RoleFlagRegistry(FlagRegistry):
+    from collections import defaultdict
+    r = defaultdict(list)
+
+
+FLAGS = Flags('BASE', 'MANAGED_POLICIES', 'INLINE_POLICIES', 'INSTANCE_PROFILES')
+
+
+@RoleFlagRegistry.register(flag=FLAGS.MANAGED_POLICIES, key='managed_policies')
+def get_managed_policies(role, **conn):
+    return get_role_managed_policies(role, **conn)
+
+
+@RoleFlagRegistry.register(flag=FLAGS.INLINE_POLICIES, key='inline_policies')
+def get_inline_policies(role, **conn):
+    return get_role_inline_policies(role, **conn)
+
+
+@RoleFlagRegistry.register(flag=FLAGS.INSTANCE_PROFILES, key='instance_profiles')
+def get_instance_profiles(role, **conn):
+    return get_role_instance_profiles(role, **conn)
+
+
+@RoleFlagRegistry.register(flag=FLAGS.BASE)
 def _get_base(role, **conn):
     """
     Determine whether the boto get_role call needs to be made or if we already have all that data
@@ -27,11 +52,12 @@ def _get_base(role, **conn):
 
     # cast CreateDate from a datetime to something JSON serializable.
     role.update(dict(CreateDate=str(role['CreateDate'])))
+    role['_version'] = 1
 
     return role
 
 
-def get_role(role, output='camelized', **conn):
+def get_role(role, output='camelized', flags=FLAGS.ALL, **conn):
     """
     Orchestrates all the calls required to fully build out an IAM Role in the following format:
 
@@ -55,13 +81,5 @@ def get_role(role, output='camelized', **conn):
     """
     role = modify(role, 'camelized')
     _conn_from_args(role, conn)
-    role = _get_base(role, **conn)
-    role.update(
-        {
-            'managed_policies': get_role_managed_policies(role, **conn),
-            'inline_policies': get_role_inline_policies(role, **conn),
-            'instance_profiles': get_role_instance_profiles(role, **conn),
-            '_version': 1
-        }
-    )
-    return modify(role, format=output)
+    RoleFlagRegistry.build_out(role, flags, role, **conn)
+    return modify(role, format=output)

cloudaux/orchestration/aws/iam/user.py

@@ -7,8 +7,48 @@
 from cloudaux.aws.iam import get_user_signing_certificates
 from cloudaux.orchestration.aws import _get_name_from_structure, _conn_from_args
 from cloudaux.orchestration import modify
+from cloudaux.orchestration.flag_registry import FlagRegistry, Flags
 
 
+class UserFlagRegistry(FlagRegistry):
+    from collections import defaultdict
+    r = defaultdict(list)
+
+
+FLAGS = Flags('BASE', 'ACCESS_KEYS', 'INLINE_POLICIES', 'MANAGED_POLICIES', 'MFA_DEVICES', 'LOGIN_PROFILE', 'SIGNING_CERTIFICATES')
+
+
+@UserFlagRegistry.register(flag=FLAGS.ACCESS_KEYS, key='access_keys')
+def get_access_keys(user, **conn):
+    return get_user_access_keys(user, **conn)
+
+
+@UserFlagRegistry.register(flag=FLAGS.INLINE_POLICIES, key='inline_policies')
+def get_inline_policies(user, **conn):
+    return get_user_inline_policies(user, **conn)
+
+
+@UserFlagRegistry.register(flag=FLAGS.MANAGED_POLICIES, key='managed_policies')
+def get_managed_policies(user, **conn):
+    return get_user_managed_policies(user, **conn)
+
+
+@UserFlagRegistry.register(flag=FLAGS.MFA_DEVICES, key='mfa_devices')
+def get_mfa_devices(user, **conn):
+    return get_user_mfa_devices(user, **conn)
+
+
+@UserFlagRegistry.register(flag=FLAGS.LOGIN_PROFILE, key='login_profile')
+def get_login_profile(user, **conn):
+    return get_user_login_profile(user, **conn)
+
+
+@UserFlagRegistry.register(flag=FLAGS.SIGNING_CERTIFICATES, key='signing_certificates')
+def get_signing_certificates(user, **conn):
+    return get_user_signing_certificates(user, **conn)
+
+
+@UserFlagRegistry.register(flag=FLAGS.BASE)
 def _get_base(user, **conn):
     base_fields = frozenset(['Arn', 'CreateDate', 'Path', 'UserId', 'UserName'])
     needs_base = False
@@ -27,10 +67,11 @@ def _get_base(user, **conn):
     if 'PasswordLastUsed' in user:
         user.update(dict(PasswordLastUsed=str(user['PasswordLastUsed'])))
 
+    user['_version'] = 1
     return user
 
 
-def get_user(user, output='camelized', **conn):
+def get_user(user, output='camelized', flags=FLAGS.ALL, **conn):
     """
     Orchestrates all the calls required to fully build out an IAM User in the following format:
 
@@ -55,16 +96,5 @@ def get_user(user, output='camelized', **conn):
     """
     user = modify(user, 'camelized')
     _conn_from_args(user, conn)
-    user = _get_base(user, **conn)
-    user.update(
-        {
-            'access_keys': get_user_access_keys(user, **conn),
-            'inline_policies': get_user_inline_policies(user, **conn),
-            'managed_policies': get_user_managed_policies(user, **conn),
-            'mfa_devices': get_user_mfa_devices(user, **conn),
-            'login_profile': get_user_login_profile(user, **conn),
-            'signing_certificates': get_user_signing_certificates(user, **conn),
-            '_version': 1
-        }
-    )
+    UserFlagRegistry.build_out(user, flags, user, **conn)
     return modify(user, format=output)