merge master

c7n/filters/offhours.py

@@ -35,19 +35,18 @@
 We provide an `onhour` and `offhour` time filter, each should be used in a
 different policy, they support the same configuration options
 
- - :weekends: default true, whether to leave resources off for the weekend
- - :weekend-only: default false, whether to turn the resource off only on the
-   weekend
- - :default_tz: which tz to utilize when evaluating time
- - :tag: default maid_offhours, which resource tag key to look for the
+ - **weekends**: default true, whether to leave resources off for the weekend
+ - **weekend-only**: default false, whether to turn the resource off only on
+   the weekend
+ - **default_tz**: which timezone to utilize when evaluating time
+ - **tag**: default maid_offhours, which resource tag key to look for the
    resource's schedule.
- - :opt-out: applies the default schedule to resource which do not specify
-   any value. a value of `off` to disable/exclude the resource.
+ - **opt-out**: applies the default schedule to resource which do not specify
+   any value.  a value of `off` to disable/exclude the resource.
 
 The default off hours and on hours are specified per the policy configuration
 along with the opt-in/opt-out behavior. Resources can specify the timezone
-that they wish to have this scheduled utilized with::
-
+that they wish to have this scheduled utilized with.
 
 Tag Based Configuration
 =======================
@@ -232,8 +231,8 @@ def process(self, resources, event=None):
 
     def __call__(self, i):
         value = self.get_tag_value(i)
-        # Sigh delayed init, due to circle dep, process/init would be better but
-        # unit testing is calling this direct.
+        # Sigh delayed init, due to circle dep, process/init would be better
+        # but unit testing is calling this direct.
         if self.id_key is None:
             self.id_key = (
                 self.manager is None and 'InstanceId'
@@ -270,9 +269,10 @@ def process_resource_schedule(self, i, value):
             # respect timezone from tag
             raw_data = self.parser.raw_data(value)
             if 'tz' in raw_data:
-                self.default_tz = raw_data['tz']
-                self.default_schedule = self.get_default_schedule()
-            schedule = self.default_schedule
+                schedule = dict(self.default_schedule)
+                schedule['tz'] = raw_data['tz']
+            else:
+                schedule = self.default_schedule
         else:
             schedule = None
 
@@ -308,13 +308,14 @@ def get_tag_value(self, i):
         for t in i.get('Tags', ()):
             if t['Key'].lower() == self.tag_key:
                 found = t['Value']
+                break
         if found is False:
             return False
         # utf8, or do translate tables via unicode ord mapping
         value = found.lower().encode('utf8')
         # Some folks seem to be interpreting the docs quote marks as
         # literal for values.
-        value = value.strip("'").strip('"').translate(None, ' ')
+        value = value.strip("'").strip('"')
         return value
 
     @classmethod
@@ -427,11 +428,15 @@ def __init__(self, default_schedule):
 
     @staticmethod
     def raw_data(tag_value):
-        """convert the tag to a dictionary, taking values as is"""
-        data = {}
+        """convert the tag to a dictionary, taking values as is
 
+        This method name and purpose are opaque...  and not true.
+        """
+        data = {}
+        pieces = []
+        for p in tag_value.split(' '):
+            pieces.extend(p.split(';'))
         # parse components
-        pieces = tag_value.split(';')
         for piece in pieces:
             kv = piece.split('=')
             # components must by key=value

c7n/resources/ec2.py

@@ -87,7 +87,7 @@ def augment(self, resources):
 
         Apriori we may be using custodian to ensure tags (including
         name), so there isn't a good default to ensure that we will
-        always get tags from describe_ calls.
+        always get tags from describe_x calls.
         """
 
         # First if we're in event based lambda go ahead and skip this,

c7n/resources/vpc.py

@@ -221,7 +221,7 @@ class SGPermission(Filter):
 
     As well for assertions that a ingress/egress permission only matches
     a given set of ports, *note* onlyports is an inverse match, it matches
-    when a permission includes ports outside of the specified set:
+    when a permission includes ports outside of the specified set::
 
       - type: egress
         OnlyPorts: [22, 443, 80]

c7n/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version = "0.8.20.2"
+version = "0.8.21.0"

docs/source/contribute.rst

@@ -0,0 +1,11 @@
+.. _contribute:
+
+Contributing to Cloud Custodian
+===============================
+
+If you're interested in contributing to Cloud Custodian development, welcome! Please take a few minutes to familiarize yourself with the project and our expectations.
+
+Developer install
+-----------------
+
+First, you'll want to perform a :ref:`developer install <developer>` and familiarize yourself with the tests.

docs/source/quickstart/developer.rst → docs/source/developer.rst

@@ -1,5 +1,7 @@
-Developer Install
-=================
+.. _developer:
+
+Developer Install and Testing
+=============================
 
 
 Requirements

docs/source/index.rst

@@ -12,34 +12,42 @@ Organizations can use Custodian to manage their AWS environments by ensuring com
 
 Navigate below and get started with Cloud Custodian!
 
-Overview
-~~~~~~~~
-
 .. toctree::
+   :maxdepth: 2
+   :caption: Introduction
 
    overview/index
-
-Quickstart
-~~~~~~~~~~
+   quickstart/index
+   quickstart/usage
 
 .. toctree::
+   :maxdepth: 2
+   :caption: Examples
 
-   quickstart/index
+   quickstart/offhours
+   quickstart/tagCompliance
+   usecases/index
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Working with AWS Lambda
 
-Sample Use Cases
-~~~~~~~~~~~~~~~~
+   policy/lambda
+   policy/mu
 
 .. toctree::
+   :maxdepth: 2
+   :caption: Policies reference
 
-  usecases/index
-
-Policy
-~~~~~~
+   policy/index.rst
+   filters
 
 .. toctree::
+   :maxdepth: 2
+   :caption: Contributing
 
-   policy/index
+   contribute
+   developer   
 
 
 Indices and tables

docs/source/overview/glossary.rst

@@ -0,0 +1,61 @@
+.. _glossary:
+
+Basic concepts and terms
+========================
+
+Cloud Custodian works with the following basic concepts, terms, and relationships between them.
+
+Policy - :py:class:`c7n.policy`
+  Defined in yaml, specifies a set of filters and actions to take
+  on a given AWS resource type.
+
+Resource - :py:class:`c7n.manager.ResourceManager`
+  Provides for retrieval of a resources of a given type (typically via AWS API)
+  and defines the vocabulary of filters and actions that can be used on those
+  resource (e.g., ASG, S3, EC2, ELBs, etc).
+
+Mode
+  Provides for retrieval of a resources of a given type (typically via AWS API) and defines the vocabulary of filters and actions that can be used on those resource. Example resource types are Auto Scaling Groups, S3 buckets, EC2 instances, Elastic Load Balancers, etc).
+
+.. code-block:: yaml
+
+   mode:
+     type: cloudtrail
+     events:
+       - RunInstances
+
+Filters - :py:class:`c7n.filters`
+  Given a set of resources, how we filter to the subset that we're
+  interested in operating on. The :ref:`filtering language<filters>` has some
+  default behaviors across resource types like value filtering with JMESPath
+  expressions against the JSON representation of a resource, as well as
+  specific filters for particular resources types (instance age,
+  tag count, etc).
+
+.. code-block:: yaml
+
+   filters:
+     - "tag:aws:autoscaling:groupName": absent
+     - type: ebs
+       key: Encrypted
+       value: false
+       skip-devices:
+         - "/dev/sda1"
+         - "/dev/xvda"
+     - type: event
+       key: "detail.userIdentity.sessionContext.sessionIssuer.userName"
+       value: "SuperUser"
+       op: ne
+
+Actions - :py:class:`c7n.actions`
+  A verb to use on a given resource, i.e. stop, start, suspend,
+  delete, encrypt, etc.
+
+.. code-block:: yaml
+
+   actions:
+     - type: tag
+       key: c7n_status
+       value: "Unencrypted EBS! Please recreate with Encryption)"
+     - type: terminate
+       force: true

docs/source/overview/index.rst

@@ -5,5 +5,7 @@ Overview
 .. toctree::
 
    capabilities
+   glossary
    roadmap
    comparisons
+   deployment

docs/source/policy/index.rst

@@ -1,126 +1,10 @@
 .. _policy:
 
-Policy
-------
-
-Sample Policy
-=============
-
-In this sample policy we are querying for only running EC2
-instances. Based on the list that comes back we are then filtering for EC2
-instances that are: not part of an Auto Scaling Group (ASG), not
-already marked for an operation, have less than 10 tags, and are missing one or more
-of the required tags. Once Custodian has filtered the list, it will
-mark all EC2 instances that match the above criteria with a tag. That tag specifies an action
-that will take place at a certain time. This policy is one of three that
-are needed to manage tag compliance. The other two policies in this set are, 1)
-checking to see if the tags have been corrected before the four day period
-is up, and 2) performing the operation of stopping all instances
-with the status to be stopped on that particular day.
-
-.. code-block:: yaml
-   :linenos:
-
-   - name: ec2-tag-compliance-mark
-     resource: ec2
-     comment: |
-       Mark non-compliant, Non-ASG EC2 instances with stoppage in 4 days
-     query:
-       - instance-state-name: running ──▶ Only apply filter to running instances
-     filters:
-   ▣──────  - "tag:aws:autoscaling:groupName": absent
-   │▣─────  - "tag:c7n_status": absent
-   │ │ ▣─── - type: tag-count
-   │ │ │    - or:                           ─┐
-   │ │ │      - "tag:Owner": absent          ├─If any of these tags are
-   │ │ │      - "tag:CostCenter": absent     │ missing, then select instance
-   │ │ │      - "tag:Project": absent       ─┘
-   │ │ │
-   │ │ │  actions: ─────────────────▶ For selected instances, run this action
-   │ │ │    - type: mark-for-op ────▶ Mark instance for operation
-   │ │ │      op: stop ─────────────▶ Stop instance
-   │ │ │      days: 4 ──────────────▶ After 4 days
-   │ │ │
-   │ │ ▣────▶ If instance has 10 tags, skip
-   │ ▣──────▶ If instance already has a c7n_status, skip
-   ▣────────▶ If instance is part of an ASG, skip
-
-
-Terms
-=====
-
-Policy - :py:class:`c7n.policy`
-  Defined in yaml, specifies a set of filters and actions to take
-  on a given AWS resource type.
-
-Resource - :py:class:`c7n.manager.ResourceManager`
-  Provides for retrieval of a resources of a given type (typically via AWS API)
-  and defines the vocabulary of filters and actions that can be used on those
-  resource (e.g., ASG, S3, EC2, ELBs, etc).
-
-Mode
-  Provides for retrieval of a resources of a given type (typically via AWS API) and defines the vocabulary of filters and actions that can be used on those resource. Example resource types are Auto Scaling Groups, S3 buckets, EC2 instances, Elastic Load Balancers, etc).
-
-.. code-block:: yaml
-
-   mode:
-     type: cloudtrail
-     events:
-       - RunInstances
-
-Filters - :py:class:`c7n.filters`
-  Given a set of resources, how we filter to the subset that we're
-  interested in operating on. The :ref:`filtering language<filters>` has some
-  default behaviors across resource types like value filtering with JMESPath
-  expressions against the JSON representation of a resource, as well as
-  specific filters for particular resources types (instance age,
-  tag count, etc).
-
-.. code-block:: yaml
-
-   filters:
-     - "tag:aws:autoscaling:groupName": absent
-     - type: ebs
-       key: Encrypted
-       value: false
-       skip-devices:
-         - "/dev/sda1"
-         - "/dev/xvda"
-     - type: event
-       key: "detail.userIdentity.sessionContext.sessionIssuer.userName"
-       value: "SuperUser"
-       op: ne
-
-Actions - :py:class:`c7n.actions`
-  A verb to use on a given resource, i.e. stop, start, suspend,
-  delete, encrypt, etc.
-
-.. code-block:: yaml
-
-   actions:
-     - type: tag
-       key: c7n_status
-       value: "Unencrypted EBS! Please recreate with Encryption)"
-     - type: terminate
-       force: true
-
-
-Real-time Policies
-==================
-  .. toctree::
-
-     lambda
-     mu
-
-Filters
-=======
-  .. toctree::
-
-     filters
-     usage
-
 Resources and Actions
 =====================
+
+These topics provide detailed reference information about the filters and actions that you can specify in policies against individual resources.
+
 .. toctree::
   :maxdepth: 2
   :titlesonly: