Add log groups and stub code for tailing logs

touchdown/aws/__init__.py

@@ -22,6 +22,7 @@
     elb,
     iam,
     kms,
+    logs,
     rds,
     route53,
     s3,
@@ -40,6 +41,7 @@
     'elb',
     'iam',
     'kms',
+    'logs',
     'rds',
     'route53',
     's3',

touchdown/aws/logs/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2015 Isotoma Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .group import LogGroup
+
+__all__ = [
+    'LogGroup',
+]

touchdown/aws/logs/group.py

@@ -0,0 +1,69 @@
+# Copyright 2015 Isotoma Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from touchdown.core.plan import Plan
+from touchdown.core import argument
+
+from ..account import Account
+from ..common import Resource, SimpleDescribe, SimpleApply, SimpleDestroy
+
+
+class LogGroup(Resource):
+
+    resource_name = "log_group"
+
+    name = argument.String(min=1, max=512, field="logGroupName")
+    retention = argument.Integer(
+        default=0,
+        choices=[
+            0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150,
+            180, 365, 400, 545, 731, 1827, 3653
+        ],
+    )
+
+    account = argument.Resource(Account)
+
+
+class Describe(SimpleDescribe, Plan):
+
+    resource = LogGroup
+    service_name = 'logs'
+    describe_action = 'describe_log_groups'
+    describe_envelope = "logGroups"
+    key = "logGroupName"
+
+    def get_describe_filters(self):
+        return {"logGroupNamePrefix": self.resource.name}
+
+
+class Apply(SimpleApply, Describe):
+
+    create_action = "create_log_group"
+    create_response = "nothing-useful"
+
+    def update_object(self):
+        for change in super(Apply, self).update_object():
+            yield change
+
+        if self.object.get('retentionInDays', 0) != self.resource.retention:
+            yield self.generic_action(
+                "Set log group retention to {} days".format(
+                    self.resource.retention
+                )
+            )
+
+
+class Destroy(SimpleDestroy, Describe):
+
+    destroy_action = "delete_log_group"

touchdown/aws/logs/tail.py

@@ -0,0 +1,56 @@
+# Copyright 2015 Isotoma Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code is not currently exposed publically. It is an example of how to
+# stream from a aws log using the FilterLogEvents API.
+
+import time
+
+from touchdown.core.datetime import parse_datetime_as_seconds
+
+
+def tail(runner, log_group, start=None, end=None, follow=False):
+    plan = runner.goal.get_plan(log_group)
+    client = plan.client
+
+    kwargs = {
+        'logGroupName': log_group.name,
+    }
+    if start:
+        kwargs['startTime'] = parse_datetime_as_seconds(start)
+    if end:
+        kwargs['endTime'] = parse_datetime_as_seconds(end)
+
+    def pull(kwargs, previous_events):
+        seen = set()
+        filters = {}
+        filters.update(kwargs)
+        results = client.filter_log_events(**filters)
+        while True:
+            for event in results.get('events', []):
+                seen.add(event['eventId'])
+                if event['eventId'] in previous_events:
+                    continue
+                print(u"[{logStreamName}] {message}".format(**event))
+                kwargs['startTime'] = event['timestamp']
+            if 'nextToken' not in results:
+                break
+            filters['nextToken'] = results['nextToken']
+            results = client.filter_log_events(**filters)
+        return seen
+
+    seen = pull(kwargs, set())
+    while follow:
+        seen = pull(kwargs, seen)
+        time.sleep(2)

touchdown/core/datetime.py

@@ -0,0 +1,94 @@
+# Copyright 2011-2015 Isotoma Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from datetime import datetime, timedelta, tzinfo
+import re
+
+try:
+    import pytz
+except ImportError:
+    pytz = None
+
+try:
+    from dateutil import parser
+except ImportError:
+    parser = None
+
+from touchdown.core import errors
+
+
+REGEX_DELTA = re.compile(
+    r'(\d+)\s?(m|minute|minutes|h|hour|hours|d|day|days|w|weeks|weeks)(?: ago)?'
+)
+
+UNITS = {
+    'm': 60,
+    'h': 60 * 60,
+    'd': 60 * 60 * 24,
+    'w': 60 * 60 * 24 * 7,
+}
+
+if not pytz:
+    class UTC(tzinfo):
+
+        def __repr__(self):
+            return "<UTC>"
+
+        def utcoffset(self, value):
+            return timedelta(0)
+
+        def tzname(self, value):
+            return "UTC"
+
+        def dst(self, value):
+            return timedelta(0)
+
+        def localize(self, value):
+            value.replace(tzinfo=self)
+
+    utc = UTC()
+else:
+    utc = pytz.utc
+
+
+def now():
+    return datetime.utcnow().replace(tzinfo=utc)
+
+
+def parse_datetime(value):
+    match = REGEX_DELTA.match(value)
+    if match:
+        amount, unit = match.groups()
+        return now() - timedelta(
+            seconds=int(amount) * UNITS[unit[0]],
+        )
+
+    if parser:
+        try:
+            return parser.parse(value)
+        except Exception:
+            raise errors.Error(
+                "Unable to parse {} as a date or time".format(value)
+            )
+
+    raise errors.Error(
+        "Unable to parse {} as a date or time".format(value)
+    )
+
+
+def parse_datetime_as_seconds(value):
+    date = parse_datetime(value)
+    return int(date.strftime("%s")) * 1000

touchdown/tests/test_core_datetime.py

@@ -0,0 +1,32 @@
+# Copyright 2015 Isotoma Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import mock
+import datetime
+
+from touchdown.core.datetime import parse_datetime, utc
+
+
+class TestDateTime(unittest.TestCase):
+
+    def test_5m_ago(self):
+        with mock.patch("touchdown.core.datetime.datetime") as mock_dt:
+            mock_dt.utcnow.return_value = datetime.datetime(2015, 5, 25, 14, 23, 46, 890132)
+            mock_dt.side_effect = datetime.datetime.now()
+
+            self.assertEqual(
+                parse_datetime("5m ago"),
+                datetime.datetime(2015, 5, 25, 14, 18, 46, 890132, tzinfo=utc)
+            )