Merge branch 'add-log-tailing'

PR #1413
Closes #1347

* add-log-tailing:
  Don't rely on parse_timestamp from botocore
  Add docs about relative time format
  Move 10 minute default for follow mode to options class
  Add changelog entry for log tail feature
  Restructure log following abstractions
  Added option to follow logs

.changes/next-release/80543201307-feature-logs-95988.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "logs",
+  "description": "Add support for tailing logs (#4)."
+}

chalice/awsclient.py

@@ -17,7 +17,7 @@
 import os
 import time
 import tempfile
-import datetime
+from datetime import datetime
 import zipfile
 import shutil
 import json
@@ -30,7 +30,10 @@
     RequestsConnectionError
 from botocore.vendored.requests.exceptions import ReadTimeout as \
     RequestsReadTimeout
+from botocore.utils import datetime2timestamp
 from typing import Any, Optional, Dict, Callable, List, Iterator, IO  # noqa
+from typing import Iterable  # noqa
+from mypy_extensions import TypedDict
 
 from chalice.constants import DEFAULT_STAGE_NAME
 from chalice.constants import MAX_LAMBDA_DEPLOYMENT_SIZE
@@ -41,6 +44,23 @@
 OptInt = Optional[int]
 OptStrList = Optional[List[str]]
 ClientMethod = Callable[..., Dict[str, Any]]
+CWLogEvent = TypedDict(
+    'CWLogEvent', {
+        'eventId': str,
+        'ingestionTime': datetime,
+        'logStreamName': str,
+        'message': str,
+        'timestamp': datetime,
+        'logShortId': str,
+    }
+)
+LogEventsResponse = TypedDict(
+    'LogEventsResponse', {
+        'events': List[CWLogEvent],
+        'nextToken': str,
+    }, total=False
+)
+
 
 _REMOTE_CALL_ERRORS = (
     botocore.exceptions.ClientError, RequestsConnectionError
@@ -668,8 +688,9 @@ def region_name(self):
         # type: () -> str
         return self._client('apigateway').meta.region_name
 
-    def iter_log_events(self, log_group_name, interleaved=True):
-        # type: (str, bool) -> Iterator[Dict[str, Any]]
+    def iter_log_events(self, log_group_name, start_time=None,
+                        interleaved=True):
+        # type: (str, Optional[datetime], bool) -> Iterator[CWLogEvent]
         logs = self._client('logs')
         paginator = logs.get_paginator('filter_log_events')
         pages = paginator.paginate(logGroupName=log_group_name,
@@ -685,7 +706,7 @@ def iter_log_events(self, log_group_name, interleaved=True):
             pass
 
     def _iter_log_messages(self, pages):
-        # type: (Iterator[Dict[str, Any]]) -> Iterator[Dict[str, Any]]
+        # type: (Iterable[Dict[str, Any]]) -> Iterator[CWLogEvent]
         for page in pages:
             events = page['events']
             for event in events:
@@ -699,8 +720,39 @@ def _iter_log_messages(self, pages):
                 yield event
 
     def _convert_to_datetime(self, integer_timestamp):
-        # type: (int) -> datetime.datetime
-        return datetime.datetime.fromtimestamp(integer_timestamp / 1000.0)
+        # type: (int) -> datetime
+        return datetime.utcfromtimestamp(integer_timestamp / 1000.0)
+
+    def filter_log_events(self,
+                          log_group_name,   # type: str
+                          start_time=None,  # type: Optional[datetime]
+                          next_token=None,  # type: Optional[str]
+                          ):
+        # type: (...) -> LogEventsResponse
+        logs = self._client('logs')
+        kwargs = {
+            'logGroupName': log_group_name,
+            'interleaved': True,
+        }
+        if start_time is not None:
+            kwargs['startTime'] = int(datetime2timestamp(start_time) * 1000)
+        if next_token is not None:
+            kwargs['nextToken'] = next_token
+        try:
+            response = logs.filter_log_events(**kwargs)
+        except logs.exceptions.ResourceNotFoundException:
+            # If there's no log messages yet then we'll just return
+            # an empty response.
+            return {'events': []}
+        # We want to convert the individual events that have integer
+        # types over to datetime objects so it's easier for us to
+        # work with.
+        self._convert_types_on_response(response)
+        return response
+
+    def _convert_types_on_response(self, response):
+        # type: (Dict[str, Any]) -> None
+        response['events'] = list(self._iter_log_messages([response]))
 
     def _client(self, service_name):
         # type: (str) -> Any

chalice/cli/__init__.py

@@ -24,7 +24,7 @@
 from chalice.cli.factory import CLIFactory
 from chalice.cli.factory import NoSuchFunctionError
 from chalice.config import Config  # noqa
-from chalice.logs import display_logs
+from chalice.logs import display_logs, LogRetrieveOptions
 from chalice.utils import create_zip_file
 from chalice.deploy.validate import validate_routes, validate_python_version
 from chalice.deploy.validate import ExperimentalFeatureError
@@ -363,10 +363,29 @@ def delete(ctx, profile, stage):
 @click.option('-n', '--name',
               help='The name of the lambda function to retrieve logs from.',
               default=DEFAULT_HANDLER_NAME)
+@click.option('-s', '--since',
+              help=('Only display logs since the provided time.  If the '
+                    '-f/--follow option is specified, then this value will '
+                    'default to 10 minutes from the current time.  Otherwise '
+                    'by default all log messages are displayed.  This value '
+                    'can either be an ISO8601 formatted timestamp or a '
+                    'relative time.  For relative times provide a number '
+                    'and a single unit.  Units can be "s" for seconds, '
+                    '"m" for minutes, "h" for hours, "d" for days, and "w" '
+                    'for weeks.  For example "5m" would indicate to display '
+                    'logs starting five minutes in the past.'),
+              default=None)
+@click.option('-f', '--follow/--no-follow',
+              default=False,
+              help=('Continuously poll for new log messages.  Note that this '
+                    'is a best effort attempt, and in certain cases can '
+                    'miss log messages.  This option is intended for '
+                    'interactive usage only.'))
 @click.option('--profile', help='The profile to use for fetching logs.')
 @click.pass_context
-def logs(ctx, num_entries, include_lambda_messages, stage, name, profile):
-    # type: (click.Context, int, bool, str, str, str) -> None
+def logs(ctx, num_entries, include_lambda_messages, stage,
+         name, since, follow, profile):
+    # type: (click.Context, int, bool, str, str, str, bool, str) -> None
     factory = ctx.obj['factory']  # type: CLIFactory
     factory.profile = profile
     config = factory.create_config_obj(stage, False)
@@ -375,9 +394,13 @@ def logs(ctx, num_entries, include_lambda_messages, stage, name, profile):
         lambda_arn = deployed.resource_values(name)['lambda_arn']
         session = factory.create_botocore_session()
         retriever = factory.create_log_retriever(
-            session, lambda_arn)
-        display_logs(retriever, num_entries, include_lambda_messages,
-                     sys.stdout)
+            session, lambda_arn, follow)
+        options = LogRetrieveOptions.create(
+            max_entries=num_entries,
+            since=since,
+            include_lambda_messages=include_lambda_messages,
+        )
+        display_logs(retriever, sys.stdout, options)
 
 
 @cli.command('gen-policy')

chalice/cli/factory.py

@@ -8,7 +8,7 @@
 import click
 from botocore.config import Config as BotocoreConfig
 from botocore.session import Session
-from typing import Any, Optional, Dict, MutableMapping  # noqa
+from typing import Any, Optional, Dict, MutableMapping, cast  # noqa
 
 from chalice import __version__ as chalice_version
 from chalice.awsclient import TypedAWSClient
@@ -20,7 +20,9 @@
 from chalice.constants import DEFAULT_STAGE_NAME
 from chalice.constants import DEFAULT_APIGATEWAY_STAGE_NAME
 from chalice.constants import DEFAULT_ENDPOINT_TYPE
-from chalice.logs import LogRetriever
+from chalice.logs import LogRetriever, LogEventGenerator
+from chalice.logs import FollowLogEventGenerator
+from chalice.logs import BaseLogEventGenerator
 from chalice import local
 from chalice.utils import UI  # noqa
 from chalice.utils import PipeReader  # noqa
@@ -190,10 +192,17 @@ def create_app_packager(self, config, package_format, template_format,
             config, package_format, template_format,
             merge_template=merge_template)
 
-    def create_log_retriever(self, session, lambda_arn):
-        # type: (Session, str) -> LogRetriever
+    def create_log_retriever(self, session, lambda_arn, follow_logs):
+        # type: (Session, str, bool) -> LogRetriever
         client = TypedAWSClient(session)
-        retriever = LogRetriever.create_from_lambda_arn(client, lambda_arn)
+        if follow_logs:
+            event_generator = cast(BaseLogEventGenerator,
+                                   FollowLogEventGenerator(client))
+        else:
+            event_generator = cast(BaseLogEventGenerator,
+                                   LogEventGenerator(client))
+        retriever = LogRetriever.create_from_lambda_arn(event_generator,
+                                                        lambda_arn)
         return retriever
 
     def create_stdin_reader(self):