Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Efficient deploy log fetching #1768

Merged
merged 9 commits into from
Feb 12, 2020
Merged

Efficient deploy log fetching #1768

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c12b5d9
When reached old entries, skip and go next iteration.
Jan 31, 2020
0d154b2
Add comment.
Feb 1, 2020
572eca5
reformatted with black
Feb 1, 2020
e67f09b
shorten sentence to pass lint check.
Feb 1, 2020
a425e82
Add unit test for describe_stack_events(), checking old event skipping.
Feb 8, 2020
0468f31
Fix the problem with new stack.
Feb 8, 2020
ad8963a
Merge branch 'develop' into efficient-deploy-log-fetching
sriram-mv Feb 10, 2020
f592aaa
Merge branch 'develop' into efficient-deploy-log-fetching
jfuss Feb 11, 2020
c04c86a
Merge branch 'develop' into efficient-deploy-log-fetching
jfuss Feb 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions samcli/lib/deploy/deployer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,10 +338,14 @@ def describe_stack_events(self, stack_name, time_stamp_marker, **kwargs):
paginator = self._client.get_paginator("describe_stack_events")
response_iterator = paginator.paginate(StackName=stack_name)
stack_status = describe_stacks_resp["Stacks"][0]["StackStatus"]
latest_time_stamp_marker = time_stamp_marker
for event_items in response_iterator:
for event in event_items["StackEvents"]:
if event["EventId"] not in events and utc_to_timestamp(event["Timestamp"]) > time_stamp_marker:
events.add(event["EventId"])
latest_time_stamp_marker = max(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand the logic here, could you add a unit test to showcase this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the review!
I've added the unit test. Could you review it?

latest_time_stamp_marker, utc_to_timestamp(event["Timestamp"])
)
row_color = self.deploy_color.get_stack_events_status_color(status=event["ResourceStatus"])
pprint_columns(
columns=[
Expand All @@ -357,6 +361,14 @@ def describe_stack_events(self, stack_name, time_stamp_marker, **kwargs):
columns_dict=DESCRIBE_STACK_EVENTS_DEFAULT_ARGS.copy(),
color=row_color,
)
# Skip already shown old event entries
elif utc_to_timestamp(event["Timestamp"]) <= time_stamp_marker:
time_stamp_marker = latest_time_stamp_marker
break
else: # go to next loop if not break from inside loop
time_stamp_marker = latest_time_stamp_marker # update marker if all events are new
continue
break # reached here only if break from inner loop!

if self._check_stack_complete(stack_status):
stack_change_in_progress = False
Expand Down
72 changes: 72 additions & 0 deletions tests/unit/lib/deploy/test_deployer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,78 @@ def test_describe_stack_events(self, patched_time):

self.deployer.describe_stack_events("test", time.time() - 1)

@patch("time.sleep")
@patch("samcli.lib.deploy.deployer.pprint_columns")
def test_describe_stack_events_skip_old_event(self, patched_pprint_columns, patched_time):
current_timestamp = datetime.utcnow()

self.deployer._client.describe_stacks = MagicMock(
side_effect=[
{"Stacks": [{"StackStatus": "CREATE_IN_PROGRESS"}]},
{"Stacks": [{"StackStatus": "CREATE_IN_PROGRESS"}]},
{"Stacks": [{"StackStatus": "CREATE_COMPLETE_CLEANUP_IN_PROGRESS"}]},
{"Stacks": [{"StackStatus": "CREATE_COMPLETE"}]},
]
)
sample_events = [
{
"StackEvents": [
{
"EventId": str(uuid.uuid4()),
"Timestamp": current_timestamp,
"ResourceStatus": "CREATE_IN_PROGRESS",
"ResourceType": "s3",
"LogicalResourceId": "mybucket",
}
]
},
{
"StackEvents": [
{
"EventId": str(uuid.uuid4()),
"Timestamp": current_timestamp + timedelta(seconds=10),
"ResourceStatus": "CREATE_IN_PROGRESS",
"ResourceType": "kms",
"LogicalResourceId": "mykms",
}
]
},
{
"StackEvents": [
{
"EventId": str(uuid.uuid4()),
"Timestamp": current_timestamp + timedelta(seconds=20),
"ResourceStatus": "CREATE_COMPLETE",
"ResourceType": "s3",
"LogicalResourceId": "mybucket",
}
]
},
{
"StackEvents": [
{
"EventId": str(uuid.uuid4()),
"Timestamp": current_timestamp + timedelta(seconds=30),
"ResourceStatus": "CREATE_COMPLETE",
"ResourceType": "kms",
"LogicalResourceId": "mykms",
}
]
},
]
invalid_event = {"StackEvents": [{}]} # if deployer() loop read this, KeyError would raise
self.deployer._client.get_paginator = MagicMock(
side_effect=[
MockPaginator([sample_events[0]]),
MockPaginator([sample_events[1], sample_events[0], invalid_event]),
MockPaginator([sample_events[2], sample_events[1], invalid_event]),
MockPaginator([sample_events[3], sample_events[2], invalid_event]),
]
)

self.deployer.describe_stack_events("test", time.time() - 1)
self.assertEqual(patched_pprint_columns.call_count, 4)

@patch("samcli.lib.deploy.deployer.math")
@patch("time.sleep")
def test_describe_stack_events_exceptions(self, patched_time, patched_math):
Expand Down