Merge e7df0c5 into 7dfc3ff

paasta_tools/check_marathon_services_replication.py

@@ -35,6 +35,7 @@
 import logging
 import pysensu_yelp
 import service_configuration_lib
+from datetime import datetime, timedelta
 
 from paasta_tools import marathon_tools
 from paasta_tools import mesos_tools
@@ -165,9 +166,12 @@ def get_healthy_marathon_instances_for_short_app_id(client, app_id):
     tasks = client.list_tasks()
     tasks_for_app = [task for task in tasks if task.app_id.startswith('/%s' % app_id)]
 
+    one_minute_ago = datetime.now() - timedelta(minutes=1)
+
     healthy_tasks = []
     for task in tasks_for_app:
-        if all([health_check_result.alive for health_check_result in task.health_check_results]):
+        if all([health_check_result.alive for health_check_result in task.health_check_results]) \
+                and task.started_at < one_minute_ago:
             healthy_tasks.append(task)
     return len(healthy_tasks)
 

tests/test_check_marathon_services_replication.py

@@ -17,6 +17,8 @@
 
 import pysensu_yelp
 
+from datetime import datetime, timedelta
+
 import check_marathon_services_replication
 from paasta_tools.marathon_tools import MarathonServiceConfig
 from paasta_tools.smartstack_tools import DEFAULT_SYNAPSE_PORT
@@ -550,7 +552,7 @@ def test_check_service_replication_for_non_smartstack():
             expected_count=100)
 
 
-def test_get_healthy_marathon_instances_for_short_app_id():
+def test_get_healthy_marathon_instances_for_short_app_id_correctly_counts_alive_tasks():
     fake_client = mock.Mock()
     fakes = []
     for i in range(0, 4):
@@ -568,6 +570,29 @@ def test_get_healthy_marathon_instances_for_short_app_id():
     assert actual == 2
 
 
+def test_get_healthy_marathon_instances_for_short_app_id_considers_new_tasks_not_healthy_yet():
+    fake_client = mock.Mock()
+    fakes = []
+    one_minute = timedelta(minutes=1)
+    for i in range(0, 4):
+        fake_task = mock.Mock()
+        fake_task.app_id = '/service.instance.foo%s.bar%s' % (i, i)
+
+        # when i == 0, produces a task that has just started (not healthy yet)
+        # otherwise produces a task that was started over a minute ago (healthy)
+        fake_task.started_at = datetime.now() - one_minute * i
+        mock_result = mock.Mock()
+        mock_result.alive = True
+        fake_task.health_check_results = [mock_result]
+        fakes.append(fake_task)
+    fake_client.list_tasks.return_value = fakes
+    actual = check_marathon_services_replication.get_healthy_marathon_instances_for_short_app_id(
+        fake_client,
+        'service.instance',
+    )
+    assert actual == 3
+
+
 @mock.patch('check_marathon_services_replication.send_event_if_under_replication')
 @mock.patch('check_marathon_services_replication.get_healthy_marathon_instances_for_short_app_id')
 def test_check_healthy_marathon_tasks_for_service_instance(mock_healthy_instances,