Fix health state not recovering after a occured failure (#54)

When an exception is raised inside the method `autoscale`, the health
endpoint indicates this by responding with status code "503". If
kube-aws-autoscaler recovers on a subsequent run, the health endpoint
still responds with status code "503".
These changes fix this behaviour by setting the global variable `Healthy`
to `True` after each run of autoscale if no exception occurred.

kube_aws_autoscaler/main.py

@@ -460,6 +460,7 @@ def main():
         t = Thread(target=start_health_endpoint, daemon=True)
         t.start()
 
+    global Healthy
     while True:
         try:
             autoscale(buffer_percentage, buffer_fixed,
@@ -468,8 +469,8 @@ def main():
                       buffer_spare_nodes=args.buffer_spare_nodes,
                       include_master_nodes=args.include_master_nodes, dry_run=args.dry_run,
                       disable_scale_down=args.no_scale_down)
+            Healthy = True
         except Exception:
-            global Healthy
             Healthy = False
             logger.exception('Failed to autoscale')
         if args.once:

tests/test_autoscaler.py

@@ -10,6 +10,7 @@
                                       resize_auto_scaling_groups,
                                       scaling_activity_in_progress,
                                       slow_down_downscale, app)
+import kube_aws_autoscaler.main
 
 
 def test_parse_resource():
@@ -511,3 +512,11 @@ def test_start_health_endpoint():
     flask.testing = True
     response = flask.get('/healthz')
     assert response.status_code == 503
+
+def test_endpoint_healthy_state(monkeypatch):
+    kube_aws_autoscaler.main.Healthy = False
+    autoscale = MagicMock()
+    monkeypatch.setattr('kube_aws_autoscaler.main.autoscale', autoscale)
+    monkeypatch.setattr('sys.argv', ['foo', '--once', '--dry-run'])
+    main()
+    assert kube_aws_autoscaler.main.Healthy == True