Merge fb12146 into 420caf0

chalicelib/app_utils.py

@@ -131,12 +131,13 @@ def check_authorization(request_dict, env=None):
                 return False  # we have no env to check auth
             # leeway accounts for clock drift between us and auth0
             payload = jwt.decode(token, b64decode(auth0_secret, '-_'), audience=auth0_client, leeway=30)
-            env_info = init_environments(env)
-            user_res = ff_utils.get_metadata('users/' + payload.get('email').lower(),
-                                            ff_env=env_info[env]['ff_env'], add_on='frame=object')
-            if 'admin' in user_res['groups'] and payload.get('email_verified'):
-                # fully authorized
-                return True
+            for env_info in init_environments(env).values():
+                user_res = ff_utils.get_metadata('users/' + payload.get('email').lower(),
+                                            ff_env=env_info['ff_env'], add_on='frame=object')
+                if not ('admin' in user_res['groups'] and payload.get('email_verified')):
+                    # if unauthorized for one, unauthorized for all
+                    return False
+            return True
         except:
             pass
     return False

chalicelib/check_setup.json

@@ -1276,10 +1276,16 @@
         "group": "Elasticsearch checks",
         "schedule" : {
             "morning_checks": {
-                "mastertest": {
+                "all": {
                     "kwargs": {"primary": true}
                 }
             }
         }
+    },
+    "clean_s3_es_checks" : {
+        "title": "Wipe Checks Older Than One Month",
+        "group": "System checks",
+        "schedule": {},
+        "display": ["all"]
     }
 }

chalicelib/check_utils.py

@@ -232,16 +232,25 @@ def get_check_results(connection, checks=[], use_latest=False):
                 found = tempCheck.get_latest_result()
             else:
                 found = tempCheck.get_primary_result()
-                # checks with no records will return None. Skip IGNORE checks
-                if found and found.get('status') != 'IGNORE':
-                    check_results.append(found)
+            # checks with no records will return None. Skip IGNORE checks
+            if found and found.get('status') != 'IGNORE':
+                check_results.append(found)
+            if not found: # add placeholder check so not yet run checks are still rendered on the UI
+                placeholder = {
+                    'name': check_name,
+                    'uuid': datetime.date(1000, 1, 1).strftime('%Y-%m-%dT%H:%M:%S.%f'), # test compatibility
+                    'status': 'WARN',
+                    'summary': 'Check has not yet run',
+                    'description': 'Check has not yet run'
+                }
+                check_results.append(placeholder)
+
     else:
         if use_latest:
-            check_results = connection.connections['es'].get_main_page_checks(primary=False)
-            check_results = list(filter(lambda obj: obj['name'] in checks, check_results))
-        else: # also filter by IGNORE on primaries
-            check_results = connection.connections['es'].get_main_page_checks()
-            check_results = list(filter(lambda obj: obj['status'] != 'IGNORE' and obj['name'] in checks, check_results))
+            check_results = connection.connections['es'].get_main_page_checks(checks, primary=False)
+        else:
+            check_results = connection.connections['es'].get_main_page_checks(checks)
+        check_results = list(filter(lambda obj: obj['status'] != 'IGNORE' and obj['name'] in checks, check_results))
 
     # sort them by status and then alphabetically by check_setup title
     stat_order = ['ERROR', 'FAIL', 'WARN', 'PASS']

chalicelib/checks/es_checks.py

@@ -67,25 +67,26 @@ def migrate_checks_to_es(connection, **kwargs):
     action.output = action_logs
     return action
 
-@check_function(timeout=270)
+@check_function(timeout=270, days=30, to_clean=None)
 def clean_s3_es_checks(connection, **kwargs):
     """
     Cleans old checks from both s3 and es older than one month. Must be called
     from a specific check as it will take too long otherwise.
     """
     check_to_clean = kwargs.get('to_clean')
-    time_limit = 270 if kwargs.get('timeout') is None else kwargs.get('timeout')
-    check = CheckResult(connection, check_to_clean)
+    time_limit = kwargs.get('timeout')
+    days_back = kwargs.get('days')
+    check = CheckResult(connection, 'clean_s3_es_checks')
     full_output = {}
-    if not check_to_clean:
-        check.status = 'FAIL'
+    if check_to_clean is None:
+        check.status = 'WARN'
         check.summary = check.description = 'A check must be given to be cleaned'
         check.full_output = full_output
         return check
-    s3 = connection.connections['s3']
-    es = connection.connections['es']
-    one_month_ago = datetime.datetime.utcnow() - datetime.timedelta(days=30)
-    n_deleted_s3, n_deleted_es = action.delete_results(prior_date=one_month_ago, timeout=time_limit)
+    clean_check = CheckResult(connection, check_to_clean)
+    past_date = datetime.datetime.utcnow() - datetime.timedelta(days=days_back)
+    n_deleted_s3, n_deleted_es = clean_check.delete_results(prior_date=past_date, timeout=time_limit)
+    full_output['check_cleared'] = check_to_clean
     full_output['n_deleted_s3'] = n_deleted_s3
     full_output['n_deleted_es'] = n_deleted_es
     check.status = 'DONE'

chalicelib/es_connection.py

@@ -8,6 +8,7 @@
 from elasticsearch_dsl import Search
 from dcicutils import es_utils
 from .utils import load_json
+import datetime
 import json
 import time
 
@@ -181,7 +182,7 @@ def get_result_history(self, prefix, start, limit):
         search.update_from_dict(doc)
         return self.search(search)
 
-    def get_main_page_checks(self, primary=True):
+    def get_main_page_checks(self, checks=None, primary=True):
         """
         Gets all checks for the main page. If primary is true then all checks will
         be primary, otherwise we use latest.
@@ -207,7 +208,23 @@ def get_main_page_checks(self, primary=True):
         }
         search = Search(using=self.es, index=self.index)
         search.update_from_dict(doc)
-        return self.search(search)
+        raw_result = self.search(search)
+        if checks is not None:
+            # figure out which checks we didn't find, add a placeholder check so
+            # that check is still rendered on the UI
+            raw_result = list(filter(lambda res: res['name'] in checks, raw_result))
+            found_checks = set(res['name'] for res in raw_result)
+            for check_name in checks:
+                if check_name not in found_checks:
+                    placeholder = {
+                        'name': check_name,
+                        'uuid': datetime.date(1000, 1, 1).strftime('%Y-%m-%dT%H:%M:%S.%f'), # test compatibility
+                        'status': 'WARN',
+                        'summary': 'Check has not yet run',
+                        'description': 'Check has not yet run'
+                    }
+                    raw_result.append(placeholder)
+        return raw_result
 
     def list_all_keys(self):
         """

dev-requirements.txt

@@ -1,4 +1,4 @@
--e git+git://github.com/4dn-dcic/chalice.git#egg=chalice
+chalice==1.11.1
 pytest
 pytest-cov
 pytest-runner

tests/test_app_routes.py

@@ -8,7 +8,7 @@ def test_view_foursight(self):
         res = app_utils.view_foursight(self.environ) # not is_admin
         assert (res.headers == {u'Content-Type': u'text/html'})
         assert (res.status_code == 200)
-        assert (set(res.to_dict().keys()) == set(['body', 'headers', 'statusCode']))
+        assert (set(res.to_dict().keys()) == set(['body', 'headers', 'statusCode', 'multiValueHeaders']))
         assert ('<!DOCTYPE html>' in res.body)
         assert ('Foursight' in res.body)
         assert  ('Not logged in as admin' in res.body)

tests/test_app_utils.py

@@ -76,7 +76,7 @@ def test_check_jwt_authorization(self):
         """ Tests same functionality as above except with a valid jwt """
         from unittest import mock
         payload1 = {
-            "email": "William_Ronchetti@hms.harvard.edu",  # use something else?
+            "email": "carl_vitzthum@hms.harvard.edu",
             "email_verified": True,
             "sub": "1234567890",
             "name": "Dummy",
@@ -86,6 +86,14 @@ def test_check_jwt_authorization(self):
             with mock.patch('jwt.decode', return_value=payload1):
                 auth = app_utils.check_authorization({}, env='mastertest')
             assert auth
+        with mock.patch('chalicelib.app_utils.get_jwt', return_value='token'):
+            with mock.patch('jwt.decode', return_value=payload1):
+                auth = app_utils.check_authorization({}, env='all') # test all
+            assert auth
+        with mock.patch('chalicelib.app_utils.get_jwt', return_value='token'):
+            with mock.patch('jwt.decode', return_value=payload1):
+                auth = app_utils.check_authorization({}, env='data,staging') # test more than one
+            assert auth
             # Unverified email should fail
             payload2 = {
                 "email": "william_ronchetti@hms.harvard.edu",

tests/test_check_utils.py

@@ -161,9 +161,10 @@ def test_get_check_results(self, use_es):
         one_res = check_utils.get_check_results(self.connection, checks=['indexing_progress'])
         assert (len(one_res) == 1)
         assert (one_res[0]['name'] == 'indexing_progress')
-        # bad check name
+        # bad check name, will now return a placeholder so len should be 1
         test_res = check_utils.get_check_results(self.connection, checks=['not_a_real_check'])
-        assert (len(test_res) == 0)
+        assert (len(test_res) == 1)
+        assert test_res[0]['description'] == 'Check has not yet run'
 
     def test_get_grouped_check_results(self):
         grouped_results = check_utils.get_grouped_check_results(self.connection)

tests/test_es_connection.py

@@ -124,6 +124,12 @@ def test_get_checks(self, type):
         else:
             res = self.es.get_main_page_checks(primary=False)
         assert len(res) == 2
+        checks_to_get = ['page_children_routes']
+        if type == 'primary':
+            res = self.es.get_main_page_checks(checks=checks_to_get)
+        else:
+            res = self.es.get_main_page_checks(checks=checks_to_get, primary=False)
+        assert len(res) == 1
         self.es.delete_keys(['page_children_routes/'+ type + '.json',
                              'check_status_mismatch/' + type + '.json'])
         self.es.refresh_index()