Update tests

The-Academic-Observatory · May 3, 2023 · b36e209 · b36e209
1 parent 7689fd9
commit b36e209
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 12 deletions.
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -47,9 +47,9 @@ jobs:
           TEST_AZURE_CONTAINER_NAME: ${{ secrets.TEST_AZURE_CONTAINER_NAME }}
           TEST_AZURE_CONTAINER_SAS_TOKEN: ${{ secrets.TEST_AZURE_CONTAINER_SAS_TOKEN }}
           TEST_AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.TEST_AZURE_STORAGE_ACCOUNT_NAME }}
-          TEST_AWS_KEY_ID: ${{ secrets.TEST_AWS_KEY_ID }}
-          TEST_AWS_SECRET_KEY: ${{ secrets.TEST_AWS_SECRET_KEY }}
-          TEST_AWS_BUCKET_NAME: ${{ secrets.TEST_AWS_BUCKET_NAME }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           TEST_GCP_DATA_LOCATION: ${{ secrets.TEST_GCP_DATA_LOCATION }}
           TEST_GCP_BUCKET_NAME: ${{ secrets.TEST_GCP_BUCKET_NAME }}
           TEST_GCP_PROJECT_ID: ${{ secrets.TEST_GCP_PROJECT_ID }}

diff --git a/observatory-platform/observatory/platform/observatory_environment.py b/observatory-platform/observatory/platform/observatory_environment.py
@@ -73,7 +73,7 @@
 from datetime import datetime, timedelta
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 from multiprocessing import Process
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Set
 
 import boto3
 import croniter
@@ -600,22 +600,24 @@ def create(self, task_logging: bool = False):
                     self.elastic_env.stop()
 
 
-def load_json(file_path: str):
+def load_and_parse_json(file_path: str, date_fields: Set[str] = None):
     """Load a JSON file for testing purposes. It parses dates and datetimes into pendulum instances."""
 
     def parse_datetime(obj):
         for key, value in obj.items():
-            if isinstance(value, str):
-                try:
-                    obj[key] = pendulum.from_format(value, "YYYY-MM-DD").date()
-                except (ValueError, TypeError):
+            # Try to parse into a date or datetime
+            if key in date_fields:
+                if isinstance(value, str):
                     try:
-                        obj[key] = pendulum.from_format(value, "YYYYMMDD").date()
+                        obj[key] = pendulum.from_format(value, "YYYY-MM-DD").date()
                     except (ValueError, TypeError):
                         try:
-                            obj[key] = pendulum.parse(value)
+                            obj[key] = pendulum.from_format(value, "YYYYMMDD").date()
                         except (ValueError, TypeError):
-                            pass
+                            try:
+                                obj[key] = pendulum.parse(value)
+                            except (ValueError, TypeError):
+                                pass
         return obj
 
     with open(file_path, mode="r") as f:
@@ -815,6 +817,10 @@ def assert_table_content(self, table_id: str, expected_content: List[dict], prim
         :param primary_key: the primary key to use to compare.
         :return: whether the table has content and the expected content is correct
         """
+
+        logging.info(
+            f"assert_table_content: {table_id}, len(expected_content)={len(expected_content), }, primary_key={primary_key}"
+        )
         rows = None
         actual_content = None
         try: