Convert updated_date into timestamp when it is supplied as a date

The-Academic-Observatory · Aug 10, 2023 · 8f2684f · 8f2684f
1 parent c59e770
commit 8f2684f
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 9 deletions.
diff --git a/...workflows/fixtures/openalex/2023-04-16/data/authors/updated_date=2023-04-16/part_000.json b/...workflows/fixtures/openalex/2023-04-16/data/authors/updated_date=2023-04-16/part_000.json
diff --git a/academic_observatory_workflows/fixtures/openalex/2023-04-16/expected/authors.json b/academic_observatory_workflows/fixtures/openalex/2023-04-16/expected/authors.json
diff --git a/academic_observatory_workflows/workflows/openalex_telescope.py b/academic_observatory_workflows/workflows/openalex_telescope.py
@@ -331,10 +331,10 @@ def __init__(
                 ("concepts", True),
                 ("institutions", True),
                 ("works", True),
-                ("authors", False),
-                ("publishers", False),
-                ("sources", False),
-                ("funders", False),
+                ("authors", True),
+                ("publishers", True),
+                ("sources", True),
+                ("funders", True),
             ]
 
         super().__init__(
@@ -1014,9 +1014,9 @@ def transform_object(obj: dict):
             value = []
         obj[field] = [x for x in value if x is not None]
 
+    # TODO: when re-ingesting entire dataset: change schema to new version
     field = "abstract_inverted_index"
     if field in obj:
-
         def parse_abstract(dict_: dict):
             keys_ = list(dict_.keys())
             values_ = [str(value_)[1:-1] for value_ in dict_.values()]
@@ -1039,3 +1039,10 @@ def parse_abstract(dict_: dict):
             values = list(obj[field][nested_field].values())
 
             obj[field][nested_field] = {"keys": keys, "values": values}
+
+    # Transform updated_date from a date into a datetime
+    # TODO: when re-ingesting entire dataset: change to date
+    field = "updated_date"
+    if field in obj:
+        obj[field] = pendulum.parse(obj[field]).to_iso8601_string()
+
diff --git a/academic_observatory_workflows/workflows/tests/test_openalex_telescope.py b/academic_observatory_workflows/workflows/tests/test_openalex_telescope.py
@@ -592,7 +592,11 @@ def test_dag_structure(self):
                 "aws_to_gcs_transfer": ["download_concepts"],
                 "download_concepts": ["download_institutions"],
                 "download_institutions": ["download_works"],
-                "download_works": ["transform"],
+                "download_works": ["download_authors"],
+                "download_authors": ["download_publishers"],
+                "download_publishers": ["download_sources"],
+                "download_sources": ["download_funders"],
+                "download_funders": ["transform"],
                 "transform": ["upload_upsert_files"],
                 "upload_upsert_files": ["bq_load_upsert_tables"],
                 "bq_load_upsert_tables": ["bq_upsert_records"],