Add partition date

docs/api/DatasetRelease.md

@@ -59,6 +59,12 @@
     <td></td>
     <td>[optional] </td>
 </tr>
+<tr>
+    <td><strong>partition_date</strong></td>
+    <td><strong>datetime, none_type</strong></td>
+    <td></td>
+    <td>[optional] </td>
+</tr>
 <tr>
     <td><strong>changefile_start_date</strong></td>
     <td><strong>datetime, none_type</strong></td>

docs/api/ObservatoryApi.md

@@ -460,6 +460,7 @@ with observatory.api.client.ApiClient(configuration) as api_client:
         data_interval_start=dateutil_parser('2020-01-02T20:01:05Z'),
         data_interval_end=dateutil_parser('2020-01-02T20:01:05Z'),
         snapshot_date=dateutil_parser('2020-01-02T20:01:05Z'),
+        partition_date=dateutil_parser('2020-01-02T20:01:05Z'),
         changefile_start_date=dateutil_parser('2020-01-02T20:01:05Z'),
         changefile_end_date=dateutil_parser('2020-01-02T20:01:05Z'),
         sequence_start=1,
@@ -591,6 +592,7 @@ with observatory.api.client.ApiClient(configuration) as api_client:
         data_interval_start=dateutil_parser('2020-01-02T20:01:05Z'),
         data_interval_end=dateutil_parser('2020-01-02T20:01:05Z'),
         snapshot_date=dateutil_parser('2020-01-02T20:01:05Z'),
+        partition_date=dateutil_parser('2020-01-02T20:01:05Z'),
         changefile_start_date=dateutil_parser('2020-01-02T20:01:05Z'),
         changefile_end_date=dateutil_parser('2020-01-02T20:01:05Z'),
         sequence_start=1,

docs/api/openapi.yaml

@@ -204,6 +204,11 @@ definitions:
         format: date-time
         example: 2020-01-02T20:01:05Z
         x-nullable: true
+      partition_date:
+        type: string
+        format: date-time
+        example: 2020-01-02T20:01:05Z
+        x-nullable: true
       changefile_start_date:
         type: string
         format: date-time

observatory-api/observatory/api/client/model/dataset_release.py

@@ -83,6 +83,7 @@ def openapi_types():
             'data_interval_start': (datetime, none_type,),  # noqa: E501, F821
             'data_interval_end': (datetime, none_type,),  # noqa: E501, F821
             'snapshot_date': (datetime, none_type,),  # noqa: E501, F821
+            'partition_date': (datetime, none_type,),  # noqa: E501, F821
             'changefile_start_date': (datetime, none_type,),  # noqa: E501, F821
             'changefile_end_date': (datetime, none_type,),  # noqa: E501, F821
             'sequence_start': (int, none_type,),  # noqa: E501, F821
@@ -105,6 +106,7 @@ def discriminator():
         'data_interval_start': 'data_interval_start',  # noqa: E501
         'data_interval_end': 'data_interval_end',  # noqa: E501
         'snapshot_date': 'snapshot_date',  # noqa: E501
+        'partition_date': 'partition_date',  # noqa: E501
         'changefile_start_date': 'changefile_start_date',  # noqa: E501
         'changefile_end_date': 'changefile_end_date',  # noqa: E501
         'sequence_start': 'sequence_start',  # noqa: E501
@@ -164,6 +166,7 @@ def _from_openapi_data(cls, *args, **kwargs):  # noqa: E501
             data_interval_start (datetime, none_type): [optional]  # noqa: E501
             data_interval_end (datetime, none_type): [optional]  # noqa: E501
             snapshot_date (datetime, none_type): [optional]  # noqa: E501
+            partition_date (datetime, none_type): [optional]  # noqa: E501
             changefile_start_date (datetime, none_type): [optional]  # noqa: E501
             changefile_end_date (datetime, none_type): [optional]  # noqa: E501
             sequence_start (int, none_type): [optional]  # noqa: E501
@@ -263,6 +266,7 @@ def __init__(self, *args, **kwargs):  # noqa: E501
             data_interval_start (datetime, none_type): [optional]  # noqa: E501
             data_interval_end (datetime, none_type): [optional]  # noqa: E501
             snapshot_date (datetime, none_type): [optional]  # noqa: E501
+            partition_date (datetime, none_type): [optional]  # noqa: E501
             changefile_start_date (datetime, none_type): [optional]  # noqa: E501
             changefile_end_date (datetime, none_type): [optional]  # noqa: E501
             sequence_start (int, none_type): [optional]  # noqa: E501

observatory-api/observatory/api/server/openapi.yaml.jinja2

@@ -219,6 +219,11 @@ definitions:
         format: date-time
         example: 2020-01-02T20:01:05Z
         x-nullable: true
+      partition_date:
+        type: string
+        format: date-time
+        example: 2020-01-02T20:01:05Z
+        x-nullable: true
       changefile_start_date:
         type: string
         format: date-time

observatory-api/observatory/api/server/orm.py

@@ -134,6 +134,7 @@ class DatasetRelease(Base):
     data_interval_start: pendulum.DateTime
     data_interval_end: pendulum.DateTime
     snapshot_date: pendulum.DateTime
+    partition_date: pendulum.DateTime
     changefile_start_date: pendulum.DateTime
     changefile_end_date: pendulum.DateTime
     sequence_start: int
@@ -149,6 +150,7 @@ class DatasetRelease(Base):
     data_interval_start = Column(DateTime(), nullable=True)
     data_interval_end = Column(DateTime(), nullable=True)
     snapshot_date = Column(DateTime(), nullable=True)
+    partition_date = Column(DateTime(), nullable=True)
     changefile_start_date = Column(DateTime(), nullable=True)
     changefile_end_date = Column(DateTime(), nullable=True)
     sequence_start = Column(Integer, nullable=True)
@@ -166,6 +168,7 @@ def __init__(
         data_interval_start: Union[pendulum.DateTime, str] = None,
         data_interval_end: Union[pendulum.DateTime, str] = None,
         snapshot_date: Union[pendulum.DateTime, str] = None,
+        partition_date: Union[pendulum.DateTime, str] = None,
         changefile_start_date: Union[pendulum.DateTime, str] = None,
         changefile_end_date: Union[pendulum.DateTime, str] = None,
         sequence_start: int = None,
@@ -183,6 +186,7 @@ def __init__(
         :param data_interval_start: the DAGs data interval start. Date is inclusive.
         :param data_interval_end: the DAGs data interval end. Date is exclusive.
         :param snapshot_date: the release date of the snapshot.
+        :param partition_date: the partition date.
         :param changefile_start_date: the date of the first changefile processed in this release.
         :param changefile_end_date: the date of the last changefile processed in this release.
         :param sequence_start: the starting sequence number of files that make up this release.
@@ -199,6 +203,7 @@ def __init__(
         self.data_interval_start = to_datetime_utc(data_interval_start)
         self.data_interval_end = to_datetime_utc(data_interval_end)
         self.snapshot_date = to_datetime_utc(snapshot_date)
+        self.partition_date = to_datetime_utc(partition_date)
         self.changefile_start_date = to_datetime_utc(changefile_start_date)
         self.changefile_end_date = to_datetime_utc(changefile_end_date)
         self.sequence_start = sequence_start
@@ -215,6 +220,7 @@ def update(
         data_interval_start: Union[pendulum.DateTime, str] = None,
         data_interval_end: Union[pendulum.DateTime, str] = None,
         snapshot_date: Union[pendulum.DateTime, str] = None,
+        partition_date: Union[pendulum.DateTime, str] = None,
         changefile_start_date: Union[pendulum.DateTime, str] = None,
         changefile_end_date: Union[pendulum.DateTime, str] = None,
         sequence_start: int = None,
@@ -230,7 +236,8 @@ def update(
         :param dag_run_id: the DAG's run ID.
         :param data_interval_start: the DAGs data interval start. Date is inclusive.
         :param data_interval_end: the DAGs data interval end. Date is exclusive.
-        :param snapshot_date: the release date of the snapshot.
+        :param snapshot_date: the snapshot date.
+        :param partition_date: the partition date.
         :param changefile_start_date: the date of the first changefile processed in this release.
         :param changefile_end_date: the date of the last changefile processed in this release.
         :param sequence_start: the starting sequence number of files that make up this release.
@@ -258,6 +265,9 @@ def update(
         if snapshot_date is not None:
             self.snapshot_date = to_datetime_utc(snapshot_date)
 
+        if partition_date is not None:
+            self.partition_date = to_datetime_utc(partition_date)
+
         if changefile_start_date is not None:
             self.changefile_start_date = to_datetime_utc(changefile_start_date)
 

tests/observatory/api/client/test_dataset_release.py

@@ -36,6 +36,7 @@ def __init__(self):
             dag_run_id="scheduled__2023-03-26T00:00:00+00:00",
             data_interval_start=dt,
             data_interval_end=dt,
+            partition_date=dt,
             snapshot_date=dt,
             changefile_start_date=dt,
             changefile_end_date=dt,
@@ -50,6 +51,7 @@ def __init__(self):
                 id=1,
                 dag_id="doi_workflow",
                 dataset_id="doi",
+                partition_date=dt,
                 snapshot_date=dt,
                 start_date=dt,
                 end_date=dt,

tests/observatory/api/client/test_observatory_api.py

@@ -76,6 +76,7 @@ def test_get_dataset_release(self):
                     dag_run_id="scheduled__2023-03-26T00:00:00+00:00",
                     data_interval_start=dt,
                     data_interval_end=dt,
+                    partition_date=dt,
                     snapshot_date=dt,
                     changefile_start_date=dt,
                     changefile_end_date=dt,
@@ -98,6 +99,7 @@ def test_get_dataset_release(self):
             self.assertEqual(dt_utc, obj.data_interval_start)
             self.assertEqual(dt_utc, obj.data_interval_end)
             self.assertEqual(dt_utc, obj.snapshot_date)
+            self.assertEqual(dt_utc, obj.partition_date)
             self.assertEqual(dt_utc, obj.changefile_start_date)
             self.assertEqual(dt_utc, obj.changefile_end_date)
             self.assertEqual(1, obj.sequence_start)
@@ -129,6 +131,7 @@ def test_post_dataset_release(self):
                 data_interval_start=dt,
                 data_interval_end=dt,
                 snapshot_date=dt,
+                partition_date=dt,
                 changefile_start_date=dt,
                 changefile_end_date=dt,
                 sequence_start=1,
@@ -144,6 +147,7 @@ def test_post_dataset_release(self):
             self.assertEqual(dt_utc, result.data_interval_start)
             self.assertEqual(dt_utc, result.data_interval_end)
             self.assertEqual(dt_utc, result.snapshot_date)
+            self.assertEqual(dt_utc, result.partition_date)
             self.assertEqual(dt_utc, result.changefile_start_date)
             self.assertEqual(dt_utc, result.changefile_end_date)
             self.assertEqual(1, result.sequence_start)
@@ -165,6 +169,7 @@ def test_put_dataset_release(self):
                 data_interval_start=dt,
                 data_interval_end=dt,
                 snapshot_date=dt,
+                partition_date=dt,
                 changefile_start_date=dt,
                 changefile_end_date=dt,
                 sequence_start=1,
@@ -180,6 +185,7 @@ def test_put_dataset_release(self):
             self.assertEqual(dt_utc, result.data_interval_start)
             self.assertEqual(dt_utc, result.data_interval_end)
             self.assertEqual(dt_utc, result.snapshot_date)
+            self.assertEqual(dt_utc, result.partition_date)
             self.assertEqual(dt_utc, result.changefile_start_date)
             self.assertEqual(dt_utc, result.changefile_end_date)
             self.assertEqual(1, result.sequence_start)
@@ -208,6 +214,7 @@ def test_put_dataset_release(self):
                         data_interval_start=dt,
                         data_interval_end=dt,
                         snapshot_date=dt,
+                        partition_date=dt,
                         changefile_start_date=dt,
                         changefile_end_date=dt,
                         sequence_start=1,