time-with-milliseconds

renku/command/schema/activity.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Activity JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema, SoftwareAgentSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -124,6 +124,28 @@ class Meta:
     started_at_time = fields.DateTime(prov.startedAtTime, add_value_types=True)
     usages = Nested(prov.qualifiedUsage, UsageSchema, many=True)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.started_at_time = obj.started_at_time.replace(microsecond=0)
+            obj.ended_at_time = obj.ended_at_time.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class WorkflowFileActivityCollectionSchema(JsonLDSchema):
     """WorkflowFileActivityCollection schema."""

renku/command/schema/composite_plan.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Represent a group of run templates."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.calamus import JsonLDSchema, Nested, fields, prov, renku, schema
@@ -49,3 +49,27 @@ class Meta:
     project_id = fields.IRI(renku.hasPlan, reverse=True)
     plans = Nested(renku.hasSubprocess, [PlanSchema, "CompositePlanSchema"], many=True)
     links = Nested(renku.workflowLinks, [ParameterLinkSchema], many=True, load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/dataset.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Datasets JSON-LD schemes."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -69,6 +69,27 @@ class Meta:
     id = fields.Id()
     name = fields.String(schema.name)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class LanguageSchema(JsonLDSchema):
     """Language schema."""
@@ -134,6 +155,27 @@ class Meta:
     is_external = fields.Boolean(renku.external, load_default=False)
     source = fields.String(renku.source, load_default=None)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.date_added = obj.date_added.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class DatasetSchema(JsonLDSchema):
     """Dataset schema."""
@@ -168,3 +210,30 @@ class Meta:
     same_as = Nested(schema.sameAs, UrlSchema, load_default=None)
     title = fields.String(schema.name)
     version = fields.String(schema.version, load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            if obj.date_created:
+                obj.date_created = obj.date_created.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            if obj.date_published:
+                obj.date_published = obj.date_published.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/plan.py

@@ -54,3 +54,27 @@ class Meta:
     parameters = Nested(renku.hasArguments, CommandParameterSchema, many=True, load_default=None)
     success_codes = fields.List(renku.successCodes, fields.Integer(), load_default=[0])
     annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
+
+    @marshmallow.pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/project.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Project JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -59,3 +59,22 @@ class Meta:
     )
     version = StringList(schema.schemaVersion, load_default="1")
     keywords = fields.List(schema.keywords, fields.String(), load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.date_created = obj.date_created.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/core/migration/m_0010__metadata_fixes.py

@@ -254,7 +254,7 @@ def fix_plan_times(plan_gateway: IPlanGateway):
             if plan.date_removed.tzinfo is None:
                 # NOTE: There was a bug that caused date_removed to be set without timezone (as UTC time)
                 # so we patch in the timezone here
-                plan.date_removed = plan.date_removed.replace(microsecond=0).astimezone(timezone.utc)
+                plan.date_removed = plan.date_removed.astimezone(timezone.utc)
             if plan.date_removed < plan.date_created:
                 # NOTE: Fix invalidation times set before creation date on plans
                 plan.date_removed = plan.date_created

renku/core/migration/models/v9.py

@@ -2236,6 +2236,26 @@ class Meta:
     ended_at_time = fields.DateTime(prov.endedAtTime, add_value_types=True)
     agents = Nested(prov.wasAssociatedWith, [OldPersonSchema, OldSoftwareAgentSchema], many=True)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.started_at_time = obj.started_at_time.replace(microsecond=0)
+            obj.ended_at_time = obj.ended_at_time.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class ProcessRunSchema(ActivitySchema):
     """ProcessRun schema."""

renku/core/util/datetime8601.py

@@ -64,8 +64,6 @@ def fix_datetime(value) -> Optional[datetime]:
     if isinstance(value, datetime):
         if not value.tzinfo:
             value = _set_to_local_timezone(value)
-        if value.microsecond:
-            value = value.replace(microsecond=0)
 
     return value
 
@@ -75,7 +73,7 @@ def _set_to_local_timezone(value):
     return value.replace(tzinfo=local_tz)
 
 
-def local_now(remove_microseconds: bool = True) -> datetime:
+def local_now(remove_microseconds: bool = False) -> datetime:
     """Return current datetime in local timezone."""
     now = datetime.now(timezone.utc).astimezone()
     return now.replace(microsecond=0) if remove_microseconds else now

renku/domain_model/workflow/plan.py

@@ -153,12 +153,14 @@ def is_derivation(self) -> bool:
         """Return if an ``AbstractPlan`` has correct derived_from."""
         raise NotImplementedError()
 
-    def delete(self, when: datetime = local_now()):
+    def delete(self, when: Optional[datetime] = None):
         """Mark a plan as deleted.
 
         NOTE: Don't call this function for deleting plans since it doesn't delete the whole plan derivatives chain. Use
         renku.core.workflow.plan::remove_plan instead.
         """
+        when = when or local_now()
+
         self.unfreeze()
         self.date_removed = when
         self.freeze()

tests/cli/test_migrate.py

@@ -194,7 +194,7 @@ def test_comprehensive_dataset_migration(isolated_runner, old_dataset_project):
     assert "Cornell University" == dataset.creators[0].affiliation
     assert "Rooth, Mats" == dataset.creators[0].name
     assert dataset.date_published is None
-    assert "2020-08-10T21:35:05+00:00" == dataset.date_created.isoformat("T")
+    assert "2020-08-10T21:35:05+00:00" == dataset.date_created.replace(microsecond=0).isoformat("T")
     assert "Replication material for a paper to be presented" in dataset.description
     assert "https://doi.org/10.7910/DVN/EV6KLF" == dataset.same_as.url
     assert "1" == tags[0].name
@@ -204,7 +204,7 @@ def test_comprehensive_dataset_migration(isolated_runner, old_dataset_project):
 
     file_ = dataset.find_file("data/dataverse/copy.sh")
     assert "https://dataverse.harvard.edu/api/access/datafile/3050656" == file_.source
-    assert "2020-08-10T21:35:10+00:00" == file_.date_added.isoformat("T")
+    assert "2020-08-10T21:35:10+00:00" == file_.date_added.replace(microsecond=0).isoformat("T")
     assert file_.based_on is None
     assert not hasattr(file_, "creators")
 
@@ -406,8 +406,8 @@ def test_migrate_preserves_date_when_preserving_ids(isolated_runner, old_dataset
 
     dataset = get_dataset_with_injection("mixed")
 
-    assert "2020-08-10 21:35:20+00:00" == dataset.date_created.isoformat(" ")
-    assert "2020-08-10 21:35:20+00:00" == dataset.date_modified.isoformat(" ")
+    assert "2020-08-10 21:35:20+00:00" == dataset.date_created.replace(microsecond=0).isoformat(" ")
+    assert "2020-08-10 21:35:20+00:00" == dataset.date_modified.replace(microsecond=0).isoformat(" ")
 
 
 @pytest.mark.migration
@@ -418,8 +418,8 @@ def test_migrate_preserves_date_for_mutated_datasets(isolated_runner, old_datase
 
     dataset = get_dataset_with_injection("local")
 
-    assert "2021-07-23 14:34:24+00:00" == dataset.date_created.isoformat(" ")
-    assert "2021-07-23 14:34:58+00:00" == dataset.date_modified.isoformat(" ")
+    assert "2021-07-23 14:34:24+00:00" == dataset.date_created.replace(microsecond=0).isoformat(" ")
+    assert "2021-07-23 14:34:58+00:00" == dataset.date_modified.replace(microsecond=0).isoformat(" ")
 
 
 @pytest.mark.migration
@@ -429,5 +429,5 @@ def test_migrate_sets_correct_date_for_non_mutated_datasets(isolated_runner, old
 
     dataset = get_dataset_with_injection("mixed")
 
-    assert "2020-08-10 21:35:20+00:00" == dataset.date_created.isoformat(" ")
-    assert "2020-08-10 23:35:56+02:00" == dataset.date_modified.isoformat(" ")
+    assert "2020-08-10 21:35:20+00:00" == dataset.date_created.replace(microsecond=0).isoformat(" ")
+    assert "2020-08-10 23:35:56+02:00" == dataset.date_modified.replace(microsecond=0).isoformat(" ")