Fix datetime serialization in extras and fix missing bcrypt

.circleci/config.yml

@@ -19,9 +19,9 @@ jobs:
           command: export BASE_BRANCH=$(base_branch)
       - restore_cache:
           keys:
-          - py3-cache-v10-{{ arch }}-{{ checksum "python.deps" }}
-          - py3-cache-v10-{{ arch }}-{{ .Branch }}
-          - py3-cache-v10-{{ arch }}-{{ .Environment.BASE_BRANCH }}
+          - py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }}
+          - py3-cache-v12-{{ arch }}-{{ .Branch }}
+          - py3-cache-v12-{{ arch }}-{{ .Environment.BASE_BRANCH }}
       - run:
           name: Install python dependencies
           # FIXME: For now, we fix setuptools due to https://github.com/etalab/data.gouv.fr/issues/1041
@@ -32,11 +32,11 @@ jobs:
             pip install -r requirements/develop.pip
             pip install -e .
       - save_cache:
-          key: py3-cache-v10-{{ arch }}-{{ checksum "python.deps" }}
+          key: py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }}
           paths:
           - venv
       - save_cache:
-          key: py3-cache-v10-{{ arch }}-{{ .Branch }}
+          key: py3-cache-v12-{{ arch }}-{{ .Branch }}
           paths:
           - venv
       - run:

CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## Current (in progress)
 
-- Nothing yet
+- **breaking change** Fix datetime serialization in extras (return ISO string in JSON). Warning, `ujson` shouldn't be installed anymore on the project to allow `cls` parameter to override the JSONEncoder [#3019](https://github.com/opendatateam/udata/pull/3019)
+- Fix missing `bcrypt` dependency [#3019](https://github.com/opendatateam/udata/pull/3019)
 
 ## 7.0.7 (2024-04-16)
 

requirements/develop.pip

@@ -42,6 +42,11 @@ babel==2.12.1
     #   flask-babel
 backports-entry-points-selectable==1.2.0
     # via virtualenv
+bcrypt==3.1.7
+    # via
+    #   -c requirements/install.pip
+    #   -c requirements/test.pip
+    #   -r requirements/install.in
 billiard==3.6.4.0
     # via
     #   -c requirements/install.pip
@@ -102,6 +107,7 @@ cffi==1.14.6
     # via
     #   -c requirements/install.pip
     #   -c requirements/test.pip
+    #   bcrypt
     #   cryptography
 cfgv==3.3.0
     # via pre-commit
@@ -539,6 +545,7 @@ six==1.16.0
     # via
     #   -c requirements/install.pip
     #   -c requirements/test.pip
+    #   bcrypt
     #   bleach
     #   cryptography
     #   faker

requirements/doc.pip

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
 #
 #    pip-compile --output-file=requirements/doc.pip requirements/doc.in
 #
@@ -40,6 +40,11 @@ babel==2.12.1
     #   -c requirements/test.pip
     #   -r requirements/install.in
     #   flask-babel
+bcrypt==3.1.7
+    # via
+    #   -c requirements/install.pip
+    #   -c requirements/test.pip
+    #   -r requirements/install.in
 billiard==3.6.4.0
     # via
     #   -c requirements/install.pip
@@ -97,6 +102,7 @@ cffi==1.14.6
     # via
     #   -c requirements/install.pip
     #   -c requirements/test.pip
+    #   bcrypt
     #   cryptography
 chardet==3.0.4
     # via
@@ -449,6 +455,7 @@ six==1.16.0
     # via
     #   -c requirements/install.pip
     #   -c requirements/test.pip
+    #   bcrypt
     #   bleach
     #   cryptography
     #   faker
@@ -480,8 +487,6 @@ tlds==2021080800
     #   -r requirements/install.in
 typing-extensions==4.7.1
     # via
-    #   -c requirements/install.pip
-    #   -c requirements/test.pip
     #   importlib-metadata
     #   mkdocs
     #   platformdirs

requirements/install.in

@@ -1,6 +1,7 @@
 authlib==0.14.3
 awesome-slugify==1.6.5
 Babel==2.12.1
+bcrypt==3.1.7
 bleach==3.1.0
 blinker==1.4
 boto3>=1.26

requirements/install.pip

@@ -20,6 +20,8 @@ babel==2.12.1
     # via
     #   -r requirements/install.in
     #   flask-babel
+bcrypt==3.1.7
+    # via -r requirements/install.in
 billiard==3.6.4.0
     # via celery
 bleach==3.1.0
@@ -51,7 +53,9 @@ celerybeat-mongo==0.2.0
 certifi==2021.5.30
     # via requests
 cffi==1.14.6
-    # via cryptography
+    # via
+    #   bcrypt
+    #   cryptography
 chardet==3.0.4
     # via requests
 click==8.0.0
@@ -233,6 +237,7 @@ s3transfer==0.6.0
     # via boto3
 six==1.16.0
     # via
+    #   bcrypt
     #   bleach
     #   cryptography
     #   faker

requirements/report.pip

@@ -40,6 +40,11 @@ babel==2.12.1
     #   -c requirements/test.pip
     #   -r requirements/install.in
     #   flask-babel
+bcrypt==3.1.7
+    # via
+    #   -c requirements/install.pip
+    #   -c requirements/test.pip
+    #   -r requirements/install.in
 billiard==3.6.4.0
     # via
     #   -c requirements/install.pip
@@ -97,6 +102,7 @@ cffi==1.14.6
     # via
     #   -c requirements/install.pip
     #   -c requirements/test.pip
+    #   bcrypt
     #   cryptography
 chardet==3.0.4
     # via
@@ -494,6 +500,7 @@ six==1.16.0
     # via
     #   -c requirements/install.pip
     #   -c requirements/test.pip
+    #   bcrypt
     #   bleach
     #   cryptography
     #   faker

requirements/test.pip

@@ -33,6 +33,10 @@ babel==2.12.1
     #   -c requirements/install.pip
     #   -r requirements/install.in
     #   flask-babel
+bcrypt==3.1.7
+    # via
+    #   -c requirements/install.pip
+    #   -r requirements/install.in
 billiard==3.6.4.0
     # via
     #   -c requirements/install.pip
@@ -80,6 +84,7 @@ certifi==2021.5.30
 cffi==1.14.6
     # via
     #   -c requirements/install.pip
+    #   bcrypt
     #   cryptography
 chardet==3.0.4
     # via
@@ -384,6 +389,7 @@ s3transfer==0.6.0
 six==1.16.0
     # via
     #   -c requirements/install.pip
+    #   bcrypt
     #   bleach
     #   cryptography
     #   faker

udata/app.py

@@ -181,6 +181,9 @@ def create_app(config='udata.settings.Defaults', override=None,
 
     app.json_encoder = UDataJsonEncoder
 
+    # `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
+    app.config['RESTX_JSON'] = { 'cls': UDataJsonEncoder }
+
     app.debug = app.config['DEBUG'] and not app.config['TESTING']
 
     app.wsgi_app = ProxyFix(app.wsgi_app)

udata/tests/apiv2/test_datasets.py

@@ -1,12 +1,14 @@
+from datetime import datetime
 from flask import url_for
 
+from udata.core.dataset.models import ResourceMixin
 from udata.tests.api import APITestCase
 
 from udata.core.dataset.apiv2 import DEFAULT_PAGE_SIZE
 from udata.core.dataset.factories import (
     DatasetFactory, ResourceFactory, CommunityResourceFactory)
 from udata.core.organization.factories import OrganizationFactory, Member
-from udata.models import Dataset
+from udata.models import db, Dataset
 from udata.tests.helpers import assert_not_emit
 
 class DatasetAPIV2Test(APITestCase):
@@ -192,12 +194,14 @@ def setUp(self):
         self.dataset = DatasetFactory(owner=self.user)
 
     def test_get_dataset_extras(self):
-        self.dataset.extras = {'test::extra': 'test-value'}
+        Dataset.extras.register('check::date', db.DateTimeField)
+        self.dataset.extras = {'test::extra': 'test-value', 'check::date': datetime.fromisoformat('2024-04-14 08:42:00')}
         self.dataset.save()
         response = self.get(url_for('apiv2.dataset_extras', dataset=self.dataset))
         self.assert200(response)
         data = response.json
         assert data['test::extra'] == 'test-value'
+        assert data['check::date'] == '2024-04-14T08:42:00'
 
     def test_update_dataset_extras(self):
         self.dataset.extras = {
@@ -331,15 +335,18 @@ def setUp(self):
 
     def test_get_ressource_extras(self):
         '''It should fetch a resource from the API'''
+        ResourceMixin.extras.register('check:date', db.DateTimeField)
+
         resource = ResourceFactory()
-        resource.extras = {'test::extra': 'test-value'}
+        resource.extras = {'test::extra': 'test-value', 'check:date': datetime(2023, 4, 20, 13, 57, 31, 289000)}
         self.dataset.resources.append(resource)
         self.dataset.save()
         response = self.get(url_for('apiv2.resource_extras', dataset=self.dataset,
                                     rid=resource.id))
         self.assert200(response)
         data = response.json
         assert data['test::extra'] == 'test-value'
+        assert data['check:date'] == '2023-04-20T13:57:31.289000'
 
     def test_update_resource_extras(self):
         resource = ResourceFactory()