Fix encrypted fields bulk_update corrupting data

fixes pulp#4359

CHANGES/4359.bugfix

@@ -0,0 +1,2 @@
+Fix encrypted fields to use json instead of repr/eval and make them fit for ``bulk_update``.
+This solves an issue with ``pulpcore-manager rotate-db-key`` corrupting data.

pulpcore/app/models/fields.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 from gettext import gettext as _
@@ -12,6 +13,7 @@
 
 
 from pulpcore.app.files import TemporaryDownloadedFile
+from pulpcore.app.loggers import deprecation_logger
 
 _logger = logging.getLogger(__name__)
 
@@ -96,14 +98,17 @@ def __init__(self, *args, **kwargs):
             raise ImproperlyConfigured("EncryptedTextField does not support db_index=True.")
         super().__init__(*args, **kwargs)
 
-    def get_db_prep_save(self, value, connection):
-        value = super().get_db_prep_save(value, connection)
+    def get_prep_value(self, value):
+        value = super().get_prep_value(value)
         if value is not None:
-            return force_str(_fernet().encrypt(force_bytes(value)))
+            assert isinstance(value, str)
+            value = force_str(_fernet().encrypt(force_bytes(value)))
+        return value
 
     def from_db_value(self, value, expression, connection):
         if value is not None:
-            return force_str(_fernet().decrypt(force_bytes(value)))
+            value = force_str(_fernet().decrypt(force_bytes(value)))
+        return value
 
 
 class EncryptedJSONField(JSONField):
@@ -124,24 +129,37 @@ def encrypt(self, value):
         elif isinstance(value, (list, tuple, set)):
             return [self.encrypt(v) for v in value]
 
-        return force_str(_fernet().encrypt(force_bytes(repr(value))))
+        return force_str(_fernet().encrypt(force_bytes(json.dumps(value, cls=self.encoder))))
 
     def decrypt(self, value):
         if isinstance(value, dict):
             return {k: self.decrypt(v) for k, v in value.items()}
         elif isinstance(value, (list, tuple, set)):
             return [self.decrypt(v) for v in value]
 
-        return eval(force_str(_fernet().decrypt(force_bytes(value))))
+        dec_value = force_str(_fernet().decrypt(force_bytes(value)))
+        try:
+            return json.loads(dec_value, cls=self.decoder)
+        except json.JSONDecodeError:
+            deprecation_logger.info(
+                "Failed to decode json in an EncryptedJSONField. Falling back to eval. "
+                "Please run pulpcore-manager rotate-db-key to repair."
+                "This is deprecated and will be removed in pulpcore 3.40."
+            )
+            return eval(dec_value)
 
-    def get_db_prep_save(self, value, connection):
-        value = self.encrypt(value)
-        return super().get_db_prep_save(value, connection)
+    def get_prep_value(self, value):
+        value = super().get_prep_value(value)
+        if value is not None:
+            if hasattr(value, "as_sql"):
+                return value
+            value = self.encrypt(value)
+        return value
 
     def from_db_value(self, value, expression, connection):
         if value is not None:
-            value = super().from_db_value(value, expression, connection)
-            return self.decrypt(value)
+            value = self.decrypt(super().from_db_value(value, expression, connection))
+        return value
 
 
 @Field.register_lookup

pulpcore/tests/unit/models/test_remote.py

@@ -1,19 +1,37 @@
 import pytest
 from uuid import uuid4
+from cryptography.fernet import InvalidToken
 
+from django.core.management import call_command
 from django.db import connection
 
-from pulpcore.app.models import Remote
+from pulpcore.app.models import Remote, Domain
 from pulpcore.app.models.fields import _fernet, EncryptedTextField
 
 
+TEST_KEY1 = b"hPCIFQV/upbvPRsEpgS7W32XdFA2EQgXnMtyNAekebQ="
+TEST_KEY2 = b"6Xyv+QezAQ+4R870F5qsgKcngzmm46caDB2gyo9qnpc="
+
+
 @pytest.fixture
 def fake_fernet(tmp_path, settings):
+    def _steps():
+        yield
+        key_file.write_bytes(TEST_KEY2 + b"\n" + TEST_KEY1)
+        _fernet.cache_clear()
+        yield
+        key_file.write_bytes(TEST_KEY2)
+        _fernet.cache_clear()
+        yield
+        key_file.write_bytes(TEST_KEY1)
+        _fernet.cache_clear()
+        yield
+
     key_file = tmp_path / "db_symmetric_key"
-    key_file.write_bytes(b"hPCIFQV/upbvPRsEpgS7W32XdFA2EQgXnMtyNAekebQ=")
+    key_file.write_bytes(TEST_KEY1)
     settings.DB_ENCRYPTION_KEY = str(key_file)
     _fernet.cache_clear()
-    yield
+    yield _steps()
     _fernet.cache_clear()
 
 
@@ -31,3 +49,29 @@ def test_encrypted_proxy_password(fake_fernet):
     proxy_password = EncryptedTextField().from_db_value(db_proxy_password, None, connection)
     assert db_proxy_password != "test"
     assert proxy_password == "test"
+
+
+@pytest.mark.django_db
+def test_rotate_db_key(fake_fernet):
+    remote = Remote.objects.create(name=uuid4(), proxy_password="test")
+    domain = Domain.objects.create(name=uuid4(), storage_settings={"base_path": "/foo"})
+
+    next(fake_fernet)  # new + old key
+
+    call_command("rotate-db-key")
+
+    next(fake_fernet)  # new key
+
+    del remote.proxy_password
+    assert remote.proxy_password == "test"
+    del domain.storage_settings
+    assert domain.storage_settings == {"base_path": "/foo"}
+
+    next(fake_fernet)  # old key
+
+    del remote.proxy_password
+    with pytest.raises(InvalidToken):
+        remote.proxy_password
+    del domain.storage_settings
+    with pytest.raises(InvalidToken):
+        domain.storage_settings