feat: Store upgrade_10_11 warning logs as notes, closes #67

process/management/commands/file_worker.py

@@ -30,6 +30,7 @@
     RECORD_PACKAGE,
     RELEASE_PACKAGE,
     consume,
+    create_logger_note,
     create_note,
     create_step,
     decorator,
@@ -254,10 +255,11 @@ def _store_data(collection_file, package, releases_or_records, data_type, upgrad
 
     for release_or_record in releases_or_records:
         if upgrade:
-            # upgrade_10_11() requires an OrderedDict. simplejson is used for native decimal support.
-            release_or_record = upgrade_10_11(
-                json.loads(json.dumps(release_or_record, use_decimal=True), object_pairs_hook=OrderedDict)
-            )
+            with create_logger_note(collection_file.collection, "ocdskit"):
+                # upgrade_10_11() requires an OrderedDict. simplejson is used for native decimal support.
+                release_or_record = upgrade_10_11(
+                    json.loads(json.dumps(release_or_record, use_decimal=True), object_pairs_hook=OrderedDict)
+                )
 
         data = get_or_create(Data, release_or_record)
 

process/util.py

@@ -1,4 +1,5 @@
 import hashlib
+import io
 import logging
 import os
 import warnings
@@ -162,3 +163,19 @@ def create_warnings_note(collection, category):
 
         if note := [str(warning.message) for warning in w if issubclass(warning.category, category)]:
             create_note(collection, CollectionNote.Level.WARNING, note)
+
+
+@contextmanager
+def create_logger_note(collection, name):
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.WARNING)
+    logger = logging.getLogger(name)
+    logger.addHandler(handler)
+
+    yield
+
+    logger.removeHandler(handler)
+
+    if note := stream.getvalue():
+        create_note(collection, CollectionNote.Level.WARNING, note)

tests/test_util.py

@@ -0,0 +1,37 @@
+import json
+from collections import OrderedDict
+from unittest.mock import patch
+
+from django.test import SimpleTestCase
+from ocdskit.upgrade import upgrade_10_11
+
+from process.models import CollectionNote
+from process.util import create_logger_note
+
+
+class UtilTests(SimpleTestCase):
+    @patch("process.util.create_note")
+    def test_create_logger_note(self, create_note):
+        with create_logger_note("collection", "ocdskit"):
+            upgrade_10_11(
+                json.loads(
+                    '{"tender":{"tenderers":[{"name":"Acme Inc.","identifier":{"id":1}}]},"awards":[{"suppliers":'
+                    '[{"name":"Acme Inc.","identifier":{"id":1},"additionalIdentifiers":[{"id":"a"}]}]}]}',
+                    object_pairs_hook=OrderedDict,
+                )
+            )
+
+        create_note.assert_called_once_with(
+            "collection",
+            CollectionNote.Level.WARNING,
+            'party in "supplier" role differs from party in ["tenderer"] roles:\n{"name": "Acme Inc.", "identifier": '
+            '{"id": 1}, "additionalIdentifiers": [{"id": "a"}], "id": "3c9756cf8983b14066a034079aa7aae4"}\n{"id": '
+            '"3c9756cf8983b14066a034079aa7aae4", "name": "Acme Inc.", "identifier": {"id": 1}}\n',
+        )
+
+    @patch("process.util.create_note")
+    def test_create_logger_note_not_called(self, create_note):
+        with create_logger_note("collection", "ocdskit"):
+            upgrade_10_11({})
+
+        create_note.assert_not_called()