allow custom logger, e.g. for non-int primary keys

(fixes #30)

.github/workflows/test.yml

@@ -29,6 +29,11 @@ jobs:
           drf-version: 3.14.0
           variant: reversion
           backend: threading
+        - python-version: "3.11"
+          django-version: 4.2.2
+          drf-version: 3.14.0
+          variant: customlog
+          backend: threading
 
         # Older Django versions
         - python-version: "3.11"

data_wizard/models.py

@@ -322,14 +322,12 @@ class Record(models.Model):
 
     def __str__(self):
         if self.success:
-            return "Imported '{obj}' at row {row}".format(
-                obj=self.content_object,
-                row=self.row,
-            )
+            if self.content_object:
+                return f"Imported '{self.content_object}' at row {self.row}"
+            else:
+                return f"Success at row {self.row}"
         else:
-            return "Failed at row {row}".format(
-                row=self.row,
-            )
+            return f"Failed at row {self.row}"
 
     class Meta:
         ordering = ("run_id", "row")

data_wizard/settings.py

@@ -15,6 +15,7 @@
         "data_wizard.tasks.check_row_identifiers",
         "data_wizard.tasks.import_data",
     ),
+    "ROW_LOGGER": "data_wizard.tasks.create_record",
 }
 
 

data_wizard/sources/rest.py

@@ -17,7 +17,7 @@ class URLSourceSerializer(ModelSerializer):
     user = serializers.HiddenField(default=serializers.CurrentUserDefault())
 
     class Meta:
-        model = FileSource
+        model = URLSource
         fields = "__all__"
 
 

data_wizard/tasks.py

@@ -1,6 +1,7 @@
 from collections import OrderedDict
 from .models import Identifier
 from .signals import import_complete, new_metadata
+from .settings import import_setting
 from . import registry, wizard_task, InputNeeded
 
 from django.db import transaction
@@ -257,7 +258,6 @@ def get_choice_groups(run):
     groups = OrderedDict()
 
     for choice in choices:
-
         groups.setdefault(choice["group"], [])
         groups[choice["group"]].append(
             {
@@ -768,6 +768,7 @@ def _do_import(run):
     run.add_event("do_import")
 
     # Loop through table rows and add each record
+    log_row = import_setting("ROW_LOGGER")
     table = run.load_iter()
     rows = len(table)
     skipped = []
@@ -806,7 +807,8 @@ def rownum(i):
 
         # Record relationship between data source and resulting report (or
         # skipped record), including specific cell range.
-        run.record_set.create(
+        log_row(
+            run,
             row=rownum(i),
             content_object=obj,
             success=success,
@@ -824,6 +826,15 @@ def rownum(i):
     return status
 
 
+def create_record(run, row, content_object, success, fail_reason):
+    return run.record_set.create(
+        row=row,
+        content_object=content_object,
+        success=success,
+        fail_reason=fail_reason,
+    )
+
+
 def build_row(run, row, instance_globals, matched):
     """
     Compile spreadsheet row into serializer data format

docs/config/settings.md

@@ -20,6 +20,7 @@ DATA_WIZARD = {
         'data_wizard.tasks.check_row_identifiers',
         'data_wizard.tasks.import_data',
     ),
+    'ROW_LOGGER': 'data_wizard.tasks.create_record',
 }
 ```
 
@@ -91,6 +92,10 @@ Task | Description
 
 See [tasks] for more details on how to define custom processing tasks.
 
+### `DATA_WIZARD['ROW_LOGGER']`
+
+This function is used to log the result of each row imported in import_data.  The default function (`data_wizard.tasks.create_record`) creates a [Record][models] entry for each row.  The Record's [GenericForeignKey] will be pointed to the imported row in the target model.  This functionality assumes that the target model has an integer primary key.  If this is not the case, override `ROW_LOGGER` to log the results to a different location, or disable it entirely.  The function should accept `run`, `row`, `content_object`, `success`, and `fail_reason` as arguments.
+
 [Django Data Wizard]: ./index.md
 [models]: ./models.md
 [backends]: ./backends.md
@@ -99,5 +104,6 @@ See [tasks] for more details on how to define custom processing tasks.
 [tasks]: ./tasks.md
 [authentication]: https://www.django-rest-framework.org/api-guide/authentication/
 [permissions]: https://www.django-rest-framework.org/api-guide/permissions/
+[GenericForeignKey]: https://docs.djangoproject.com/en/4.2/ref/contrib/contenttypes/#django.contrib.contenttypes.fields.GenericForeignKey
 
 [idmap.py]: https://github.com/wq/django-data-wizard/blob/master/data_wizard/idmap.py

tests/settings.py

@@ -116,6 +116,9 @@
     "BACKEND": f"data_wizard.backends.{TEST_BACKEND}",
 }
 
+if TEST_VARIANT == "customlog":
+    DATA_WIZARD["ROW_LOGGER"] = "tests.source_app.wizard.custom_logger"
+
 STATIC_URL = "/static/"
 
 DEBUG = True

tests/source_app/wizard.py

@@ -10,11 +10,11 @@ class CustomWorkflowSerializer(serializers.ModelSerializer):
     class Meta:
         model = CustomWorkflow
         data_wizard = {
-            'auto_import_tasks': [
-                 'tests.source_app.wizard.validate',
-                 'tests.source_app.wizard.check_confirm',
-                 'tests.source_app.wizard.finalize',
-             ],
+            "auto_import_tasks": [
+                "tests.source_app.wizard.validate",
+                "tests.source_app.wizard.check_confirm",
+                "tests.source_app.wizard.finalize",
+            ],
         }
 
 
@@ -25,18 +25,18 @@ class Meta:
 def validate(run):
     workflow = run.content_object
     if not workflow.validated:
-        run.add_event('validated')
+        run.add_event("validated")
         workflow.validated = True
         workflow.save()
     return {}
 
 
 @data_wizard.wizard_task(label="Check Confirmation", url_path=False)
 def check_confirm(run):
-    run.add_event('check_confirm')
+    run.add_event("check_confirm")
     workflow = run.content_object
     if not workflow.confirmed:
-        raise data_wizard.InputNeeded('confirm')
+        raise data_wizard.InputNeeded("confirm")
 
 
 @data_wizard.wizard_task(label="Confirm", url_path="confirm")
@@ -50,25 +50,42 @@ def confirm(run):
 @data_wizard.wizard_task(label="Save Confirmation", url_path="saveconfirm")
 def process_confirm(run, post={}):
     workflow = run.content_object
-    if post.get('confirm'):
-        run.add_event('process_confirm')
+    if post.get("confirm"):
+        run.add_event("process_confirm")
         workflow.confirmed = True
         workflow.save()
 
     return {
         **confirm(run),
-        'current_mode': 'confirm',
+        "current_mode": "confirm",
     }
 
 
 @data_wizard.wizard_task(label="Finalize", url_path="finalize")
 def finalize(run):
-    run.add_event('finalized')
+    run.add_event("finalized")
     workflow = run.content_object
     workflow.finalized = True
     workflow.save()
-    run.send_progress({
-        "current": 3,
-        "total": 3,
-        "skipped": [],
-    }, "SUCCESS")
+    run.send_progress(
+        {
+            "current": 3,
+            "total": 3,
+            "skipped": [],
+        },
+        "SUCCESS",
+    )
+
+
+def custom_logger(run, row, content_object, success, fail_reason):
+    print(
+        f"Logging Row {row} for {run}:",
+        content_object if success else fail_reason,
+    )
+
+    return run.record_set.create(
+        row=row,
+        content_object=content_object,
+        success=success,
+        fail_reason=fail_reason,
+    )