Merge pull request #29 from saritasa-nest/detailed-import-export-seri…

…alizers

Provide detailed response for import api

HISTORY.rst

@@ -2,6 +2,10 @@
 History
 =======
 
+UNRELEASED
+----------
+* Extend response of import job api
+
 0.5.0 (2023-12-19)
 ------------------
 * Drop support of python 3.9

import_export_extensions/admin/model_admins/import_job_admin.py

@@ -141,6 +141,7 @@ def get_readonly_fields(
             readonly_fields.extend(
                 [
                     "resource_path",
+                    "input_errors_file",
                     "data_file",
                     "resource_kwargs",
                 ],
@@ -241,7 +242,10 @@ def get_fieldsets(
         data = (
             _("Importing data"),
             {
-                "fields": ("_input_errors",),
+                "fields": (
+                    "input_errors_file",
+                    "_input_errors",
+                ),
                 "classes": ("collapse",),
             },
         )

import_export_extensions/api/serializers/import_job.py

@@ -5,6 +5,7 @@
 from celery import states
 
 from ... import models, resources
+from . import import_job_details as details
 from .progress import ProgressSerializer
 
 
@@ -26,13 +27,54 @@ class ImportJobSerializer(serializers.ModelSerializer):
 
     progress = ImportProgressSerializer()
 
+    import_params = details.ImportParamsSerializer(
+        read_only=True,
+        source="*",
+    )
+    totals = details.TotalsSerializer(
+        read_only=True,
+        source="*",
+    )
+    parse_error = serializers.CharField(
+        source="error_message",
+        read_only=True,
+        allow_blank=True,
+    )
+    input_error = details.InputErrorSerializer(
+        source="*",
+        read_only=True,
+    )
+    skipped_errors = details.SkippedErrorsSerializer(
+        source="*",
+        read_only=True,
+    )
+    importing_data = details.ImportingDataSerializer(
+        read_only=True,
+        source="*",
+    )
+    input_errors_file = serializers.FileField(
+        read_only=True,
+        allow_null=True,
+    )
+    is_all_rows_shown = details.IsAllRowsShowField(
+        source="*",
+        read_only=True,
+    )
+
     class Meta:
         model = models.ImportJob
         fields = (
             "id",
-            "import_status",
-            "data_file",
             "progress",
+            "import_status",
+            "import_params",
+            "totals",
+            "parse_error",
+            "input_error",
+            "skipped_errors",
+            "is_all_rows_shown",
+            "importing_data",
+            "input_errors_file",
             "import_started",
             "import_finished",
             "force_import",

import_export_extensions/api/serializers/import_job_details.py

@@ -0,0 +1,199 @@
+import itertools
+import typing
+
+from rest_framework import serializers
+
+from import_export.results import RowResult
+
+from ... import models
+
+
+class SkippedErrorsDict(typing.TypedDict):
+    """Typed dict for skipped errors."""
+    non_field_skipped_errors: list[str]
+    field_skipped_errors: dict[str, list[str]]
+
+
+class ImportParamsSerializer(serializers.Serializer):
+    """Serializer for representing import parameters."""
+    data_file = serializers.FileField()
+    resource_path = serializers.CharField()
+    resource_kwargs = serializers.CharField()
+
+
+class ImportDiffSerializer(serializers.Serializer):
+    """Serializer for representing importing rows diff."""
+    previous = serializers.CharField(allow_blank=True, allow_null=True)
+    current = serializers.CharField(allow_blank=True, allow_null=True)
+
+
+class ImportRowSerializer(serializers.Serializer):
+    """Serializer for representing importing rows.
+
+    Used to generate correct openapi spec.
+
+    """
+    operation = serializers.CharField()
+    parsed_fields = serializers.ListField(
+        child=ImportDiffSerializer(allow_null=True),
+        allow_null=True,
+    )
+
+
+class ImportingDataSerializer(serializers.Serializer):
+    """Serializer for representing importing data."""
+    headers = serializers.ListField(
+        child=serializers.CharField(),
+    )
+    rows = serializers.ListField(
+        child=ImportRowSerializer(),
+    )
+
+    def to_representation(self, instance: models.ImportJob):
+        """Return dict with import details."""
+        if instance.import_status not in models.ImportJob.success_statuses:
+            return super().to_representation(self.get_initial())
+
+        rows = []
+        resource = instance.resource
+        for row in instance.result.rows:
+            # errors displayed in input_error.row_errors(InputErrorSerializer)
+            if row.import_type == RowResult.IMPORT_TYPE_ERROR:
+                continue
+
+            original_fields = [
+                resource.export_field(field, row.original)
+                if row.original else ""
+                for field in resource.get_user_visible_fields()
+            ]
+            current_fields = [
+                resource.export_field(field, row.instance)
+                for field in resource.get_user_visible_fields()
+            ]
+
+            rows.append({
+                "operation": row.import_type,
+                "parsed_fields": [
+                    {
+                        "previous": original_field,
+                        "current": current_field,
+                    } for original_field, current_field
+                    in itertools.zip_longest(
+                        original_fields,
+                        current_fields,
+                        fillvalue="",
+                    )
+                ],
+            })
+
+        importing_data = {
+            "headers": instance.result.diff_headers,
+            "rows": rows,
+        }
+        return super().to_representation(importing_data)
+
+
+class TotalsSerializer(serializers.Serializer):
+    """Serializer to represent import totals."""
+    new = serializers.IntegerField(allow_null=True, required=False)
+    update = serializers.IntegerField(allow_null=True, required=False)
+    delete = serializers.IntegerField(allow_null=True, required=False)
+    skip = serializers.IntegerField(allow_null=True, required=False)
+    error = serializers.IntegerField(allow_null=True, required=False)
+
+    def to_representation(self, instance):
+        """Return dict with import totals."""
+        if instance.import_status not in models.ImportJob.results_statuses:
+            return super().to_representation(self.get_initial())
+        return super().to_representation(instance.result.totals)
+
+
+class RowError(serializers.Serializer):
+    """Represent single row errors."""
+    line = serializers.IntegerField()
+    error = serializers.CharField()
+    row = serializers.ListField(
+        child=serializers.CharField(),
+    )
+
+
+class InputErrorSerializer(serializers.Serializer):
+    """Represent Input errors."""
+    base_errors = serializers.ListField(
+        child=serializers.CharField(),
+    )
+    row_errors = serializers.ListField(
+        child=serializers.ListField(
+            child=RowError(),
+        ),
+    )
+
+    def to_representation(self, instance: models.ImportJob):
+        """Return dict with input errors."""
+        if instance.import_status not in models.ImportJob.results_statuses:
+            return super().to_representation(self.get_initial())
+
+        input_errors: dict[str, list[typing.Any]] = {
+            "base_errors": [],
+            "row_errors": [],
+        }
+
+        if instance.result.base_errors:
+            input_errors["base_errors"] = [
+                str(error.error) for error in instance.result.base_errors
+            ]
+
+        if instance.result.row_errors():
+            for line, errors in instance.result.row_errors():
+                line_errors = [
+                    {
+                        "line": line,
+                        "error": str(error.error),
+                        "row": error.row.values(),
+                    } for error in errors
+                ]
+                input_errors["row_errors"].append(line_errors)
+
+        return super().to_representation(input_errors)
+
+
+class IsAllRowsShowField(serializers.BooleanField):
+    """Field for representing `all_rows_saved` value."""
+
+    def to_representation(self, instance):
+        """Return boolean if all rows shown in importing data."""
+        if instance.import_status not in models.ImportJob.success_statuses:
+            return False
+        return instance.result.total_rows == len(instance.result.rows)
+
+
+class SkippedErrorsSerializer(serializers.Serializer):
+    """Serializer for import job skipped rows."""
+
+    non_field_skipped_errors = serializers.ListField(
+        child=serializers.CharField(),
+    )
+    field_skipped_errors = serializers.DictField(
+        child=serializers.ListField(child=serializers.CharField()),
+    )
+
+    def to_representation(self, instance: models.ImportJob):
+        """Parse skipped errors from import job result."""
+        if (
+            instance.import_status
+            not in models.ImportJob.results_statuses
+        ):
+            return super().to_representation(self.get_initial())
+        skipped_errors: SkippedErrorsDict = {
+            "non_field_skipped_errors": [],
+            "field_skipped_errors": {},
+        }
+        for row in instance.result.skipped_rows:
+            non_field_errors = [
+                error.error for error in row.non_field_skipped_errors
+            ]
+            skipped_errors["non_field_skipped_errors"].extend(non_field_errors)
+            for field, errors in row.field_skipped_errors.items():
+                errors = [error.messages for error in errors]
+                skipped_errors["field_skipped_errors"][field] = errors
+        return super().to_representation(skipped_errors)

import_export_extensions/migrations/0006_importjob_input_errors_file.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.7 on 2024-01-15 10:40
+
+from django.db import migrations, models
+import functools
+import import_export_extensions.models.tools
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("import_export_extensions", "0005_importjob_force_import"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="input_errors_file",
+            field=models.FileField(
+                help_text="File that contain failed rows",
+                max_length=512,
+                null=True,
+                upload_to=functools.partial(
+                    import_export_extensions.models.tools.upload_file_to,
+                    *(),
+                    **{"main_folder_name": "import"}
+                ),
+                verbose_name="Input errors file",
+            ),
+        ),
+    ]