[fix] Fix import errors from files generated by OW #793

Ensured the following works:

- Export devices using the export button on device changelist to CSV format
- Import the exported devices in the previous step

Added tests for main use cases.
Some cases may be missing and we may
have to fix them as we encounter them.

Closes #793

README.rst

@@ -1057,7 +1057,7 @@ for example, the following CSV file will import a device named
 ``TestImport`` with mac address ``00:11:22:09:44:55`` in the organization with
 UUID ``3cb5e18c-0312-48ab-8dbd-038b8415bd6f``::
 
-    organization,name,mac_address
+    organization_id,name,mac_address
     3cb5e18c-0312-48ab-8dbd-038b8415bd6f,TestImport,00:11:22:09:44:55
 
 .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.1/import-export/import-page.png

openwisp_controller/config/base/config.py

@@ -211,7 +211,8 @@ def templates_changed(cls, action, instance, **kwargs):
         (m2m relationships are first cleared and then added back),
         there fore we need to ignore it to avoid emitting signals twice
         """
-        if action not in ['post_add', 'post_remove']:
+        # execute only after a config has been saved or deleted
+        if action not in ['post_add', 'post_remove'] or instance._state.adding:
             return
         # use atomic to ensure any code bound to
         # be executed via transaction.on_commit
@@ -235,7 +236,8 @@ def manage_vpn_clients(cls, action, instance, pk_set, **kwargs):
         This method is called from a django signal (m2m_changed)
         see config.apps.ConfigConfig.connect_signals
         """
-        if action not in ['post_add', 'post_remove']:
+        # execute only after a config has been saved or deleted
+        if action not in ['post_add', 'post_remove'] or instance._state.adding:
             return
         vpn_client_model = cls.vpn.through
         # coming from signal

openwisp_controller/config/exportable.py

@@ -1,4 +1,5 @@
 import json
+import uuid
 
 from django.core.exceptions import ObjectDoesNotExist
 from import_export import resources, widgets
@@ -9,60 +10,162 @@
 
 Device = load_model('config', 'Device')
 Config = load_model('config', 'Config')
+Template = load_model('config', 'Template')
+
+
+class ManyToManyWidget(widgets.ManyToManyWidget):
+    """
+    https://github.com/django-import-export/django-import-export/issues/1788
+    """
+
+    def clean(self, value, row=None, **kwargs):
+        cleaned = list(super().clean(value, row=None, **kwargs))
+        return cleaned or ''
 
 
 class DeviceResource(resources.ModelResource):
-    organization = Field(attribute='organization__name', column_name='organization')
-    group = Field(attribute='group__name', column_name='group')
-    config_status = Field(attribute='config__status', column_name='config_status')
-    config_backend = Field(attribute='config__backend', column_name='config_backend')
-    config_data = Field(attribute='config__config', column_name='config_data')
-    config_context = Field(attribute='config__context', column_name='config_context')
+    organization = Field(
+        attribute='organization__name', column_name='organization', readonly=True
+    )
+    group = Field(attribute='group__name', column_name='group', readonly=True)
+    # readonly because config status is dynamically handled by the system
+    config_status = Field(
+        attribute='config__status', column_name='config_status', readonly=True
+    )
+    config_backend = Field(
+        attribute='config__backend',
+        column_name='config_backend',
+        default=None,
+        saves_null_values=False,
+    )
+    config_data = Field(
+        attribute='config__config',
+        column_name='config_data',
+        default=None,
+        saves_null_values=False,
+    )
+    config_context = Field(
+        attribute='config__context',
+        column_name='config_context',
+        default=None,
+        saves_null_values=False,
+    )
     config_templates = Field(
         attribute='config__templates',
         column_name='config_templates',
-        widget=widgets.ManyToManyWidget(Config, field='pk', separator=','),
+        widget=ManyToManyWidget(Template, field='pk', separator=','),
+        default=None,
+        saves_null_values=False,
+    )
+    organization_id = Field(
+        attribute='organization_id',
+        column_name='organization_id',
+        default=None,
+        saves_null_values=False,
+    )
+    group_id = Field(
+        attribute='group_id',
+        column_name='group_id',
+        default=None,
+        saves_null_values=False,
     )
-    organization_id = Field(attribute='organization_id', column_name='organization_id')
-    group_id = Field(attribute='group_id', column_name='group_id')
 
     def dehydrate_config_data(self, device):
+        """returns JSON instead of OrderedDict representation"""
         try:
             return json.dumps(device.config.config, sort_keys=True)
         except ObjectDoesNotExist:
             pass
 
     def dehydrate_config_context(self, device):
+        """returns JSON instead of OrderedDict representation"""
         try:
             return json.dumps(device.config.context, sort_keys=True)
         except ObjectDoesNotExist:
             pass
 
+    def before_import_row(self, row, **kwargs):
+        if 'id' in row:
+            row['id'] = uuid.UUID(row['id'])
+        # if JSON is invalid this line will fail
+        # but will be catched by the import-export app
+        if row.get('config_data'):
+            row['config_data'] = json.loads(row['config_data'])
+        if row.get('config_context'):
+            row['config_context'] = json.loads(row['config_context'])
+            if not row['config_context']:
+                row['config_context'] = {}
+
+    def get_or_init_instance(self, instance_loader, row):
+        instance, new = super().get_or_init_instance(instance_loader, row)
+        self._after_init_instance(instance, new, row)
+        return instance, new
+
+    def _row_has_config_data(self, row):
+        """
+        Returns True if dict row has at
+        least one valid config attribute
+        """
+        return any(row.get(attr) for attr in self.Meta.config_fields)
+
+    def _after_init_instance(self, instance, new, row):
+        # initialize empty Config instance to allow
+        # deeper level code to set attributes to it
+        if self._row_has_config_data(row) and not instance._has_config():
+            instance.config = Config()
+
+    def validate_instance(
+        self, instance, import_validation_errors=None, validate_unique=True
+    ):
+        super().validate_instance(
+            instance, import_validation_errors=None, validate_unique=True
+        )
+        if not instance._has_config():
+            return
+        config = instance.config
+        # make sure device_id on config instance is set correctly
+        if config.device_id != instance.id:
+            config.device_id = instance.id
+
+    def after_save_instance(self, instance, using_transactions, dry_run):
+        super().after_save_instance(instance, using_transactions, dry_run)
+        if not dry_run:
+            # save config afte device has been imported
+            if instance._has_config():
+                instance.config.save()
+
     class Meta:
         model = Device
-        fields = [
-            'name',
-            'mac_address',
-            'organization',
-            'group',
-            'model',
-            'os',
-            'system',
-            'notes',
-            'last_ip',
-            'management_ip',
+        config_fields = [
             'config_status',
             'config_backend',
             'config_data',
             'config_context',
             'config_templates',
-            'created',
-            'modified',
-            'id',
-            'key',
-            'organization_id',
-            'group_id',
         ]
+        fields = (
+            [
+                'name',
+                'mac_address',
+                'organization',
+                'group',
+                'model',
+                'os',
+                'system',
+                'notes',
+                'last_ip',
+                'management_ip',
+            ]
+            + config_fields
+            + [
+                'created',
+                'modified',
+                'id',
+                'key',
+                'organization_id',
+                'group_id',
+            ]
+        )
         if app_settings.HARDWARE_ID_ENABLED:
             fields.insert(1, 'hardware_id')
         export_order = fields