[WIP] Data importer

.github/actions/setup/action.yaml

@@ -98,4 +98,4 @@ runs:
       - name: Run invoke update
         if: ${{ inputs.update == 'true' }}
         shell: bash
-        run: invoke update --uv
+        run: invoke update --uv --skip-backup --skip-static

src/backend/InvenTree/InvenTree/api.py

@@ -383,42 +383,6 @@ class ListCreateDestroyAPIView(BulkDeleteMixin, ListCreateAPI):
     ...
 
 
-class APIDownloadMixin:
-    """Mixin for enabling a LIST endpoint to be downloaded a file.
-
-    To download the data, add the ?export=<fmt> to the query string.
-
-    The implementing class must provided a download_queryset method,
-    e.g.
-
-    def download_queryset(self, queryset, export_format):
-        dataset = StockItemResource().export(queryset=queryset)
-
-        filedata = dataset.export(export_format)
-
-        filename = 'InvenTree_Stocktake_{date}.{fmt}'.format(
-            date=datetime.now().strftime("%d-%b-%Y"),
-            fmt=export_format
-        )
-
-        return DownloadFile(filedata, filename)
-    """
-
-    def get(self, request, *args, **kwargs):
-        """Generic handler for a download request."""
-        export_format = request.query_params.get('export', None)
-
-        if export_format and export_format in ['csv', 'tsv', 'xls', 'xlsx']:
-            queryset = self.filter_queryset(self.get_queryset())
-            return self.download_queryset(queryset, export_format)
-        # Default to the parent class implementation
-        return super().get(request, *args, **kwargs)
-
-    def download_queryset(self, queryset, export_format):
-        """This function must be implemented to provide a downloadFile request."""
-        raise NotImplementedError('download_queryset method not implemented!')
-
-
 class APISearchViewSerializer(serializers.Serializer):
     """Serializer for the APISearchView."""
 

src/backend/InvenTree/InvenTree/api_version.py

@@ -1,11 +1,14 @@
 """InvenTree API version information."""
 
 # InvenTree API version
-INVENTREE_API_VERSION = 210
+INVENTREE_API_VERSION = 211
 
 """Increment this API version number whenever there is a significant change to the API that any clients need to know about."""
 
 INVENTREE_API_TEXT = """
+v211 - 2024-06-26 : https://github.com/inventree/InvenTree/pull/6911
+    - Adds API endpoints for managing data import and export
+
 v210 - 2024-06-26 : https://github.com/inventree/InvenTree/pull/7518
     - Adds translateable text to User API fields
 

src/backend/InvenTree/InvenTree/forms.py

@@ -190,7 +190,7 @@ def __init__(self, *args, **kwargs):
 
         # check for two password fields
         if not get_global_setting('LOGIN_SIGNUP_PWD_TWICE'):
-            self.fields.pop('password2')
+            self.fields.pop('password2', None)
 
         # reorder fields
         set_form_field_order(

src/backend/InvenTree/InvenTree/helpers.py

@@ -429,8 +429,8 @@ def MakeBarcode(cls_name, object_pk: int, object_data=None, **kwargs):
 
 
 def GetExportFormats():
-    """Return a list of allowable file formats for exporting data."""
-    return ['csv', 'tsv', 'xls', 'xlsx', 'json', 'yaml']
+    """Return a list of allowable file formats for importing or exporting tabular data."""
+    return ['csv', 'xlsx', 'tsv', 'json']
 
 
 def DownloadFile(

src/backend/InvenTree/InvenTree/helpers_model.py

@@ -252,7 +252,7 @@ def render_currency(
 
 
 def getModelsWithMixin(mixin_class) -> list:
-    """Return a list of models that inherit from the given mixin class.
+    """Return a list of database models that inherit from the given mixin class.
 
     Args:
         mixin_class: The mixin class to search for

src/backend/InvenTree/InvenTree/metadata.py

@@ -137,10 +137,10 @@ def override_value(self, field_name: str, field_key: str, field_value, model_val
             - field_value: The value of the field (if available)
             - model_value: The equivalent value of the model (if available)
         """
-        if model_value and not field_value:
+        if field_value is None and model_value is not None:
             return model_value
 
-        if field_value and not model_value:
+        if model_value is None and field_value is not None:
             return field_value
 
         # Callable values will be evaluated later
@@ -160,6 +160,8 @@ def get_serializer_info(self, serializer):
         """Override get_serializer_info so that we can add 'default' values to any fields whose Meta.model specifies a default value."""
         self.serializer = serializer
 
+        request = getattr(self, 'request', None)
+
         serializer_info = super().get_serializer_info(serializer)
 
         # Look for any dynamic fields which were not available when the serializer was instantiated
@@ -169,12 +171,19 @@ def get_serializer_info(self, serializer):
                     # Already know about this one
                     continue
 
-                if hasattr(serializer, field_name):
-                    field = getattr(serializer, field_name)
+                if field := getattr(serializer, field_name, None):
                     serializer_info[field_name] = self.get_field_info(field)
 
         model_class = None
 
+        # Extract read_only_fields and write_only_fields from the Meta class (if available)
+        if meta := getattr(serializer, 'Meta', None):
+            read_only_fields = getattr(meta, 'read_only_fields', [])
+            write_only_fields = getattr(meta, 'write_only_fields', [])
+        else:
+            read_only_fields = []
+            write_only_fields = []
+
         # Attributes to copy extra attributes from the model to the field (if they don't exist)
         # Note that the attributes may be named differently on the underlying model!
         extra_attributes = {
@@ -188,16 +197,20 @@ def get_serializer_info(self, serializer):
 
             model_fields = model_meta.get_field_info(model_class)
 
-            model_default_func = getattr(model_class, 'api_defaults', None)
-
-            if model_default_func:
-                model_default_values = model_class.api_defaults(self.request)
+            if model_default_func := getattr(model_class, 'api_defaults', None):
+                model_default_values = model_default_func(request=request) or {}
             else:
                 model_default_values = {}
 
             # Iterate through simple fields
             for name, field in model_fields.fields.items():
                 if name in serializer_info.keys():
+                    if name in read_only_fields:
+                        serializer_info[name]['read_only'] = True
+
+                    if name in write_only_fields:
+                        serializer_info[name]['write_only'] = True
+
                     if field.has_default():
                         default = field.default
 
@@ -231,6 +244,12 @@ def get_serializer_info(self, serializer):
                     # Ignore reverse relations
                     continue
 
+                if name in read_only_fields:
+                    serializer_info[name]['read_only'] = True
+
+                if name in write_only_fields:
+                    serializer_info[name]['write_only'] = True
+
                 # Extract and provide the "limit_choices_to" filters
                 # This is used to automatically filter AJAX requests
                 serializer_info[name]['filters'] = (
@@ -261,7 +280,8 @@ def get_serializer_info(self, serializer):
 
         if instance is None and model_class is not None:
             # Attempt to find the instance based on kwargs lookup
-            kwargs = getattr(self.view, 'kwargs', None)
+            view = getattr(self, 'view', None)
+            kwargs = getattr(view, 'kwargs', None) if view else None
 
             if kwargs:
                 pk = None
@@ -318,8 +338,10 @@ def get_field_info(self, field):
 
         # Force non-nullable fields to read as "required"
         # (even if there is a default value!)
-        if not field.allow_null and not (
-            hasattr(field, 'allow_blank') and field.allow_blank
+        if (
+            'required' not in field_info
+            and not field.allow_null
+            and not (hasattr(field, 'allow_blank') and field.allow_blank)
         ):
             field_info['required'] = True
 
@@ -346,8 +368,11 @@ def get_field_info(self, field):
                     field_info['api_url'] = '/api/user/'
                 elif field_info['model'] == 'contenttype':
                     field_info['api_url'] = '/api/contenttype/'
-                else:
+                elif hasattr(model, 'get_api_url'):
                     field_info['api_url'] = model.get_api_url()
+                else:
+                    logger.warning("'get_api_url' method not defined for %s", model)
+                    field_info['api_url'] = getattr(model, 'api_url', None)
 
                 # Handle custom 'primary key' field
                 field_info['pk_field'] = getattr(field, 'pk_field', 'pk') or 'pk'

src/backend/InvenTree/InvenTree/models.py

@@ -222,6 +222,9 @@ class DataImportMixin(object):
     Models which implement this mixin should provide information on the fields available for import
     """
 
+    # TODO: This mixin should be removed after https://github.com/inventree/InvenTree/pull/6911 is implemented
+    # TODO: This approach to data import functionality is *outdated*
+
     # Define a map of fields available for import
     IMPORT_FIELDS = {}
 

src/backend/InvenTree/InvenTree/serializers.py

@@ -856,7 +856,7 @@ def skip_create_fields(self):
 
     remote_image = serializers.URLField(
         required=False,
-        allow_blank=False,
+        allow_blank=True,
         write_only=True,
         label=_('Remote Image'),
         help_text=_('URL of remote image file'),

src/backend/InvenTree/InvenTree/settings.py

@@ -198,6 +198,7 @@
     'stock.apps.StockConfig',
     'users.apps.UsersConfig',
     'machine.apps.MachineConfig',
+    'importer.apps.ImporterConfig',
     'web',
     'generic',
     'InvenTree.apps.InvenTreeConfig',  # InvenTree app runs last

src/backend/InvenTree/InvenTree/static/script/inventree/inventree.js

@@ -60,10 +60,6 @@ function exportFormatOptions() {
             value: 'tsv',
             display_name: 'TSV',
         },
-        {
-            value: 'xls',
-            display_name: 'XLS',
-        },
         {
             value: 'xlsx',
             display_name: 'XLSX',

src/backend/InvenTree/InvenTree/tasks.py

@@ -256,8 +256,8 @@ def offload_task(
             _func(*args, **kwargs)
         except Exception as exc:
             log_error('InvenTree.offload_task')
-            raise_warning(f"WARNING: '{taskname}' not started due to {str(exc)}")
-            return False
+            raise_warning(f"WARNING: '{taskname}' failed due to {str(exc)}")
+            raise exc
 
     # Finally, task either completed successfully or was offloaded
     return True

src/backend/InvenTree/InvenTree/templatetags/inventree_extras.py

@@ -455,7 +455,10 @@ def get_user_color_theme(user):
     """Get current user color theme."""
     from common.models import ColorTheme
 
-    if not user.is_authenticated:
+    try:
+        if not user.is_authenticated:
+            return 'default'
+    except Exception:
         return 'default'
 
     try:

src/backend/InvenTree/InvenTree/unit_test.py

@@ -412,12 +412,12 @@ def download_file(
         # Extract filename
         disposition = response.headers['Content-Disposition']
 
-        result = re.search(r'attachment; filename="([\w.]+)"', disposition)
+        result = re.search(r'attachment; filename="([\w\d\-.]+)"', disposition)
 
         fn = result.groups()[0]
 
         if expected_fn is not None:
-            self.assertEqual(expected_fn, fn)
+            self.assertRegex(fn, expected_fn)
 
         if decode:
             # Decode data and return as StringIO file object

src/backend/InvenTree/InvenTree/urls.py

@@ -21,6 +21,7 @@
 import build.api
 import common.api
 import company.api
+import importer.api
 import machine.api
 import order.api
 import part.api
@@ -80,11 +81,19 @@
 
 apipatterns = [
     # Global search
-    path('search/', APISearchView.as_view(), name='api-search'),
-    path('settings/', include(common.api.settings_api_urls)),
-    path('part/', include(part.api.part_api_urls)),
+    path('admin/', include(common.api.admin_api_urls)),
     path('bom/', include(part.api.bom_api_urls)),
+    path('build/', include(build.api.build_api_urls)),
     path('company/', include(company.api.company_api_urls)),
+    path('importer/', include(importer.api.importer_api_urls)),
+    path('label/', include(report.api.label_api_urls)),
+    path('machine/', include(machine.api.machine_api_urls)),
+    path('order/', include(order.api.order_api_urls)),
+    path('part/', include(part.api.part_api_urls)),
+    path('report/', include(report.api.report_api_urls)),
+    path('search/', APISearchView.as_view(), name='api-search'),
+    path('settings/', include(common.api.settings_api_urls)),
+    path('stock/', include(stock.api.stock_api_urls)),
     path(
         'generate/',
         include([
@@ -100,14 +109,7 @@
             ),
         ]),
     ),
-    path('stock/', include(stock.api.stock_api_urls)),
-    path('build/', include(build.api.build_api_urls)),
-    path('order/', include(order.api.order_api_urls)),
-    path('label/', include(report.api.label_api_urls)),
-    path('report/', include(report.api.report_api_urls)),
-    path('machine/', include(machine.api.machine_api_urls)),
     path('user/', include(users.api.user_urls)),
-    path('admin/', include(common.api.admin_api_urls)),
     path('web/', include(web_api_urls)),
     # Plugin endpoints
     path('', include(plugin.api.plugin_api_urls)),