Merge PR #109 into 14.0

Signed-off-by simahawk

connector_importer/__manifest__.py

@@ -19,6 +19,7 @@
         "security/ir.model.access.csv",
         "views/backend_views.xml",
         "views/recordset_views.xml",
+        "views/import_type_views.xml",
         "views/source_views.xml",
         "views/report_template.xml",
         "views/docs_template.xml",

connector_importer/components/__init__.py

@@ -6,3 +6,4 @@
 from . import importer_csv_std
 from . import mapper
 from . import automapper
+from . import dynamicmapper

connector_importer/components/dynamicmapper.py

@@ -0,0 +1,161 @@
+# Copyright 2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)
+
+from odoo.addons.component.core import Component
+from odoo.addons.connector.components.mapper import mapping
+
+from ..log import logger
+from ..utils.mapper_utils import backend_to_rel, convert, xmlid_to_rel
+
+
+class DynamicMapper(Component):
+    """A mapper that dynamically converts input data to odoo fields values."""
+
+    _name = "importer.mapper.dynamic"
+    _inherit = "importer.base.mapper"
+    _usage = "importer.dynamicmapper"
+
+    @mapping
+    def dynamic_fields(self, record):
+        """Resolve values for non mapped keys.
+
+        Source keys = destination keys.
+        """
+        # TODO: add tests!
+        model = self.work.model_name
+        vals = {}
+        available_fields = self.env[model].fields_get()
+        prefix = self._source_key_prefix
+        clean_record = self._clean_record(record)
+        required_keys = self._required_keys()
+        missing_required_keys = []
+        for source_fname in self._non_mapped_keys(clean_record):
+            if source_fname in ("id", "xid::id"):
+                # Never convert IDs
+                continue
+            fname = source_fname
+            if prefix and source_fname.startswith(prefix):
+                # Eg: prefix all supplier fields w/ `supplier_`
+                fname = fname[len(prefix) :]
+                clean_record[fname] = clean_record.pop(source_fname)
+            if "::" in fname:
+                # Eg: transformers like `xid::``
+                fname = fname.split("::")[-1]
+                clean_record[fname] = clean_record.pop(source_fname)
+            if available_fields.get(fname):
+                fspec = available_fields.get(fname)
+                ftype = fspec["type"]
+                if self._is_xmlid_key(source_fname, ftype):
+                    ftype = "_xmlid"
+                converter = self._get_converter(fname, ftype)
+                if converter:
+                    value = converter(self, clean_record, fname)
+                    if not value:
+                        if source_fname in self._source_key_empty_skip:
+                            continue
+                        if fname in required_keys:
+                            missing_required_keys.append(fname)
+                    vals[fname] = value
+                else:
+                    logger.debug(
+                        "Dynamic mapper cannot find converte for field `%s`", fname
+                    )
+        if missing_required_keys:
+            vals.update(self._get_defaults(missing_required_keys))
+            for k in missing_required_keys:
+                if k in vals and not vals[k]:
+                    # Discard empty values for required keys.
+                    # Avoids overriding values that might be already set
+                    # and that cannot be emptied.
+                    vals.pop(k)
+        return vals
+
+    def _clean_record(self, record):
+        valid_keys = self._get_valid_keys(record)
+        return {k: v for k, v in record.items() if k in valid_keys}
+
+    def _get_valid_keys(self, record):
+        valid_keys = [k for k in record.keys() if not k.startswith("_")]
+        prefix = self._source_key_prefix
+        if prefix:
+            valid_keys = [k for k in valid_keys if k.startswith(prefix)]
+        whitelist = self._source_key_whitelist
+        if whitelist:
+            valid_keys = [k for k in valid_keys if k in whitelist]
+        return tuple(valid_keys)
+
+    def _required_keys(self):
+        return [k for k, v in self.model.fields_get().items() if v["required"]]
+
+    @property
+    def _source_key_whitelist(self):
+        return self.work.options.mapper.get("source_key_whitelist", [])
+
+    @property
+    def _source_key_empty_skip(self):
+        """List of source keys to skip when empty.
+
+        Use cases:
+
+            * field w/ unique constraint but not populated (eg: product barcode)
+            * field not to override when empty
+        """
+        return self.work.options.mapper.get("source_key_empty_skip", [])
+
+    @property
+    def _source_key_prefix(self):
+        return self.work.options.mapper.get("source_key_prefix", "")
+
+    @property
+    def _source_key_xid_module(self):
+        """Module name to use to sanitize XMLids"""
+        return self.work.options.mapper.get("source_key_xid_module", "")
+
+    def _is_xmlid_key(self, fname, ftype):
+        return fname.startswith("xid::") and ftype in (
+            "many2one",
+            "one2many",
+            "many2many",
+        )
+
+    def _dynamic_keys_mapping(self, fname):
+        return {
+            "char": lambda self, rec, fname: rec[fname],
+            "text": lambda self, rec, fname: rec[fname],
+            "selection": lambda self, rec, fname: rec[fname],
+            "integer": convert(fname, "safe_int"),
+            "float": convert(fname, "safe_float"),
+            "boolean": convert(fname, "bool"),
+            "date": convert(fname, "date"),
+            "datetime": convert(fname, "utc_date"),
+            "many2one": backend_to_rel(fname),
+            "many2many": backend_to_rel(fname),
+            "one2many": backend_to_rel(fname),
+            "_xmlid": xmlid_to_rel(
+                fname, sanitize_default_mod_name=self._source_key_xid_module
+            ),
+        }
+
+    def _get_converter(self, fname, ftype):
+        return self._dynamic_keys_mapping(fname).get(ftype)
+
+    _non_mapped_keys_cache = None
+
+    def _non_mapped_keys(self, record):
+        if self._non_mapped_keys_cache is None:
+            all_keys = set(record.keys())
+            mapped_keys = set()
+            # NOTE: keys coming from `@mapping` methods can't be tracked.
+            # Worse case: they get computed twice.
+            # TODO: make sure `dynamic_fields` runs at the end
+            # or move it to `finalize`
+            for pair in self.direct:
+                if isinstance(pair[0], str):
+                    mapped_keys.add(pair[0])
+                elif hasattr(pair[0], "_from_key"):
+                    mapped_keys.add(pair[0]._from_key)
+            self._non_mapped_keys_cache = tuple(all_keys - mapped_keys)
+        return self._non_mapped_keys_cache
+
+    def _get_defaults(self, fnames):
+        return self.model.default_get(fnames)

connector_importer/components/importer.py

@@ -57,25 +57,22 @@ class RecordImporter(Component):
     _name = "importer.record"
     _inherit = ["importer.base.component"]
     _usage = "record.importer"
-    _apply_on = "import.record"
     # log and report errors
     # do not make the whole import fail
     _break_on_error = False
     _record_handler_usage = "odoorecord.handler"
     _tracking_handler_usage = "tracking.handler"
     # a unique key (field name) to retrieve the odoo record
-    # if this key is an external/XML ID, set 'odoo_unique_key_is_xmlid' to True
+    # if this key is an external/XML ID, prefix the name with `xid::` (eg: xid::id)
     odoo_unique_key = ""
-    odoo_unique_key_is_xmlid = False
 
     def _init_importer(self, recordset):
         self.recordset = recordset
         # record handler is responsible for create/write on odoo records
         self.record_handler = self.component(usage=self._record_handler_usage)
         self.record_handler._init_handler(
             importer=self,
-            unique_key=self.odoo_unique_key,
-            unique_key_is_xmlid=self.odoo_unique_key_is_xmlid,
+            unique_key=self.unique_key,
         )
         # tracking handler is responsible for logging and chunk reports
         self.tracker = self.component(usage=self._tracking_handler_usage)
@@ -85,6 +82,14 @@ def _init_importer(self, recordset):
             log_prefix=self.recordset.import_type_id.key + " ",
         )
 
+    @property
+    def unique_key(self):
+        return self.work.options.importer.get("odoo_unique_key", self.odoo_unique_key)
+
+    @property
+    def unique_key_is_xmlid(self):
+        return self.unique_key.startswith("xid::") or self.unique_key == "id"
+
     # Override to not rely on automatic mapper lookup.
     # This is especially needed if you register more than one importer
     # for a given odoo model. Eg: 2 importers for res.partner
@@ -94,11 +99,13 @@ def _init_importer(self, recordset):
     # just an instance cache for the mapper
     _mapper = None
 
-    # TODO: add tests
+    # TODO: do the same for record handler and tracking handler
     def _get_mapper(self):
-        if self._mapper_name:
-            return self.component_by_name(self._mapper_name)
-        return self.component(usage=self._mapper_usage)
+        mapper_name = self.work.options.mapper.get("name", self._mapper_name)
+        if mapper_name:
+            return self.component_by_name(mapper_name)
+        mapper_usage = self.work.options.mapper.get("usage", self._mapper_usage)
+        return self.component(usage=mapper_usage)
 
     @property
     def mapper(self):
@@ -110,6 +117,12 @@ def mapper(self):
     def must_break_on_error(self):
         return self.work.options.importer.get("break_on_error", self._break_on_error)
 
+    @property
+    def must_override_existing(self):
+        return self.work.options.importer.get(
+            "override_existing", self.recordset.override_existing
+        )
+
     def required_keys(self, create=False):
         """Keys that are mandatory to import a line."""
         req = self.mapper.required_keys()
@@ -120,7 +133,7 @@ def required_keys(self, create=False):
             if not isinstance(v, (tuple, list)):
                 req[k] = (v,)
             all_values.extend(req[k])
-        unique_key = self.odoo_unique_key
+        unique_key = self.unique_key
         if (
             unique_key
             and unique_key not in list(req.keys())
@@ -145,7 +158,7 @@ def translatable_langs(self):
     def make_translation_key(self, key, lang):
         sep = self.work.options.importer.get("translation_key_sep", ":")
         regional_lang = self.work.options.importer.get(
-            "translation_use_regional_lang", True
+            "translation_use_regional_lang", False
         )
         if not regional_lang:
             lang = lang[:2]  # eg: "de_DE" -> "de"
@@ -183,14 +196,14 @@ def _check_missing(self, source_key, dest_key, values, orig_values):
         missing = (
             not source_key.startswith("__") and orig_values.get(source_key) is None
         )
-        unique_key = self.odoo_unique_key
+        unique_key = self.unique_key
         if missing:
             msg = "MISSING REQUIRED SOURCE KEY={}".format(source_key)
             if unique_key and values.get(unique_key):
                 msg += ": {}={}".format(unique_key, values[unique_key])
             return {"message": msg}
         missing = not dest_key.startswith("__") and values.get(dest_key) is None
-        is_xmlid = dest_key == unique_key and self.odoo_unique_key_is_xmlid
+        is_xmlid = dest_key == unique_key and self.unique_key_is_xmlid
         if missing and not is_xmlid:
             msg = "MISSING REQUIRED DESTINATION KEY={}".format(dest_key)
             if unique_key and values.get(unique_key):
@@ -217,13 +230,11 @@ def skip_it(self, values, orig_values):
 
         if (
             self.record_handler.odoo_exists(values, orig_values)
-            and not self.recordset.override_existing
+            and not self.must_override_existing
         ):
             msg = "ALREADY EXISTS"
-            if self.odoo_unique_key:
-                msg += ": {}={}".format(
-                    self.odoo_unique_key, values[self.odoo_unique_key]
-                )
+            if self.unique_key:
+                msg += ": {}={}".format(self.unique_key, values[self.unique_key])
             return {
                 "message": msg,
                 "odoo_record": self.record_handler.odoo_find(values, orig_values).id,
@@ -278,7 +289,7 @@ def _record_lines(self):
 
     def _load_mapper_options(self):
         """Retrieve mapper options."""
-        return {"override_existing": self.recordset.override_existing}
+        return {"override_existing": self.must_override_existing}
 
     # TODO: make these contexts customizable via recordset settings
     def _odoo_default_context(self):

connector_importer/components/mapper.py

@@ -2,11 +2,14 @@
 # Copyright 2018 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
+
 from odoo import _, exceptions
 
 from odoo.addons.component.core import Component
 from odoo.addons.connector.components.mapper import mapping
 
+from ..log import logger
+
 
 class ImportMapper(Component):
     _name = "importer.base.mapper"
@@ -49,7 +52,9 @@ def required_keys(self, create=False):
 
         The recordset will use this to show required fields to users.
         """
-        return self.required
+        req = dict(self.required)
+        req.update(self.work.options.mapper.get("required_keys", {}))
+        return req
 
     translatable = []
 
@@ -61,7 +66,23 @@ def translatable_keys(self, create=False):
 
         The recordset will use this to show translatable fields to users.
         """
-        return self.translatable
+        translatable = list(self.translatable)
+        translatable += self.work.options.mapper.get("translatable_keys", [])
+        translatable = self._validate_translate_keys(set(translatable))
+        return tuple(translatable)
+
+    def _validate_translate_keys(self, translatable):
+        valid = []
+        fields_spec = self.model.fields_get(translatable)
+        for fname in translatable:
+            if not fields_spec.get(fname):
+                logger.error("%s - translate key not found: `%s`.", self._name, fname)
+                continue
+            if not fields_spec[fname]["translate"]:
+                logger.error("%s - `%s` key is not translatable.", self._name, fname)
+                continue
+            valid.append(fname)
+        return valid
 
     defaults = [
         # odoo field, value
@@ -91,4 +112,5 @@ def default_values(self, record=None):
                 xmlid, field_value = real_val.split(":")
                 v = self.env.ref(xmlid)[field_value]
             values[k] = v
+        values.update(self.work.options.mapper.get("default_keys", {}))
         return values