[IMP] base_import: Import images,icon,logo all via URL.

Currently images must be imported one by one, unless we use a script, but it's too technical.
it's especially annoying when deploying ecommerce apps with lots of products.

Using this commit we can import images, via URL or base64 in the csv:
    * main image ( example product image)
    * several shop images (activate multi-image in ecommerce settings)
    * variant-specific image (activate variants)

* Allow maximum 10mb size for images to import and only administrator can import image via url.
   used requests.session with stream=True which allow you to read large streams or files without reading them into memory.
* set timeout default 3 seconds if requests will wait for your client to establish a connection to a remote.
* added _can_import_remote_urls method to provide hook for trial/pack check in saas.
* three parameter in tools config to set import image timeout, import image maxbytes and import image regex.

addons/base_import/models/base_import.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import base64
 import datetime
 import io
 import itertools
@@ -9,15 +10,19 @@
 import operator
 import os
 import re
+import requests
 
 from odoo import api, fields, models
+from odoo.exceptions import AccessError
 from odoo.tools.translate import _
 from odoo.tools.mimetypes import guess_mimetype
 from odoo.tools.misc import ustr
-from odoo.tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT, pycompat
+from odoo.tools import config, DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT, pycompat
 
 FIELDS_RECURSION_LIMIT = 2
 ERROR_PREVIEW_BYTES = 200
+DEFAULT_IMAGE_CHUNK_SIZE = 32768
+IMAGE_FIELDS = ["icon", "image", "logo", "picture"]
 _logger = logging.getLogger(__name__)
 
 try:
@@ -607,6 +612,9 @@ def _parse_import_data(self, data, import_fields, options):
     def _parse_import_data_recursive(self, model, prefix, data, import_fields, options):
         # Get fields of type date/datetime
         all_fields = self.env[model].fields_get()
+        url_regex = config.get("import_image_regex")
+        request_timeout = int(config.get("import_image_timeout"))
+        maxsize = int(config.get("import_image_maxbytes"))
         for name, field in all_fields.items():
             name = prefix + name
             if field['type'] in ('date', 'datetime') and name in import_fields:
@@ -637,6 +645,29 @@ def _parse_import_data_recursive(self, model, prefix, data, import_fields, optio
                 # We should be able to manage both case
                 index = import_fields.index(name)
                 self._parse_float_from_data(data, index, name, options)
+            elif field['type'] == 'binary' and field.get('attachment') and any(f in name for f in IMAGE_FIELDS) and name in import_fields:
+                if not self.env.user._can_import_remote_urls():
+                    raise AccessError(_("Only administrator can import images via URL"))
+
+                index = import_fields.index(name)
+
+                with requests.Session() as session:
+                    for num, line in enumerate(data):
+                        if re.match(url_regex, line[index]):
+                            try:
+                                response = session.get(line[index], stream=True, timeout=request_timeout)
+                                response.raise_for_status()
+                                if response.headers.get('Content-Length') and int(response.headers['Content-Length']) > maxsize:
+                                    raise ValueError(_("File size exceeds configured maximum (%s bytes)") % maxsize)
+                                content = bytearray()
+                                for chunk in response.iter_content(DEFAULT_IMAGE_CHUNK_SIZE):
+                                    content += chunk
+                                    if len(content) > maxsize:
+                                        raise ValueError(_("File size exceeds configured maximum (%s bytes)") % maxsize)
+                                line[index] = base64.b64encode(content)
+                            except Exception as e:
+                                raise ValueError(_("Could not retrieve URL:") +  "%s [%s: L%d]: %s" % (line[index], name, num + 1, e))
+
         return data
 
     @api.multi
@@ -667,10 +698,10 @@ def do(self, fields, options, dryrun=False):
             data, import_fields = self._convert_import_data(fields, options)
             # Parse date and float field
             data = self._parse_import_data(data, import_fields, options)
-        except ValueError as error:
+        except Exception as error:
             return [{
                 'type': 'error',
-                'message': pycompat.text_type(error),
+                'message': pycompat.text_type(error.args[0]),
                 'record': False,
             }]
 

odoo/addons/base/models/res_users.py

@@ -587,6 +587,12 @@ def _is_superuser(self):
         self.ensure_one()
         return self.id == SUPERUSER_ID
 
+    @api.multi
+    def _can_import_remote_urls(self):
+        # For the SaaS, to provide a hook for the trial/paid check.
+        self.ensure_one()
+        return self._is_admin()
+
     @api.model
     def get_company_currency_id(self):
         return self.env.user.company_id.currency_id.id

odoo/tools/config.py

@@ -18,6 +18,9 @@
 crypt_context = CryptContext(schemes=['pbkdf2_sha512', 'plaintext'],
                              deprecated=['plaintext'])
 
+DEFAULT_IMPORT_IMAGE_TIMEOUT = 3
+DEFAULT_IMPORT_IMAGE_MAXBYTES = 10 * 1024 * 1024
+DEFAULT_IMPORT_IMAGE_REGEX = r"(?:http|https)://.*(?:png|jpe?g|tiff?|gif|bmp)"
 class MyOption (optparse.Option, object):
     """ optparse Option with two additional attributes.
 
@@ -75,6 +78,9 @@ def __init__(self, fname=None):
             'admin_passwd': 'admin',
             'csv_internal_sep': ',',
             'publisher_warranty_url': 'http://services.openerp.com/publisher-warranty/',
+            'import_image_timeout': DEFAULT_IMPORT_IMAGE_TIMEOUT,
+            'import_image_maxbytes': DEFAULT_IMPORT_IMAGE_MAXBYTES,
+            'import_image_regex': DEFAULT_IMPORT_IMAGE_REGEX,
             'reportgz': False,
             'root_path': None,
         }