Merge branch '4.1.x' into 4.2.x

oio/account/client.py

@@ -18,7 +18,7 @@
 import time
 from oio.api.base import HttpApi
 from oio.common.logger import get_logger
-from oio.common.exceptions import ClientException, OioNetworkException
+from oio.common.exceptions import OioException, OioNetworkException
 from oio.conscience.client import ConscienceClient
 
 
@@ -47,11 +47,8 @@ def __init__(self, conf, endpoint=None, proxy_endpoint=None,
 
     def _get_account_addr(self):
         """Fetch IP and port of an account service from Conscience."""
-        try:
-            acct_instance = self.cs.next_instance('account')
-            acct_addr = acct_instance.get('addr')
-        except Exception:
-            raise ClientException("No Account service found")
+        acct_instance = self.cs.next_instance('account')
+        acct_addr = acct_instance.get('addr')
         return acct_addr
 
     def _refresh_endpoint(self, now=None):
@@ -75,7 +72,7 @@ def _maybe_refresh_endpoint(self):
                         raise
                     self.logger.warn(
                             "Failed to refresh account endpoint: %s", exc)
-                except ClientException:
+                except OioException:
                     if not self.endpoint:
                         # Cannot use the previous one
                         raise

oio/cli/object/object.py

@@ -163,8 +163,9 @@ def take_action(self, parsed_args):
                 results.append((name, 0, None, 'Interrupted'))
                 any_error = True
                 break
-            except Exception:
-                self.log.exception("Failed to upload %s in %s", obj, container)
+            except Exception as exc:
+                self.log.error('Failed to upload %s in %s: %s',
+                               obj, container, exc)
                 any_error = True
                 results.append((name, 0, None, 'Failed'))
 
@@ -313,14 +314,12 @@ def take_action(self, parsed_args):
             version=parsed_args.object_version)
         info = {'account': account,
                 'container': container,
-                'object': obj,
-                'id': data['id'],
-                'version': data['version'],
-                'mime-type': data['mime_type'],
-                'size': data['length'],
-                'hash': data['hash'],
-                'ctime': data['ctime'],
-                'policy': data['policy']}
+                'object': obj}
+        conv = {'id': 'id', 'version': 'version', 'mime-type': 'mime_type',
+                'size': 'length', 'hash': 'hash', 'ctime': 'ctime',
+                'policy': 'policy'}
+        for key0, key1 in conv.items():
+            info[key0] = data.get(key1, 'n/a')
         for k, v in data['properties'].iteritems():
             info['meta.' + k] = v
         return zip(*sorted(info.iteritems()))
@@ -536,12 +535,12 @@ def _autocontainer_loop(self, account, marker=None, limit=None,
                     return
 
     def _list_autocontainer_objects(self, container, account, **kwargs):
+        object_list = []
         if not self.flatns_manager.verify(container):
             self.log.debug("Container %s is not an autocontainer",
                            container)
-            return list()
+            return object_list
         self.log.debug("Listing autocontainer %s", container)
-        object_list = []
         for i in depaginate(
                 self.app.client_manager.storage.object_list,
                 listing_key=lambda x: x['objects'],
@@ -593,6 +592,10 @@ def take_action(self, parsed_args):
                 resp = self.app.client_manager.storage.object_list(
                     account, container, **kwargs)
                 obj_gen = resp['objects']
+                if resp.get('truncated'):
+                    self.log.info(
+                        'Object listing has been truncated, next marker: %s',
+                        resp.get('next_marker'))
 
         if parsed_args.long_listing:
             from oio.common.timestamp import Timestamp
@@ -610,23 +613,33 @@ def _format_props(props):
 
             def _gen_results(objects):
                 for obj in objects:
-                    result = (obj['name'], obj['size'],
-                              obj['hash'], obj['version'],
-                              obj['deleted'], obj['mime_type'],
-                              Timestamp(obj['ctime']).isoformat,
-                              obj['policy'],
-                              _format_props(obj.get('properties', {})))
-                    yield result
-            results = _gen_results(obj_gen)
+                    try:
+                        result = (obj['name'], obj['size'],
+                                  obj['hash'], obj['version'],
+                                  obj['deleted'], obj['mime_type'],
+                                  Timestamp(obj['ctime']).isoformat,
+                                  obj['policy'],
+                                  _format_props(obj.get('properties', {})))
+                        yield result
+                    except KeyError as exc:
+                        self.log.warn("Bad object entry, missing '%s': %s",
+                                      exc, obj)
             columns = ('Name', 'Size', 'Hash', 'Version', 'Deleted',
                        'Content-Type', 'Last-Modified', 'Policy', 'Properties')
         else:
-            results = ((obj['name'],
-                        obj['size'] if not obj['deleted'] else 'deleted',
-                        obj['hash'],
-                        obj['version'])
-                       for obj in obj_gen)
+            def _gen_results(objects):
+                for obj in objects:
+                    try:
+                        yield (
+                            obj['name'],
+                            obj['size'] if not obj['deleted'] else 'deleted',
+                            obj['hash'],
+                            obj['version'])
+                    except KeyError as exc:
+                        self.log.warn("Bad object entry, missing '%s': %s",
+                                      exc, obj)
             columns = ('Name', 'Size', 'Hash', 'Version')
+        results = _gen_results(obj_gen)
         return (columns, results)
 
 

proxy/m2_actions.c

@@ -866,7 +866,6 @@ static GError *
 _max (struct req_args_s *args, gint64 *pmax)
 {
 	const char *s = OPT("max");
-	*pmax = 0;
 	if (!s)
 		return NULL;
 
@@ -1630,6 +1629,9 @@ enum http_rc_e action_container_list (struct req_args_s *args) {
 	list_in.prefix = OPT("prefix");
 	list_in.marker_start = OPT("marker");
 	list_in.marker_end = OPT("end_marker");
+	/* This is the default when no limit is passed in the request.
+	 * The client can still pass a larger limit. */
+	list_in.maxkeys = 1000;
 	if (!list_in.marker_end)
 		list_in.marker_end = OPT("marker_end");  // backward compatibility
 	if (OPT("deleted"))

tests/functional/api/test_objectstorage.py

@@ -283,12 +283,12 @@ def _wait_account_meta2(self):
         while True:
             try:
                 for service in cluster.all_services("account"):
-                    if int(service['score']) < 70:
+                    if int(service['score']) < 7:
                         wait = True
                         continue
                 if not wait:
                     for service in cluster.all_services("meta2"):
-                        if int(service['score']) < 70:
+                        if int(service['score']) < 7:
                             wait = True
                             continue
                     if not wait:
@@ -626,10 +626,10 @@ def test_container_refresh(self):
 
         self.api.container_create(account, name)
         ref_time = time.time()
-        time.sleep(0.5)  # ensure container event have been processed
+        time.sleep(1)  # ensure container event have been processed
         # container_refresh on existing container
         self.api.container_refresh(account, name)
-        time.sleep(0.5)  # ensure container event have been processed
+        time.sleep(1)  # ensure container event have been processed
         res = self.api.container_list(account, prefix=name)
         name_container, nb_objects, nb_bytes, _, mtime = res[0]
         self.assertEqual(name, name_container)
@@ -639,10 +639,10 @@ def test_container_refresh(self):
 
         ref_time = mtime
         self.api.object_create(account, name, data="data", obj_name=name)
-        time.sleep(0.5)  # ensure container event have been processed
+        time.sleep(1)  # ensure container event have been processed
         # container_refresh on existing container with data
         self.api.container_refresh(account, name)
-        time.sleep(0.5)  # ensure container event have been processed
+        time.sleep(1)  # ensure container event have been processed
         res = self.api.container_list(account, prefix=name)
         name_container, nb_objects, nb_bytes, _, mtime = res[0]
         self.assertEqual(name, name_container)
@@ -651,9 +651,9 @@ def test_container_refresh(self):
         self.assertGreater(mtime, ref_time)
 
         self.api.object_delete(account, name, name)
-        time.sleep(0.5)  # ensure container event have been processed
+        time.sleep(1)  # ensure container event have been processed
         self.api.container_delete(account, name)
-        time.sleep(0.5)  # ensure container event have been processed
+        time.sleep(1)  # ensure container event have been processed
         # container_refresh on deleted container
         self.assertRaises(
             exc.NoSuchContainer, self.api.container_refresh, account, name)

tests/functional/cli/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2017 OpenIO SAS
+# Copyright (C) 2016-2018 OpenIO SAS
 
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -33,15 +33,16 @@ def __str__(self):
                                  self.stderr))
 
 
-def execute(cmd):
+def execute(cmd, stdin=None):
     """Executes command."""
     cmdlist = shlex.split(cmd)
     result = ''
     result_err = ''
     stdout = subprocess.PIPE
     stderr = subprocess.PIPE
-    proc = subprocess.Popen(cmdlist, stdout=stdout, stderr=stderr)
-    result, result_err = proc.communicate()
+    in_ = subprocess.PIPE if stdin else None
+    proc = subprocess.Popen(cmdlist, stdin=in_, stdout=stdout, stderr=stderr)
+    result, result_err = proc.communicate(stdin)
     result = result.decode('utf-8')
     if proc.returncode != 0:
         raise CommandFailed(proc.returncode, cmd, result, result_err)
@@ -54,6 +55,11 @@ def openio(cls, cmd):
         """Executes openio CLI command."""
         return execute('openio ' + cmd)
 
+    @classmethod
+    def openio_batch(cls, commands):
+        """Execute several commands in the same openio CLI process."""
+        return execute('openio', stdin='\n'.join(commands))
+
     @classmethod
     def get_opts(cls, fields, format='value'):
         return ' -f {0} {1}'.format(

tests/functional/cli/object/test_object.py

@@ -196,3 +196,47 @@ def test_drain(self):
                           self.openio,
                           ' '.join(['object drain', cname,
                                     'should not exist']))
+
+    def test_autocontainer_object_listing(self):
+        obj_count = 7
+        with tempfile.NamedTemporaryFile() as myfile:
+            myfile.write('something')
+            myfile.flush()
+            prefix = random_str(8)
+            # TODO(FVE): find a quicker way to upload several objects
+            commands = list()
+            for i in range(obj_count):
+                obj_name = '%s_%d' % (prefix, i)
+                commands.append(' '.join(['object create --auto ',
+                                          myfile.name, '--name ', obj_name]))
+            self.openio_batch(commands)
+
+        # Default listing
+        opts = self.get_opts([], 'json')
+        output = self.openio('object list --auto --prefix ' +
+                             prefix + ' ' + opts)
+        listing = self.json_loads(output)
+        self.assertEqual(obj_count, len(listing))
+        for obj in listing:
+            # 4 columns
+            self.assertEqual(4, len(obj))
+
+        # Listing with properties
+        opts = self.get_opts([], 'json')
+        output = self.openio('object list --auto --properties --prefix ' +
+                             prefix + ' ' + opts)
+        listing = self.json_loads(output)
+        self.assertEqual(obj_count, len(listing))
+        for obj in listing:
+            # 9 columns
+            self.assertEqual(9, len(obj))
+
+        # Unpaged listing
+        opts = self.get_opts([], 'json')
+        output = self.openio('object list --auto --no-paging --prefix ' +
+                             prefix + ' ' + opts)
+        listing = self.json_loads(output)
+        self.assertEqual(obj_count, len(listing))
+        for obj in listing:
+            # 4 columns
+            self.assertEqual(4, len(obj))