Merge branch '4.2.x'

all-requirements.txt

@@ -9,6 +9,6 @@ PyYAML>=3.10
 redis>=2.10.3
 requests!=2.13.0
 simplejson>=2.0.9
-urllib3>=1.13.1,<1.24
+urllib3>=1.13.1
 werkzeug>=0.9.1
 zkpython

core/oiostr.h

@@ -201,6 +201,9 @@ void oio_str_gstring_append_json_pair_boolean (GString *base,
 	} \
 } while (0)
 
+#define OIO_JSON_append_null(g,k) do { \
+	g_string_append_static(g, "\"" k "\":null"); \
+} while (0)
 
 static inline int oio_str_is_set (const char *s) { return NULL!=s && 0!=*s; }
 

meta2v2/meta2_utils.c

@@ -2617,7 +2617,7 @@ _purge_exceeding_aliases(struct sqlx_sqlite3_s *sq3, gint64 max_versions,
 			"HAVING COUNT(*) > ?";
 	}
 	const gchar *sql_delete = " rowid IN "
-		"(SELECT rowid FROM aliases WHERE alias = ? "
+		"(SELECT rowid FROM aliases WHERE NOT deleted AND alias = ? "
 		" ORDER BY version ASC LIMIT ? ) ";
 
 	int rc = SQLITE_OK;

oio/account/server.py

@@ -278,7 +278,10 @@ def on_account_show(self, req):
     # ACCT{{
     # GET /v1.0/account/containers?id=<account_name>
     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    # Get information about containers on account named account_name
+    #
+    # Get information about the containers belonging to the specified account.
+    #
+    # Sample request:
     #
     # .. code-block:: http
     #
@@ -287,6 +290,8 @@ def on_account_show(self, req):
     #    User-Agent: curl/7.47.0
     #    Accept: */*
     #
+    # Sample response:
+    #
     # .. code-block:: http
     #
     #    HTTP/1.1 200 OK
@@ -328,6 +333,7 @@ def on_account_containers(self, req):
             prefix=prefix, delimiter=delimiter)
 
         info['listing'] = user_list
+        # TODO(FVE): add "truncated" entry telling if the listing is truncated
         result = json.dumps(info)
         return Response(result, mimetype='text/json')
 

oio/cli/object/object.py

@@ -19,6 +19,7 @@
 from eventlet import GreenPool
 from oio.common.http_urllib3 import get_pool_manager
 from oio.common.utils import depaginate
+from oio.common import exceptions
 
 
 class ContainerCommandMixin(object):
@@ -438,6 +439,7 @@ def take_action(self, parsed_args):
             self.app.client_manager.account,
             container,
             obj,
+            version=parsed_args.object_version,
             key_file=key_file,
             properties=False,
             cid=cid
@@ -561,6 +563,7 @@ def _autocontainer_loop(self, account, marker=None, limit=None,
                         account=account, **kwargs),
                 depaginate(self.app.client_manager.storage.container_list,
                            item_key=lambda x: x[0],
+                           marker_key=lambda x: x[-1][0],
                            account=account,
                            marker=container_marker)):
             for element in object_list:
@@ -576,13 +579,17 @@ def _list_autocontainer_objects(self, container, account, **kwargs):
                            container)
             return object_list
         self.log.debug("Listing autocontainer %s", container)
-        for i in depaginate(
-                self.app.client_manager.storage.object_list,
-                listing_key=lambda x: x['objects'],
-                marker_key=lambda x: x.get('next_marker'),
-                truncated_key=lambda x: x['truncated'],
-                account=account, container=container, **kwargs):
-            object_list.append(i)
+        try:
+            for i in depaginate(
+                    self.app.client_manager.storage.object_list,
+                    listing_key=lambda x: x['objects'],
+                    marker_key=lambda x: x.get('next_marker'),
+                    truncated_key=lambda x: x['truncated'],
+                    account=account, container=container, **kwargs):
+                object_list.append(i)
+        except exceptions.OioException as err:
+            self.log.warn('Listing may be incomplete: container %s: %s',
+                          container, err)
         return object_list
 
     def take_action(self, parsed_args):
@@ -673,7 +680,7 @@ def _gen_results(objects):
                             obj['hash'],
                             obj['version'])
                     except KeyError as exc:
-                        self.log.warn("Bad object entry, missing '%s': %s",
+                        self.log.warn("Bad object entry, missing %s: %s",
                                       exc, obj)
             columns = ('Name', 'Size', 'Hash', 'Version')
         results = _gen_results(obj_gen)
@@ -896,7 +903,7 @@ def take_action(self, parsed_args):
         self.app.client_manager.storage.object_link(
             account, container, parsed_args.object,
             parsed_args.link_account, parsed_args.link_container,
-            parsed_args.link_object, maxvers=parsed_args.object_version,
+            parsed_args.link_object, target_version=parsed_args.object_version,
             target_content_id=parsed_args.content_id,
             link_content_id=parsed_args.link_content_id,
             properties_directive=directive, cid=cid, **kwargs

proxy/m2_actions.c

@@ -301,28 +301,29 @@ _dump_json_aliases_and_headers(GString *gstr, GSList *aliases,
 		OIO_JSON_append_gba(gstr, "content", ALIASES_get_content(a));
 
 		if (h) {
-			g_string_append_c (gstr, ',');
-			GString *pol = CONTENTS_HEADERS_get_policy(h);
-			GByteArray *hh = CONTENTS_HEADERS_get_hash(h);
-
-			if (pol)
-				g_string_append_printf(gstr, "\"policy\":\"%s\",\"hash\":",
-						pol->str);
-			else
-				g_string_append_printf(gstr, "\"policy\":null,\"hash\":");
-			if (hh) {
-				g_string_append_c (gstr, '"');
-				metautils_gba_to_hexgstr(gstr, hh);
-				g_string_append_c (gstr, '"');
-			} else {
-				g_string_append_static(gstr, "null");
-			}
-
-			g_string_append_printf(gstr, ",\"size\":%"G_GINT64_FORMAT,
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_gstr(gstr, "policy",
+					CONTENTS_HEADERS_get_policy(h));
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_gba(gstr, "hash",
+					CONTENTS_HEADERS_get_hash(h));
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_int(gstr, "size",
 					CONTENTS_HEADERS_get_size(h));
-			g_string_append_printf(gstr, ",\"mime-type\":\"%s\"",
-					CONTENTS_HEADERS_get_mime_type(h)->str);
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_gstr(gstr, "mime-type",
+					CONTENTS_HEADERS_get_mime_type(h));
+		} else {
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_null(gstr, "policy");
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_null(gstr, "hash");
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_null(gstr, "size");
+			g_string_append_c(gstr, ',');
+			OIO_JSON_append_null(gstr, "mime-type");
 		}
+
 		if (prop_list) {
 			g_string_append_static(gstr, ",\"properties\":{");
 			gboolean inner_first = TRUE;

tests/functional/content/test_content_versioning.py

@@ -119,29 +119,70 @@ def test_content_purge(self):
         self.assertEqual(5, len(objects))
 
     def test_delete_exceeding_version(self):
-        def check_num_objects_and_get_oldest_version(expected):
+        def check_num_objects_and_get_oldest_version(
+                expected_objects, expected_deleted_aliases, oldest_version):
             listing = self.api.object_list(self.account, self.container,
                                            versions=True)
             objects = listing['objects']
-            self.assertEqual(expected, len(objects))
-            return min(objects, key=lambda x: x['version'])
+            nb_objects = 0
+            nb_deleted = 0
+            new_oldest_version = 0
+            for obj in objects:
+                if obj['deleted']:
+                    nb_deleted += 1
+                else:
+                    nb_objects += 1
+                    if new_oldest_version == 0 \
+                            or new_oldest_version > obj['version']:
+                        new_oldest_version = obj['version']
+            self.assertEqual(expected_objects, nb_objects)
+            self.assertEqual(expected_deleted_aliases, nb_deleted)
+            if oldest_version is not None:
+                self.assertLess(oldest_version,
+                                new_oldest_version)
+            return new_oldest_version
 
         system = {'sys.m2.policy.version.delete_exceeding': '1'}
         self.api.container_set_properties(self.account, self.container,
                                           system=system)
         self.api.object_create(self.account, self.container,
                                obj_name="versioned", data="content0")
+        oldest_version = check_num_objects_and_get_oldest_version(
+            1, 0, None)
         self.api.object_create(self.account, self.container,
                                obj_name="versioned", data="content1")
+        self.assertEqual(oldest_version,
+                         check_num_objects_and_get_oldest_version(2, 0, None))
         self.api.object_create(self.account, self.container,
                                obj_name="versioned", data="content2")
-        oldest_version = check_num_objects_and_get_oldest_version(3)
+        self.assertEqual(oldest_version,
+                         check_num_objects_and_get_oldest_version(3, 0, None))
 
         self.api.object_create(self.account, self.container,
                                obj_name="versioned", data="content3")
-        new_oldest_version = check_num_objects_and_get_oldest_version(3)
-        self.assertLess(oldest_version['version'],
-                        new_oldest_version['version'])
+        oldest_version = check_num_objects_and_get_oldest_version(
+            3, 0, oldest_version)
+
+        self.api.object_delete(self.account, self.container, "versioned")
+        self.assertEqual(oldest_version,
+                         check_num_objects_and_get_oldest_version(3, 1, None))
+        self.api.object_create(self.account, self.container,
+                               obj_name="versioned", data="content4")
+        oldest_version = check_num_objects_and_get_oldest_version(
+            3, 1, oldest_version)
+        self.api.object_create(self.account, self.container,
+                               obj_name="versioned", data="content5")
+        oldest_version = check_num_objects_and_get_oldest_version(
+            3, 1, oldest_version)
+        self.api.object_create(self.account, self.container,
+                               obj_name="versioned", data="content6")
+        # FIXME(adu) The deleted alias should be deleted at the same time
+        oldest_version = check_num_objects_and_get_oldest_version(
+            3, 1, oldest_version)
+        self.api.object_create(self.account, self.container,
+                               obj_name="versioned", data="content7")
+        oldest_version = check_num_objects_and_get_oldest_version(
+            3, 1, oldest_version)
 
     def test_change_flag_delete_exceeding_versions(self):
         def check_num_objects(expected):