CI: enable tests with predictible chunk IDs

.travis.yml

@@ -90,19 +90,19 @@ jobs:
       env: TEST_SUITE=cli
       python: 3.6
     - script: ./tools/oio-travis-suites.sh
-      env: TEST_SUITE=rebuilder,with-service-id,zlib
+      env: TEST_SUITE=rebuilder,with-service-id,zlib,predictible-chunk-ids
       python: 3.6
     - script: ./tools/oio-travis-suites.sh
       env: TEST_SUITE=repli,with_tls
       python: 3.6
     - script: ./tools/oio-travis-suites.sh
-      env: TEST_SUITE=ec,with-service-id
+      env: TEST_SUITE=ec,with-service-id,predictible-chunk-ids
       python: 3.6
     - script: ./tools/oio-travis-suites.sh
-      env: TEST_SUITE=3copies,with-service-id
+      env: TEST_SUITE=3copies,with-service-id,predictible-chunk-ids
       python: 3.6
     - script: ./tools/oio-travis-suites.sh
-      env: TEST_SUITE=mover,with-service-id
+      env: TEST_SUITE=mover,with-service-id,predictible-chunk-ids
       python: 3.6
 
     - stage: Functional tests (Python 2)
@@ -111,7 +111,7 @@ jobs:
     - script: ./tools/oio-travis-suites.sh
       env: TEST_SUITE=ec,with-service-id,with_tls
     - script: ./tools/oio-travis-suites.sh
-      env: TEST_SUITE=3copies,with-service-id
+      env: TEST_SUITE=3copies,with-service-id,predictible-chunk-ids
     - script: ./tools/oio-travis-suites.sh
       env: TEST_SUITE=single,small-cache,fsync,webhook,zlib
     - script: ./tools/oio-travis-suites.sh

etc/bootstrap-option-predictible-chunk-ids.yml

@@ -0,0 +1,3 @@
+config:
+  core.lb.generate_random_chunk_ids: false
+

tests/functional/content/test_content.py

@@ -249,19 +249,21 @@ def _test_move_chunk(self, policy):
         data = random_data(self.chunk_size)
         content = self._new_content(policy, data)
 
-        chunk_id = content.chunks.filter(metapos=0)[0].id
-        chunk_url = content.chunks.filter(metapos=0)[0].url
+        mc = content.chunks.filter(metapos=0)
+        chunk_id = mc[0].id
+        chunk_url = mc[0].url
+        chunk_host = mc[0].host
         chunk_meta, chunk_stream = self.blob_client.chunk_get(chunk_url)
         chunk_hash = md5_stream(chunk_stream)
-        new_chunk = content.move_chunk(chunk_id)
+        new_chunk = content.move_chunk(chunk_id, service_id=chunk_host)
 
         content_updated = self.content_factory.get(self.container_id,
                                                    content.content_id)
 
         hosts = []
         for c in content_updated.chunks.filter(metapos=0):
             self.assertThat(hosts, Not(Contains(c.host)))
-            self.assertNotEqual(c.id, chunk_id)
+            self.assertNotEqual(c.url, chunk_url)
             hosts.append(c.host)
 
         new_chunk_meta, new_chunk_stream = self.blob_client.chunk_get(

tests/functional/content/test_ec.py

@@ -59,7 +59,7 @@ def setUp(self):
                                                reference=self.container_name)
         self.container_id = cid_from_name(self.account,
                                           self.container_name).upper()
-        self.content = random_str(64)
+        self.content = "%s-%s" % (self.__class__.__name__, random_str(4))
         self.stgpol = "EC"
         self.size = 1024*1024 + 320
         self.k = 6

tests/functional/content/test_plain.py

@@ -16,6 +16,7 @@
 import math
 import time
 from io import BytesIO
+from six.moves.urllib_parse import urlparse
 from testtools.matchers import NotEquals
 from testtools.testcase import ExpectedException
 
@@ -55,7 +56,7 @@ def setUp(self):
                                                reference=self.container_name)
         self.container_id = cid_from_name(self.account,
                                           self.container_name).upper()
-        self.content = random_str(64)
+        self.content = "%s-%s" % (self.__class__.__name__, random_str(4))
         self.stgpol = "SINGLE"
         self.stgpol_twocopies = "TWOCOPIES"
         self.stgpol_threecopies = "THREECOPIES"
@@ -166,7 +167,8 @@ def _rebuild_and_check(self, content, broken_chunks_info, full_rebuild_pos,
                            allow_frozen_container=False):
         rebuild_pos, rebuild_idx = full_rebuild_pos
         rebuild_chunk_info = broken_chunks_info[rebuild_pos][rebuild_idx]
-        content.rebuild_chunk(rebuild_chunk_info["id"],
+        service_id = urlparse(rebuild_chunk_info['url']).netloc
+        content.rebuild_chunk(rebuild_chunk_info["id"], service_id=service_id,
                               allow_frozen_container=allow_frozen_container)
 
         # get the new structure of the content
@@ -179,7 +181,7 @@ def _rebuild_and_check(self, content, broken_chunks_info, full_rebuild_pos,
                 # not the rebuilt chunk
                 # if this chunk is broken, it must not have been rebuilt
                 for b_c_i in broken_chunks_info[rebuild_pos].values():
-                    if c.id == b_c_i["id"]:
+                    if c.url == b_c_i['url']:
                         with ExpectedException(NotFound):
                             _, _ = self.blob_client.chunk_get(c.url)
                 continue
@@ -235,8 +237,10 @@ def test_rebuild_chunk_in_frozen_container(self):
             full_rebuild_pos = (0, 0)
             rebuild_pos, rebuild_idx = full_rebuild_pos
             rebuild_chunk_info = broken_chunks_info[rebuild_pos][rebuild_idx]
+            service_id = urlparse(rebuild_chunk_info['url']).netloc
             self.assertRaises(ServiceBusy,
-                              content.rebuild_chunk, rebuild_chunk_info["id"])
+                              content.rebuild_chunk,
+                              rebuild_chunk_info["id"], service_id=service_id)
         finally:
             system['sys.status'] = str(OIO_DB_ENABLED)
             self.container_client.container_set_properties(

tools/oio-test-suites.sh

@@ -3,6 +3,7 @@
 
 # oio-test-suites.sh
 # Copyright (C) 2016-2019 OpenIO SAS, as part of OpenIO SDS
+# Copyright (C) 2021 OVH SAS
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -212,6 +213,9 @@ func_tests () {
 	if is_running_test_suite "with_tls"; then
 		args="${args} -f ${SRCDIR}/etc/bootstrap-option-tls.yml"
 	fi
+	if is_running_test_suite "predictible-chunk-ids"; then
+		args="${args} -f ${SRCDIR}/etc/bootstrap-option-predictible-chunk-ids.yml"
+	fi
 	$OIO_RESET ${args} -N $OIO_NS $@
 
 	test_proxy_forward
@@ -307,39 +311,6 @@ test_cli () {
 	test_oio_logger
 }
 
-func_tests_rebuilder_mover () {
-	randomize_env
-	args=
-	if is_running_test_suite "with-service-id"; then
-		args="${args} -U"
-	fi
-	if is_running_test_suite "with-random-service-id"; then
-		args="${args} -R"
-	fi
-	$OIO_RESET ${args} -N $OIO_NS $@
-
-	test_proxy_forward
-
-	wait_proxy_cache
-
-	for i in $(seq 1 100); do
-		dd if=/dev/urandom of=/tmp/openio_object_$i bs=1K \
-				count=$(shuf -i 1-2000 -n 1) 2> /dev/null
-		echo "object create container-${RANDOM} /tmp/openio_object_$i" \
-				"--name object-${RANDOM} -f value"
-	done | ${PYTHON} $(command -v openio)
-
-	if [ -n "${REBUILDER}" ]; then
-		${SRCDIR}/tools/oio-test-rebuilder.sh -n "${OIO_NS}"
-	fi
-	if [ -n "${MOVER}" ]; then
-		${SRCDIR}/tools/oio-test-mover.sh -n "${OIO_NS}"
-	fi
-
-	gridinit_cmd -S $HOME/.oio/sds/run/gridinit.sock stop
-	sleep 0.5
-}
-
 #-------------------------------------------------------------------------------
 
 set -e
@@ -436,6 +407,9 @@ func_tests_rebuilder_mover () {
 	if is_running_test_suite "with-random-service-id"; then
 		args="${args} -R"
 	fi
+	if is_running_test_suite "predictible-chunk-ids"; then
+		args="${args} -f ${SRCDIR}/etc/bootstrap-option-predictible-chunk-ids.yml"
+	fi
 	$OIO_RESET ${args} -N $OIO_NS $@
 
 	test_proxy_forward