[#1937] Update for new libs3

* Change code for libs3 signatures
* Removed references to S3_SIGNATURE_VERSION
* added ceph and minio test suite
* reorganized and cleanup up tests

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR) #CPACK_DEBIAN_<COMPONENT>_PACKAGE_NAME
 
 find_package(IRODS 4.2.8 EXACT REQUIRED)
-set(IRODS_PLUGIN_REVISION "0")
+set(IRODS_PLUGIN_REVISION "1")
 
 if (NOT IRODS_EXTERNALS_PACKAGE_ROOT)
   set(IRODS_EXTERNALS_PACKAGE_ROOT "/opt/irods-externals" CACHE STRING "Choose the location of iRODS external packages." FORCE)
@@ -23,7 +23,7 @@ macro(IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH_ADD_TO_IRODS_PACKAGE_DEPENDENCIE
   IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH(${DEPENDENCY_NAME} ${DEPENDENCY_SUBDIRECTORY})
   list(APPEND IRODS_PACKAGE_DEPENDENCIES_LIST irods-externals-${DEPENDENCY_SUBDIRECTORY})
 endmacro()
-IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH_ADD_TO_IRODS_PACKAGE_DEPENDENCIES_LIST(S3 libs3a30e55e8-1)
+IRODS_MACRO_CHECK_DEPENDENCY_SET_FULLPATH_ADD_TO_IRODS_PACKAGE_DEPENDENCIES_LIST(S3 libs34e684077-0)
 string(REPLACE ";" ", " IRODS_PACKAGE_DEPENDENCIES_STRING "${IRODS_PACKAGE_DEPENDENCIES_LIST}")
 
 set(CMAKE_C_COMPILER ${IRODS_EXTERNALS_FULLPATH_CLANG}/bin/clang)
@@ -52,6 +52,7 @@ add_compile_options(-nostdinc++ -Wall -Wextra -Werror)
 add_compile_options(-Wno-unused-function) # due to OPENSSL_NO_DEPRECATED_1_1_0
 add_compile_options(-Wno-unused-parameter) # Until fix of irods_re_plugin.hpp
 add_compile_options(-Wno-unneeded-internal-declaration) # Until fix of https://github.com/irods/irods/issues/3396
+add_compile_options(-Wno-unused-function) # Due to openssl_unused_function
 link_libraries(c++abi curl xml2 crypto pthread)
 include_directories(${IRODS_EXTERNALS_FULLPATH_CLANG}/include/c++/v1
                     ${IRODS_EXTERNALS_FULLPATH_JSON}/include)
@@ -114,7 +115,10 @@ install(
   FILES
   ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3.py
   ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_for_gcp.py
+  ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_ceph.py
+  ${CMAKE_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_minio.py
   ${CMAKE_SOURCE_DIR}/packaging/resource_suite_s3_nocache.py
+  ${CMAKE_SOURCE_DIR}/packaging/resource_suite_s3_cache.py
   DESTINATION ${IRODS_HOME_DIRECTORY}/scripts/irods/test
   PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
   )

packaging/resource_suite_s3_nocache.py

@@ -33,10 +33,28 @@
 from . import session
 
 
-class ResourceBase(session.make_sessions_mixin([('otherrods', 'rods')], [('alice', 'apass'), ('bobby', 'bpass')])):
+class Test_S3_NoCache_Base(session.make_sessions_mixin([('otherrods', 'rods')], [('alice', 'apass'), ('bobby', 'bpass')])):
+
+    def __init__(self, *args, **kwargs):
+
+        # if self.proto is defined use it else default to HTTPS
+        try:
+            self.proto = self.proto
+        except AttributeError:
+            self.proto = 'HTTPS'
+
+        # if self.archive_naming_policy is defined use it
+        # else default to 'consistent'
+        try:
+            self.archive_naming_policy = self.archive_naming_policy
+        except AttributeError:
+            self.archive_naming_policy = 'consistent'
+
+        super(Test_S3_NoCache_Base, self).__init__(*args, **kwargs)
 
     def setUp(self):
-        super(ResourceBase, self).setUp()
+
+        super(Test_S3_NoCache_Base, self).setUp()
         self.admin = self.admin_sessions[0]
         self.user0 = self.user_sessions[0]
         self.user1 = self.user_sessions[1]
@@ -51,7 +69,11 @@ def setUp(self):
         self.read_aws_keys()
 
         # set up s3 bucket
-        s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key)
+        if self.proto == 'HTTPS':
+            s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key)
+        else:
+            s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key, secure=False)
+
 
         distro_str = ''.join(platform.linux_distribution()[:2]).replace(' ','')
         self.s3bucketname = 'irods-ci-' + distro_str + datetime.datetime.utcnow().strftime('-%Y-%m-%d.%H-%M-%S-%f-')
@@ -64,7 +86,7 @@ def setUp(self):
         self.anotherresc = "AnotherResc"
         self.anothervault = "/tmp/" + self.anotherresc
 
-        self.s3_context = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=HTTPS;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_attached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d;S3_SIGNATURE_VERSION=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.s3EnableMPU, self.s3signature_version)
+        self.s3_context = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=%s;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_attached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.proto, self.s3EnableMPU)
 
         self.admin.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n')
 
@@ -91,7 +113,7 @@ def tearDown(self):
         self.admin.run_icommand(['irm', self.testfile, '../public/' + self.testfile])
         self.admin.run_icommand('irm -rf ../../bundle')
 
-        super(ResourceBase, self).tearDown()
+        super(Test_S3_NoCache_Base, self).tearDown()
         with session.make_session_for_existing_admin() as admin_session:
             admin_session.run_icommand('irmtrash -M')
             admin_session.run_icommand(['iadmin', 'rmresc', self.testresc])
@@ -101,7 +123,10 @@ def tearDown(self):
             print("run_resource_teardown - END")
 
         # delete s3 bucket
-        s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key)
+        if self.proto == 'HTTPS':
+            s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key)
+        else:
+            s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key, secure=False)
         objects = s3_client.list_objects_v2(self.s3bucketname, recursive=True)
 
         try:
@@ -118,6 +143,12 @@ def read_aws_keys(self):
             self.aws_access_key_id = f.readline().rstrip()
             self.aws_secret_access_key = f.readline().rstrip()
 
+    # read the endpoint address from the file endpointfile
+    def read_endpoint(self, endpointfile):
+        # read endpoint file
+        with open(endpointfile) as f:
+            return f.readline().rstrip()
+
     def set_up_aws_config_dir(self):
         # read access keys from keypair file
         with open(self.keypairfile) as f:
@@ -143,9 +174,6 @@ def set_up_aws_config_dir(self):
             cred_file.write('aws_access_key_id = ' + aws_access_key_id + '\n')
             cred_file.write('aws_secret_access_key = ' + aws_secret_access_key + '\n')
 
-class ResourceSuite_S3_NoCache(ResourceBase):
-
-
     ###################
     # iget
     ###################
@@ -1121,8 +1149,7 @@ def test_itrim_returns_on_negative_status__ticket_3531(self):
             os.unlink(filepath)
 
 
-# specific test cases unique to cacheless S3
-class Test_S3_NoCache_Base(ResourceSuite_S3_NoCache):
+    # tests add for cacheless S3
 
     def test_iput_large_file_over_smaller(self):
 
@@ -1455,7 +1482,8 @@ def test_detached_mode(self):
         resource_host = "irods.org"
 
         # change demoResc to use detached mode
-        s3_context_detached = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=HTTPS;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_detached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d;S3_SIGNATURE_VERSION=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.s3EnableMPU, self.s3signature_version)
+
+        s3_context_detached = "S3_DEFAULT_HOSTNAME=%s;S3_AUTH_FILE=%s;S3_REGIONNAME=%s;S3_RETRY_COUNT=2;S3_WAIT_TIME_SEC=3;S3_PROTO=%s;ARCHIVE_NAMING_POLICY=consistent;HOST_MODE=cacheless_detached;S3_ENABLE_MD5=1;S3_ENABLE_MPU=%d" % (self.s3endPoint, self.keypairfile, self.s3region, self.proto, self.s3EnableMPU)
 
         self.admin.assert_icommand("iadmin modresc demoResc context %s" % s3_context_detached , 'EMPTY')
         self.admin.assert_icommand("iadmin modresc demoResc host %s" % resource_host, 'EMPTY')
@@ -1517,7 +1545,10 @@ def test_attached_mode_invalid_resource_host(self):
     def recursive_register_from_s3_bucket(self):
 
         # create some files on s3
-        s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key)
+        if self.proto == 'HTTPS':
+            s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key)
+        else:
+            s3_client = Minio(self.s3endPoint, access_key=self.aws_access_key_id, secret_key=self.aws_secret_access_key, secure=False)
         file_contents = b'random test data'
         f = io.BytesIO(file_contents)
         size = len(file_contents)