Update openssl to v3 on pypi

.github/actions/set_persistent_storage_env_vars/action.yml

@@ -2,7 +2,7 @@ name: 'Set Persistent storages env variables'
 description: 'Set the necessary variables for Persistent storage tests'
 inputs:
   bucket:            {default: 'arcticdb-ci-test-bucket-02', type: string, description: The name of the S3 bucket that we will test against}
-  endpoint:          {default: 's3.eu-west-1.amazonaws.com', type: string, description: The address of the S3 endpoint}
+  endpoint:          {default: 'https://s3.eu-west-1.amazonaws.com', type: string, description: The address of the S3 endpoint}
   region:            {default: 'eu-west-1', type: string, description: The S3 region of the bucket}
   aws_access_key:    {required: true, type: string, description: The value for the AWS Access key}      
   aws_secret_key:    {required: true, type: string, description: The value for the AWS Secret key}

build_tooling/build_many_linux_image.sh

@@ -44,7 +44,7 @@ RUN rpmkeys --import 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3F
             s/#?exclude.*/exclude=.edu/' /etc/yum/pluginconf.d/fastestmirror.conf
 ADD sccache /usr/local/bin/
 RUN yum update -y && \
-    yum install -y zip jq less devtoolset-11-gdb \
+    yum install -y zip jq less devtoolset-11-gdb perl-IPC-Cmd \
       openssl-devel cyrus-sasl-devel devtoolset-10-libatomic-devel libcurl-devel python3-devel flex && \
     rpm -Uvh --nodeps \$(repoquery --location mono-{core,web,devel,data,wcf,winfx}) && \
     yum clean all && touch /etc/arcticdb_deps_installed

cpp/arcticdb/storage/s3/s3_storage.hpp

@@ -212,6 +212,7 @@ auto get_s3_config(const ConfigType& conf) {
     auto endpoint_scheme = conf.https() ? Aws::Http::Scheme::HTTPS : Aws::Http::Scheme::HTTP;
     Aws::Client::ClientConfiguration client_configuration = get_proxy_config(endpoint_scheme);
     client_configuration.scheme = endpoint_scheme;
+    ARCTICDB_RUNTIME_DEBUG(log::storage(), "Endpoint Scheme: {}", conf.https() ? "https" : "http");
 
     if (!conf.region().empty()) {
         client_configuration.region = conf.region();

cpp/vcpkg.json

@@ -67,7 +67,7 @@
     "benchmark"
   ],
   "overrides": [
-    { "name": "openssl", "version-string": "1.1.1n#1" },
+    { "name": "openssl", "version-string": "3.2.1" },
     { "name": "aws-sdk-cpp", "version": "1.11.201" },
     { "name": "aws-c-s3", "version": "0.3.24" },
     { "name": "bitmagic", "version": "7.12.3" },

python/arcticdb/adapters/s3_library_adapter.py

@@ -235,7 +235,7 @@ def add_library_to_env(self, env_cfg: EnvironmentConfigsMap, name: str):
 
     def _configure_aws(self):
         if not self._query_params.region:
-            match = re.match(r"s3\.(?P<region>[a-z0-9-]+)\.amazonaws.*", self._endpoint)
+            match = re.match(r"s3[-\.](?P<region>[a-z0-9-]+)\.amazonaws.*", self._endpoint)
             if match:
                 match_groups = match.groupdict()
                 self._query_params.region = match_groups["region"]

python/arcticdb/storage_fixtures/api.py

@@ -29,6 +29,7 @@ class ArcticUriFields(Enum):
     PASSWORD = "PASSWORD"
     BUCKET = "BUCKET"
     CA_PATH = "CA_PATH"
+    PATH_PREFIX = "PATH_PREFIX"
 
     def __str__(self):
         return self.value

python/arcticdb/storage_fixtures/azure.py

@@ -28,6 +28,7 @@ class AzureContainer(StorageFixture):
         ArcticUriFields.PASSWORD: re.compile("[/;](AccountKey=)([^;]+)(;)"),
         ArcticUriFields.BUCKET: re.compile("[/;](Container=)([^;]+)(;)"),
         ArcticUriFields.CA_PATH: re.compile("[/;](CA_cert_path=)([^;]*)(;?)"),
+        ArcticUriFields.PATH_PREFIX: re.compile("[/;](Path_prefix=)([^;]*)(;?)"),
     }
     _sequence = 0
 

python/arcticdb/storage_fixtures/s3.py

@@ -40,6 +40,7 @@ class S3Bucket(StorageFixture):
         ArcticUriFields.BUCKET: re.compile("^s3://[^:]+(:)([^?]+)"),
         ArcticUriFields.USER: re.compile("[?&](access=)([^&]+)(&?)"),
         ArcticUriFields.PASSWORD: re.compile("[?&](secret=)([^&]+)(&?)"),
+        ArcticUriFields.PATH_PREFIX: re.compile("[?&](path_prefix=)([^&]+)(&?)"),
     }
 
     key: Key
@@ -88,7 +89,7 @@ def create_test_cfg(self, lib_name: str) -> EnvironmentConfigsMap:
             bucket_name=self.bucket,
             endpoint=self.factory.endpoint,
             with_prefix=with_prefix,
-            is_https=self.factory.endpoint.startswith("s3s:"),
+            is_https=self.factory.endpoint.startswith("https://"),
             region=self.factory.region,
             use_mock_storage_for_testing=self.factory.use_mock_storage_for_testing,
             ssl=self.factory.ssl,
@@ -174,6 +175,7 @@ def real_s3_from_environment_variables(*, shared_path: bool):
     secret_key = os.getenv("ARCTICDB_REAL_S3_SECRET_KEY")
     out.default_key = Key(access_key, secret_key, "unknown user")
     out.clean_bucket_on_fixture_exit = os.getenv("ARCTICDB_REAL_S3_CLEAR").lower() in ["true", "1"]
+    out.ssl = out.endpoint.startswith("https://")
     if shared_path:
         out.default_prefix = os.getenv("ARCTICDB_PERSISTENT_STORAGE_SHARED_PATH_PREFIX")
     else:

python/tests/integration/arcticdb/test_arctic_library_management.py

@@ -361,14 +361,14 @@ def test_separation_between_libraries(arctic_client):
     assert ac["pytest_test_lib_2"].list_symbols() == ["test_2"]
 
 
-def add_path_prefix(uri, prefix):
-    if "path_prefix" in uri:
-        return uri + prefix
+def add_path_prefix(storage_fixture, prefix):
+    if "path_prefix".casefold() in storage_fixture.arctic_uri.casefold():
+        return storage_fixture.replace_uri_field(storage_fixture.arctic_uri, ArcticUriFields.PATH_PREFIX, prefix, start=3, end=2)
 
-    if "azure" in uri:  # azure connection string has a different format
-        return f"{uri};Path_prefix={prefix}"
+    if "azure" in storage_fixture.arctic_uri:  # azure connection string has a different format
+        return f"{storage_fixture.arctic_uri};Path_prefix={prefix}"
     else:
-        return f"{uri}&path_prefix={prefix}"
+        return f"{storage_fixture.arctic_uri}&path_prefix={prefix}"
 
 
 @pytest.mark.parametrize(
@@ -385,11 +385,10 @@ def test_separation_between_libraries_with_prefixes(fixture, request):
     creating a new bucket is time-consuming, for example due to organizational issues.
     """
     storage_fixture: StorageFixture = request.getfixturevalue(fixture)
-
-    mercury_uri = add_path_prefix(storage_fixture.arctic_uri, "/planet/mercury")
+    mercury_uri = add_path_prefix(storage_fixture, "/planet/mercury")
     ac_mercury = Arctic(mercury_uri)
 
-    mars_uri = add_path_prefix(storage_fixture.arctic_uri, "/planet/mars")
+    mars_uri = add_path_prefix(storage_fixture, "/planet/mars")
     ac_mars = Arctic(mars_uri)
 
     assert ac_mars.list_libraries() == []
@@ -416,7 +415,7 @@ def test_separation_between_libraries_with_prefixes(fixture, request):
 @pytest.mark.parametrize("fixture", ["s3_storage", pytest.param("azurite_storage", marks=AZURE_TESTS_MARK)])
 def test_library_management_path_prefix(fixture, request):
     storage_fixture: StorageFixture = request.getfixturevalue(fixture)
-    uri = add_path_prefix(storage_fixture.arctic_uri, "hello/world")
+    uri = add_path_prefix(storage_fixture, "hello/world")
     ac = Arctic(uri)
     assert ac.list_libraries() == []