Merge branch 'master' into maint/update-to-folly-2023.09.25.00

.github/actions/set_persistent_storage_env_vars/action.yml

@@ -2,7 +2,7 @@ name: 'Set Persistent storages env variables'
 description: 'Set the necessary variables for Persistent storage tests'
 inputs:
   bucket:            {default: 'arcticdb-ci-test-bucket-02', type: string, description: The name of the S3 bucket that we will test against}
-  endpoint:          {default: 's3.eu-west-1.amazonaws.com', type: string, description: The address of the S3 endpoint}
+  endpoint:          {default: 'https://s3.eu-west-1.amazonaws.com', type: string, description: The address of the S3 endpoint}
   region:            {default: 'eu-west-1', type: string, description: The S3 region of the bucket}
   aws_access_key:    {required: true, type: string, description: The value for the AWS Access key}      
   aws_secret_key:    {required: true, type: string, description: The value for the AWS Secret key}

build_tooling/build_many_linux_image.sh

@@ -44,7 +44,7 @@ RUN rpmkeys --import 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3F
             s/#?exclude.*/exclude=.edu/' /etc/yum/pluginconf.d/fastestmirror.conf
 ADD sccache /usr/local/bin/
 RUN yum update -y && \
-    yum install -y zip jq less devtoolset-11-gdb \
+    yum install -y zip jq less devtoolset-11-gdb perl-IPC-Cmd \
       openssl-devel cyrus-sasl-devel devtoolset-10-libatomic-devel libcurl-devel python3-devel flex && \
     rpm -Uvh --nodeps \$(repoquery --location mono-{core,web,devel,data,wcf,winfx}) && \
     yum clean all && touch /etc/arcticdb_deps_installed

cpp/arcticdb/async/async_store.hpp

@@ -371,6 +371,10 @@ std::vector<folly::Future<bool>> batch_key_exists(
         library_->set_failure_sim(cfg);
     }
 
+    std::string name() const override {
+        return library_->name();
+    }
+
 private:
     std::shared_ptr<storage::Library> library_;
     std::shared_ptr<arcticdb::proto::encoding::VariantCodec> codec_;

cpp/arcticdb/entity/key.cpp

@@ -64,6 +64,7 @@ KeyData get_key_data(KeyType key_type) {
     STRING_KEY(KeyType::APPEND_DATA, app, 'b')
     // Unused
     STRING_KEY(KeyType::PARTITION, pref, 'p')
+    STRING_KEY(KeyType::REPLICATION_FAIL_INFO, rfail, 'F')
     STRING_REF(KeyType::STORAGE_INFO, sref, 'h')
     NUMERIC_KEY(KeyType::STREAM_GROUP, sg, 'g')
     NUMERIC_KEY(KeyType::GENERATION, gen, 'G')

cpp/arcticdb/entity/key.hpp

@@ -178,6 +178,10 @@ enum class KeyType : int {
      * Contains column stats about the index key with the same stream ID and version number
      */
     COLUMN_STATS = 25,
+    /*
+     * Used for storing the ids of storages that failed to sync
+     */
+    REPLICATION_FAIL_INFO = 26,
     UNDEFINED
 };
 

cpp/arcticdb/entity/type_utils.hpp

@@ -7,7 +7,9 @@
 
 #pragma once
 #include<optional>
+#include <fmt/format.h>
 #include<arcticdb/entity/descriptors.hpp>
+#include <arcticdb/entity/types.hpp>
 
 namespace arcticdb {
 
@@ -38,4 +40,9 @@ namespace entity {
     const proto::descriptors::TypeDescriptor& left,
     const proto::descriptors::TypeDescriptor& right
 );
+
+inline std::string get_user_friendly_type_string(const entity::TypeDescriptor& type) {
+    return is_sequence_type(type.data_type()) ? fmt::format("TD<type=STRING, dim={}>", type.dimension_) : fmt::format("{}", type);
+}
+
 }

cpp/arcticdb/pipeline/value.hpp

@@ -132,6 +132,16 @@ struct Value {
         *str_data() = data;
     }
 
+    template<typename RawType>
+    std::string to_string() const {
+        if (has_sequence_type()) {
+            return "\"" + std::string(*str_data(), len()) + "\"";
+        }
+        else {
+            return fmt::format("{}", get<RawType>());
+        }
+    }
+
     ~Value() {
         if(has_sequence_type())
             delete[] *str_data();
@@ -208,4 +218,4 @@ inline std::optional<std::string> ascii_to_padded_utf32(std::string_view str, si
     return rv;
 }
 
-} //namespace arcticdb
+} //namespace arcticdb

cpp/arcticdb/processing/expression_node.hpp

@@ -14,6 +14,7 @@
 #include <arcticdb/processing/operation_types.hpp>
 #include <arcticdb/pipeline/value.hpp>
 #include <arcticdb/pipeline/value_set.hpp>
+#include <utility>
 
 
 namespace arcticdb {
@@ -42,19 +43,23 @@ class StringPool;
 struct ColumnWithStrings {
     std::shared_ptr<Column> column_;
     const std::shared_ptr<StringPool> string_pool_;
+    std::string column_name_;
 
-    explicit ColumnWithStrings(std::unique_ptr<Column>&& col) :
-        column_(std::move(col)) {
+    ColumnWithStrings(std::unique_ptr<Column>&& col, std::string_view col_name) :
+        column_(std::move(col)),
+        column_name_(col_name) {
     }
 
-    ColumnWithStrings(Column&& col, std::shared_ptr<StringPool> string_pool) :
+    ColumnWithStrings(Column&& col, std::shared_ptr<StringPool> string_pool, std::string_view col_name) :
         column_(std::make_shared<Column>(std::move(col))),
-        string_pool_(std::move(string_pool)) {
+        string_pool_(std::move(string_pool)),
+        column_name_(col_name) {
     }
 
-    ColumnWithStrings(std::shared_ptr<Column> column, const std::shared_ptr<StringPool>& string_pool) :
+    ColumnWithStrings(std::shared_ptr<Column> column, const std::shared_ptr<StringPool>& string_pool, std::string_view col_name) :
         column_(std::move(column)),
-        string_pool_(string_pool) {
+        string_pool_(string_pool),
+        column_name_(col_name) {
     }
 
     [[nodiscard]] std::optional<std::string_view> string_at_offset(entity::position_t offset, bool strip_fixed_width_trailing_nulls=false) const;