Polymorphic parts (compact format).

dbms/src/Common/ErrorCodes.cpp

@@ -484,6 +484,7 @@ namespace ErrorCodes
     extern const int CACHE_DICTIONARY_UPDATE_FAIL = 510;
     extern const int UNKNOWN_ROLE = 511;
     extern const int SET_NON_GRANTED_ROLE = 512;
+    extern const int UNKNOWN_PART_TYPE = 513;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/DataStreams/MarkInCompressedFile.h

@@ -40,6 +40,15 @@ struct MarkInCompressedFile
 
 };
 
-using MarksInCompressedFile = PODArray<MarkInCompressedFile>;
+class MarksInCompressedFile : public PODArray<MarkInCompressedFile>
+{
+public:
+    MarksInCompressedFile(size_t n) : PODArray(n) {}
+
+    void read(ReadBuffer & buffer, size_t from, size_t count)
+    {
+        buffer.readStrict(reinterpret_cast<char *>(data() + from), count * sizeof(MarkInCompressedFile));
+    }
+};
 
 }

dbms/src/DataStreams/TTLBlockInputStream.cpp

@@ -36,7 +36,7 @@ TTLBlockInputStream::TTLBlockInputStream(
     {
         if (force || isTTLExpired(ttl_info.min))
         {
-            new_ttl_infos.columns_ttl.emplace(name, MergeTreeDataPart::TTLInfo{});
+            new_ttl_infos.columns_ttl.emplace(name, IMergeTreeDataPart::TTLInfo{});
             empty_columns.emplace(name);
 
             auto it = column_defaults.find(name);
@@ -98,7 +98,7 @@ void TTLBlockInputStream::readSuffixImpl()
     new_ttl_infos.updatePartMinMaxTTL(new_ttl_infos.table_ttl.min, new_ttl_infos.table_ttl.max);
 
     data_part->ttl_infos = std::move(new_ttl_infos);
-    data_part->empty_columns = std::move(empty_columns);
+    data_part->expired_columns = std::move(empty_columns);
 
     if (rows_removed)
         LOG_INFO(log, "Removed " << rows_removed << " rows with expired TTL from part " << data_part->name);

dbms/src/DataStreams/TTLBlockInputStream.h

@@ -1,7 +1,7 @@
 #pragma once
 #include <DataStreams/IBlockInputStream.h>
 #include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/MergeTree/MergeTreeDataPart.h>
+#include <Storages/MergeTree/IMergeTreeDataPart.h>
 #include <Core/Block.h>
 
 #include <common/DateLUT.h>
@@ -39,8 +39,8 @@ class TTLBlockInputStream : public IBlockInputStream
     time_t current_time;
     bool force;
 
-    MergeTreeDataPart::TTLInfos old_ttl_infos;
-    MergeTreeDataPart::TTLInfos new_ttl_infos;
+    IMergeTreeDataPart::TTLInfos old_ttl_infos;
+    IMergeTreeDataPart::TTLInfos new_ttl_infos;
     NameSet empty_columns;
 
     size_t rows_removed = 0;

dbms/src/Interpreters/MutationsInterpreter.cpp

@@ -165,7 +165,7 @@ bool isStorageTouchedByMutations(
 
 MutationsInterpreter::MutationsInterpreter(
     StoragePtr storage_,
-    std::vector<MutationCommand> commands_,
+    MutationCommands commands_,
     const Context & context_,
     bool can_execute_)
     : storage(std::move(storage_))
@@ -437,6 +437,8 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
 
         if (i > 0)
             prepared_stages[i].output_columns = prepared_stages[i - 1].output_columns;
+        else if (!commands.additional_columns.empty())
+            prepared_stages[i].output_columns.insert(commands.additional_columns.begin(), commands.additional_columns.end());
 
         if (prepared_stages[i].output_columns.size() < all_columns.size())
         {

dbms/src/Interpreters/MutationsInterpreter.h

@@ -23,7 +23,7 @@ class MutationsInterpreter
 public:
     /// Storage to mutate, array of mutations commands and context. If you really want to execute mutation
     /// use can_execute = true, in other cases (validation, amount of commands) it can be false
-    MutationsInterpreter(StoragePtr storage_, std::vector<MutationCommand> commands_, const Context & context_, bool can_execute_);
+    MutationsInterpreter(StoragePtr storage_, MutationCommands commands_, const Context & context_, bool can_execute_);
 
     void validate(TableStructureReadLockHolder & table_lock_holder);
 
@@ -44,7 +44,7 @@ class MutationsInterpreter
     BlockInputStreamPtr addStreamsForLaterStages(const std::vector<Stage> & prepared_stages, BlockInputStreamPtr in) const;
 
     StoragePtr storage;
-    std::vector<MutationCommand> commands;
+    MutationCommands commands;
     const Context & context;
     bool can_execute;
 

dbms/src/Interpreters/PartLog.cpp

@@ -7,7 +7,7 @@
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeEnum.h>
-#include <Storages/MergeTree/MergeTreeDataPart.h>
+#include <Storages/MergeTree/IMergeTreeDataPart.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Interpreters/PartLog.h>
 

dbms/src/Interpreters/PartLog.h

@@ -51,15 +51,15 @@ struct PartLogElement
     void appendToBlock(Block & block) const;
 };
 
-struct MergeTreeDataPart;
+class IMergeTreeDataPart;
 
 
 /// Instead of typedef - to allow forward declaration.
 class PartLog : public SystemLog<PartLogElement>
 {
     using SystemLog<PartLogElement>::SystemLog;
 
-    using MutableDataPartPtr = std::shared_ptr<MergeTreeDataPart>;
+    using MutableDataPartPtr = std::shared_ptr<IMergeTreeDataPart>;
     using MutableDataPartsVector = std::vector<MutableDataPartPtr>;
 
 public:

dbms/src/Storages/MergeTree/AlterAnalysisResult.h

@@ -0,0 +1,24 @@
+#pragma once
+#include <Interpreters/ExpressionActions.h>
+
+namespace DB
+{
+    struct AlterAnalysisResult
+    {
+        /// Expression for column type conversion.
+        /// If no conversions are needed, expression=nullptr.
+        ExpressionActionsPtr expression = nullptr;
+
+        /// Denotes if metadata must be changed even if no file should be overwritten
+        /// (used for transformation-free changing of Enum values list).
+        bool force_update_metadata = false;
+
+        std::map<String, const IDataType *> new_types;
+
+        /// For every column that need to be converted: source column name,
+        ///  column name of calculated expression for conversion.
+        std::vector<std::pair<String, String>> conversions;
+        NamesAndTypesList removed_columns;
+        Names removed_indices;
+    };
+}

dbms/src/Storages/MergeTree/DataPartsExchange.cpp

@@ -233,7 +233,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
         readBinary(sum_files_size, in);
         if (server_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS)
         {
-            MergeTreeDataPart::TTLInfos ttl_infos;
+            IMergeTreeDataPart::TTLInfos ttl_infos;
             String ttl_infos_string;
             readBinary(ttl_infos_string, in);
             ReadBufferFromString ttl_infos_buffer(ttl_infos_string);
@@ -279,11 +279,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart(
 
     part_file.createDirectory();
 
-    MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared<MergeTreeData::DataPart>(data, reservation->getDisk(), part_name);
-    new_data_part->relative_path = relative_part_path;
-    new_data_part->is_temp = true;
-
-
     MergeTreeData::DataPart::Checksums checksums;
     for (size_t i = 0; i < files; ++i)
     {
@@ -327,6 +322,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart(
 
     assertEOF(in);
 
+    MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(), relative_part_path);
+    new_data_part->is_temp = true;
     new_data_part->modification_time = time(nullptr);
     new_data_part->loadColumnsChecksumsIndexes(true, false);
     new_data_part->checksums.checkEqual(checksums, false);