Backport #15192 to 20.8: Fix insert into storage Buffer after alter.

src/Storages/StorageBuffer.cpp

@@ -547,7 +547,7 @@ bool StorageBuffer::optimize(
     if (deduplicate)
         throw Exception("DEDUPLICATE cannot be specified when optimizing table of type Buffer", ErrorCodes::NOT_IMPLEMENTED);
 
-    flushAllBuffers(false);
+    flushAllBuffers(false, true);
     return true;
 }
 
@@ -595,14 +595,14 @@ bool StorageBuffer::checkThresholdsImpl(size_t rows, size_t bytes, time_t time_p
 }
 
 
-void StorageBuffer::flushAllBuffers(const bool check_thresholds)
+void StorageBuffer::flushAllBuffers(bool check_thresholds, bool reset_blocks_structure)
 {
     for (auto & buf : buffers)
-        flushBuffer(buf, check_thresholds);
+        flushBuffer(buf, check_thresholds, false, reset_blocks_structure);
 }
 
 
-void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool locked)
+void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool locked, bool reset_block_structure)
 {
     Block block_to_write;
     time_t current_time = time(nullptr);
@@ -655,6 +655,8 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc
     try
     {
         writeBlockToDestination(block_to_write, DatabaseCatalog::instance().tryGetTable(destination_id, global_context));
+        if (reset_block_structure)
+            buffer.data.clear();
     }
     catch (...)
     {
@@ -829,7 +831,9 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context,
     checkAlterIsPossible(params, context.getSettingsRef());
     auto metadata_snapshot = getInMemoryMetadataPtr();
 
-    /// So that no blocks of the old structure remain.
+    /// Flush all buffers to storages, so that no non-empty blocks of the old
+    /// structure remain. Structure of empty blocks will be updated during first
+    /// insert.
     optimize({} /*query*/, metadata_snapshot, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context);
 
     StorageInMemoryMetadata new_metadata = *metadata_snapshot;

src/Storages/StorageBuffer.h

@@ -130,9 +130,11 @@ friend class BufferBlockOutputStream;
 
     Poco::Logger * log;
 
-    void flushAllBuffers(bool check_thresholds = true);
-    /// Reset the buffer. If check_thresholds is set - resets only if thresholds are exceeded.
-    void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false);
+    void flushAllBuffers(bool check_thresholds = true, bool reset_blocks_structure = false);
+    /// Reset the buffer. If check_thresholds is set - resets only if thresholds
+    /// are exceeded. If reset_block_structure is set - clears inner block
+    /// structure inside buffer (useful in OPTIMIZE and ALTER).
+    void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false, bool reset_block_structure = false);
     bool checkThresholds(const Buffer & buffer, time_t current_time, size_t additional_rows = 0, size_t additional_bytes = 0) const;
     bool checkThresholdsImpl(size_t rows, size_t bytes, time_t time_passed) const;
 

tests/queries/0_stateless/01506_buffer_table_alter_block_structure.reference

@@ -0,0 +1,3 @@
+2020-01-01 00:05:00	
+2020-01-01 00:05:00	
+2020-01-01 00:06:00	hello

tests/queries/0_stateless/01506_buffer_table_alter_block_structure.sql

@@ -0,0 +1,22 @@
+DROP TABLE IF EXISTS buf_dest;
+DROP TABLE IF EXISTS buf;
+
+CREATE TABLE buf_dest (timestamp DateTime)
+ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp)
+ORDER BY (timestamp);
+
+CREATE TABLE buf (timestamp DateTime) Engine = Buffer(currentDatabase(), buf_dest, 16, 3, 20, 2000000, 20000000, 100000000, 300000000);;
+
+INSERT INTO buf (timestamp) VALUES (toDateTime('2020-01-01 00:05:00'));
+
+ALTER TABLE buf_dest ADD COLUMN s String;
+ALTER TABLE buf ADD COLUMN s String;
+
+SELECT * FROM buf;
+
+INSERT INTO buf (timestamp, s) VALUES (toDateTime('2020-01-01 00:06:00'), 'hello');
+
+SELECT * FROM buf ORDER BY timestamp;
+
+DROP TABLE IF EXISTS buf;
+DROP TABLE IF EXISTS buf_dest;