Include PODArray.h into fewer files.

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp

@@ -1,9 +1,12 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
-#include <AggregateFunctions/AggregateFunctionGroupBitmap.h>
 #include <AggregateFunctions/Helpers.h>
 #include <AggregateFunctions/FactoryHelpers.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
 
+// TODO include this last because of a broken roaring header. See the comment
+// inside.
+#include <AggregateFunctions/AggregateFunctionGroupBitmap.h>
+
 namespace DB
 {
 

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h

@@ -3,10 +3,13 @@
 #include <Columns/ColumnVector.h>
 #include <Common/assert_cast.h>
 #include <AggregateFunctions/IAggregateFunction.h>
-#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Columns/ColumnAggregateFunction.h>
 
+// TODO include this last because of a broken roaring header. See the comment
+// inside.
+#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
+
 namespace DB
 {
 

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h

@@ -1,14 +1,18 @@
 #pragma once
 
 #include <algorithm>
-#include <roaring/roaring.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <boost/noncopyable.hpp>
-#include <roaring/roaring.hh>
 #include <Common/HashTable/SmallTable.h>
 #include <Common/PODArray.h>
 
+// Include this header last, because it is an auto-generated dump of questionable
+// garbage that breaks the build (e.g. it changes _POSIX_C_SOURCE).
+// TODO: find out what it is. On github, they have proper inteface headers like
+// this one: https://github.com/RoaringBitmap/CRoaring/blob/master/include/roaring/roaring.h
+#include <roaring/roaring.h>
+
 namespace DB
 {
 /**

dbms/src/Client/ConnectionPoolWithFailover.cpp

@@ -3,6 +3,7 @@
 #include <Poco/Net/NetException.h>
 #include <Poco/Net/DNS.h>
 
+#include <Common/BitHelpers.h>
 #include <Common/getFQDNOrHostName.h>
 #include <Common/isLocalAddress.h>
 #include <Common/ProfileEvents.h>

dbms/src/Columns/ColumnAggregateFunction.h

@@ -3,6 +3,7 @@
 #include <AggregateFunctions/IAggregateFunction.h>
 
 #include <Columns/IColumn.h>
+#include <Common/PODArray.h>
 
 #include <Core/Field.h>
 

dbms/src/Columns/ColumnArray.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Columns/IColumn.h>
+#include <Columns/IColumnImpl.h>
 #include <Columns/ColumnVector.h>
 #include <Core/Defines.h>
 #include <Common/typeid_cast.h>

dbms/src/Columns/ColumnConst.cpp

@@ -2,6 +2,7 @@
 
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnsCommon.h>
+#include <Common/PODArray.h>
 #include <Common/typeid_cast.h>
 
 

dbms/src/Columns/ColumnDecimal.h

@@ -4,6 +4,7 @@
 
 #include <Common/typeid_cast.h>
 #include <Columns/IColumn.h>
+#include <Columns/IColumnImpl.h>
 #include <Columns/ColumnVectorHelper.h>
 #include <Core/Field.h>
 

dbms/src/Columns/ColumnFixedString.h

@@ -5,6 +5,7 @@
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>
 #include <Columns/IColumn.h>
+#include <Columns/IColumnImpl.h>
 #include <Columns/ColumnVectorHelper.h>
 #include <Core/Field.h>
 

dbms/src/Columns/ColumnFunction.cpp

@@ -1,6 +1,7 @@
 #include <Interpreters/ExpressionActions.h>
 #include <Columns/ColumnFunction.h>
 #include <Columns/ColumnsCommon.h>
+#include <Common/PODArray.h>
 #include <IO/WriteHelpers.h>
 #include <Functions/IFunction.h>
 

dbms/src/Columns/ColumnNullable.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Columns/IColumn.h>
+#include <Columns/IColumnImpl.h>
 #include <Columns/ColumnsNumber.h>
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>

dbms/src/Columns/ColumnString.h

@@ -4,6 +4,7 @@
 #include <cassert>
 
 #include <Columns/IColumn.h>
+#include <Columns/IColumnImpl.h>
 #include <Common/PODArray.h>
 #include <Common/SipHash.h>
 #include <Common/memcpySmall.h>

dbms/src/Columns/ColumnVector.h

@@ -2,6 +2,7 @@
 
 #include <cmath>
 #include <Columns/IColumn.h>
+#include <Columns/IColumnImpl.h>
 #include <Columns/ColumnVectorHelper.h>
 #include <common/unaligned.h>
 #include <Core/Field.h>

dbms/src/Columns/IColumn.h

@@ -1,10 +1,11 @@
 #pragma once
 
 #include <Common/COW.h>
-#include <Common/PODArray.h>
+#include <Common/PODArray_fwd.h>
 #include <Common/Exception.h>
 #include <Common/typeid_cast.h>
 #include <common/StringRef.h>
+#include <Core/Types.h>
 
 
 class SipHash;
@@ -373,32 +374,7 @@ class IColumn : public COW<IColumn>
     /// Template is to devirtualize calls to insertFrom method.
     /// In derived classes (that use final keyword), implement scatter method as call to scatterImpl.
     template <typename Derived>
-    std::vector<MutablePtr> scatterImpl(ColumnIndex num_columns, const Selector & selector) const
-    {
-        size_t num_rows = size();
-
-        if (num_rows != selector.size())
-            throw Exception(
-                    "Size of selector: " + std::to_string(selector.size()) + " doesn't match size of column: " + std::to_string(num_rows),
-                    ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
-
-        std::vector<MutablePtr> columns(num_columns);
-        for (auto & column : columns)
-            column = cloneEmpty();
-
-        {
-            size_t reserve_size = num_rows * 1.1 / num_columns;    /// 1.1 is just a guess. Better to use n-sigma rule.
-
-            if (reserve_size > 1)
-                for (auto & column : columns)
-                    column->reserve(reserve_size);
-        }
-
-        for (size_t i = 0; i < num_rows; ++i)
-            static_cast<Derived &>(*columns[selector[i]]).insertFrom(*this, i);
-
-        return columns;
-    }
+    std::vector<MutablePtr> scatterImpl(ColumnIndex num_columns, const Selector & selector) const;
 };
 
 using ColumnPtr = IColumn::Ptr;

dbms/src/Columns/IColumnDummy.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Common/Arena.h>
+#include <Common/PODArray.h>
 #include <Columns/IColumn.h>
 #include <Columns/ColumnsCommon.h>
 

dbms/src/Columns/IColumnImpl.h

@@ -0,0 +1,45 @@
+/**
+  * This file implements template methods of IColumn that depend on other types
+  * we don't want to include.
+  * Currently, this is only the scatterImpl method that depends on PODArray
+  * implementation.
+  */
+
+#pragma once
+
+#include <Columns/IColumn.h>
+#include <Common/PODArray.h>
+
+namespace DB
+{
+
+template <typename Derived>
+std::vector<IColumn::MutablePtr> IColumn::scatterImpl(ColumnIndex num_columns,
+                                             const Selector & selector) const
+{
+    size_t num_rows = size();
+
+    if (num_rows != selector.size())
+        throw Exception(
+                "Size of selector: " + std::to_string(selector.size()) + " doesn't match size of column: " + std::to_string(num_rows),
+                ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
+
+    std::vector<MutablePtr> columns(num_columns);
+    for (auto & column : columns)
+        column = cloneEmpty();
+
+    {
+        size_t reserve_size = num_rows * 1.1 / num_columns;    /// 1.1 is just a guess. Better to use n-sigma rule.
+
+        if (reserve_size > 1)
+            for (auto & column : columns)
+                column->reserve(reserve_size);
+    }
+
+    for (size_t i = 0; i < num_rows; ++i)
+        static_cast<Derived &>(*columns[selector[i]]).insertFrom(*this, i);
+
+    return columns;
+}
+
+}

dbms/src/Common/Allocator.h

@@ -30,6 +30,8 @@
 #include <Common/Exception.h>
 #include <Common/formatReadable.h>
 
+#include <Common/Allocator_fwd.h>
+
 
 /// Required for older Darwin builds, that lack definition of MAP_ANONYMOUS
 #ifndef MAP_ANONYMOUS
@@ -84,7 +86,7 @@ namespace ErrorCodes
   * - random hint address for mmap
   * - mmap_threshold for using mmap less or more
   */
-template <bool clear_memory_, bool mmap_populate = false>
+template <bool clear_memory_, bool mmap_populate>
 class Allocator
 {
 public:
@@ -270,7 +272,7 @@ class Allocator
 
 /** Allocator with optimization to place small memory ranges in automatic memory.
   */
-template <typename Base, size_t N = 64, size_t Alignment = 1>
+template <typename Base, size_t N, size_t Alignment>
 class AllocatorWithStackMemory : private Base
 {
 private:

dbms/src/Common/Allocator_fwd.h

@@ -0,0 +1,10 @@
+/**
+  * This file provides forward declarations for Allocator.
+  */
+#pragma once
+
+template <bool clear_memory_, bool mmap_populate = false>
+class Allocator;
+
+template <typename Base, size_t N = 64, size_t Alignment = 1>
+class AllocatorWithStackMemory;

dbms/src/Common/PODArray.h

@@ -21,6 +21,8 @@
     #include <sys/mman.h>
 #endif
 
+#include <Common/PODArray_fwd.h>
+
 
 namespace DB
 {
@@ -30,11 +32,6 @@ namespace ErrorCodes
     extern const int CANNOT_MPROTECT;
 }
 
-inline constexpr size_t integerRoundUp(size_t value, size_t dividend)
-{
-    return ((value + dividend - 1) / dividend) * dividend;
-}
-
 /** A dynamic array for POD types.
   * Designed for a small number of large arrays (rather than a lot of small ones).
   * To be more precise - for use in ColumnVector.
@@ -258,7 +255,7 @@ class PODArrayBase : private boost::noncopyable, private TAllocator    /// empty
     }
 };
 
-template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>, size_t pad_right_ = 0, size_t pad_left_ = 0>
+template <typename T, size_t initial_bytes, typename TAllocator, size_t pad_right_, size_t pad_left_>
 class PODArray : public PODArrayBase<sizeof(T), initial_bytes, TAllocator, pad_right_, pad_left_>
 {
 protected:
@@ -625,17 +622,5 @@ void swap(PODArray<T, initial_bytes, TAllocator, pad_right_> & lhs, PODArray<T,
     lhs.swap(rhs);
 }
 
-/** For columns. Padding is enough to read and write xmm-register at the address of the last element. */
-template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>>
-using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, 15, 16>;
-
-/** A helper for declaring PODArray that uses inline memory.
-  * The initial size is set to use all the inline bytes, since using less would
-  * only add some extra allocation calls.
-  */
-template <typename T, size_t inline_bytes,
-          size_t rounded_bytes = integerRoundUp(inline_bytes, sizeof(T))>
-using PODArrayWithStackMemory = PODArray<T, rounded_bytes,
-    AllocatorWithStackMemory<Allocator<false>, rounded_bytes, alignof(T)>>;
 
 }

dbms/src/Common/PODArray_fwd.h

@@ -0,0 +1,35 @@
+/**
+  * This file contains some using-declarations that define various kinds of
+  * PODArray.
+  */
+#pragma once
+
+#include <Common/Allocator_fwd.h>
+
+namespace DB
+{
+
+inline constexpr size_t integerRoundUp(size_t value, size_t dividend)
+{
+    return ((value + dividend - 1) / dividend) * dividend;
+}
+
+template <typename T, size_t initial_bytes = 4096,
+          typename TAllocator = Allocator<false>, size_t pad_right_ = 0,
+          size_t pad_left_ = 0>
+class PODArray;
+
+/** For columns. Padding is enough to read and write xmm-register at the address of the last element. */
+template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>>
+using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, 15, 16>;
+
+/** A helper for declaring PODArray that uses inline memory.
+  * The initial size is set to use all the inline bytes, since using less would
+  * only add some extra allocation calls.
+  */
+template <typename T, size_t inline_bytes,
+          size_t rounded_bytes = integerRoundUp(inline_bytes, sizeof(T))>
+using PODArrayWithStackMemory = PODArray<T, rounded_bytes,
+    AllocatorWithStackMemory<Allocator<false>, rounded_bytes, alignof(T)>>;
+
+}

dbms/src/Common/formatReadable.h

@@ -1,8 +1,13 @@
 #pragma once
 
 #include <string>
-#include <IO/WriteBuffer.h>
 
+namespace DB
+{
+
+class WriteBuffer;
+
+}
 
 /// Displays the passed size in bytes as 123.45 GiB.
 void formatReadableSizeWithBinarySuffix(double value, DB::WriteBuffer & out, int precision = 2);

dbms/src/Compression/CompressionCodecMultiple.cpp

@@ -1,5 +1,6 @@
 #include <Compression/CompressionCodecMultiple.h>
 #include <Compression/CompressionInfo.h>
+#include <Common/PODArray.h>
 #include <common/unaligned.h>
 #include <Compression/CompressionFactory.h>
 #include <IO/ReadHelpers.h>

dbms/src/Compression/ICompressionCodec.h

@@ -4,7 +4,6 @@
 #include <IO/ReadBuffer.h>
 #include <IO/WriteBuffer.h>
 #include <IO/BufferWithOwnMemory.h>
-#include <Common/PODArray.h>
 #include <DataTypes/IDataType.h>
 #include <boost/noncopyable.hpp>
 #include <IO/UncompressedCache.h>

dbms/src/DataStreams/LimitByBlockInputStream.cpp

@@ -1,4 +1,5 @@
 #include <DataStreams/LimitByBlockInputStream.h>
+#include <Common/PODArray.h>
 #include <Common/SipHash.h>
 
 

dbms/src/DataStreams/ReverseBlockInputStream.cpp

@@ -1,5 +1,7 @@
 #include "ReverseBlockInputStream.h"
 
+#include <Common/PODArray.h>
+
 namespace DB
 {
     ReverseBlockInputStream::ReverseBlockInputStream(const BlockInputStreamPtr & input)