Merge pull request #55982 from yariks5s/npy_input_format

New input format Npy

docs/en/interfaces/formats.md

@@ -74,6 +74,7 @@ The supported formats are:
 | [ArrowStream](#data-format-arrow-stream)                                                  | ✔    | ✔     |
 | [ORC](#data-format-orc)                                                                   | ✔    | ✔     |
 | [One](#data-format-one)                                                                   | ✔    | ✗     |
+| [Npy](#data-format-npy)                                                                   | ✔    | ✗     |
 | [RowBinary](#rowbinary)                                                                   | ✔    | ✔     |
 | [RowBinaryWithNames](#rowbinarywithnamesandtypes)                                         | ✔    | ✔     |
 | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes)                                 | ✔    | ✔     |
@@ -2454,6 +2455,50 @@ Result:
 └──────────────┘
 ```
 
+## Npy {#data-format-npy}
+
+This function is designed to load a NumPy array from a .npy file into ClickHouse. The NumPy file format is a binary format used for efficiently storing arrays of numerical data. During import, ClickHouse treats top level dimension as an array of rows with single column. Supported Npy data types and their corresponding type in ClickHouse: 
+| Npy type | ClickHouse type |
+|:--------:|:---------------:|
+| b1       |    UInt8        |
+| i1       |    Int8         |
+| i2       |    Int16        |
+| i4       |    Int32        |
+| i8       |    Int64        |
+| u1       |    UInt8        |
+| u2       |    UInt16       |
+| u4       |    UInt32       |
+| u8       |    UInt64       |
+| f4       |    Float32      |
+| f8       |    Float64      |
+| S        |    String       |
+| U        |    String       |
+
+**Example of saving an array in .npy format using Python**
+
+
+```Python
+import numpy as np
+arr = np.array([[[1],[2],[3]],[[4],[5],[6]]])
+np.save('example_array.npy', arr)
+```
+
+**Example of reading a NumPy file in ClickHouse**
+
+Query:
+```sql
+SELECT *
+FROM file('example_array.npy', Npy)
+```
+
+Result:
+```
+┌─array─────────┐
+│ [[1],[2],[3]] │
+│ [[4],[5],[6]] │
+└───────────────┘
+```
+
 ## LineAsString {#lineasstring}
 
 In this format, every line of input data is interpreted as a single string value. This format can only be parsed for table with a single field of type [String](/docs/en/sql-reference/data-types/string.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized), or omitted.

src/Formats/NumpyDataTypes.h

@@ -0,0 +1,123 @@
+#pragma once
+#include <cstddef>
+#include <Storages/NamedCollectionsHelpers.h>
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+enum class NumpyDataTypeIndex
+{
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Float32,
+    Float64,
+    String,
+    Unicode,
+};
+
+class NumpyDataType
+{
+public:
+    enum Endianness
+    {
+        LITTLE,
+        BIG,
+        NONE,
+    };
+    NumpyDataTypeIndex type_index;
+
+    explicit NumpyDataType(Endianness endianness_) : endianness(endianness_) {}
+    virtual ~NumpyDataType() = default;
+
+    Endianness getEndianness() const { return endianness; }
+
+    virtual NumpyDataTypeIndex getTypeIndex() const = 0;
+
+private:
+    Endianness endianness;
+};
+
+class NumpyDataTypeInt : public NumpyDataType
+{
+public:
+    NumpyDataTypeInt(Endianness endianness, size_t size_, bool is_signed_) : NumpyDataType(endianness), size(size_), is_signed(is_signed_)
+    {
+        switch (size)
+        {
+            case 1: type_index = is_signed ? NumpyDataTypeIndex::Int8 : NumpyDataTypeIndex::UInt8; break;
+            case 2: type_index = is_signed ? NumpyDataTypeIndex::Int16 : NumpyDataTypeIndex::UInt16; break;
+            case 4: type_index = is_signed ? NumpyDataTypeIndex::Int32 : NumpyDataTypeIndex::UInt32; break;
+            case 8: type_index = is_signed ? NumpyDataTypeIndex::Int64 : NumpyDataTypeIndex::UInt64; break;
+            default:
+                throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Incorrect int type with size {}", size);
+        }
+    }
+
+    NumpyDataTypeIndex getTypeIndex() const override
+    {
+        return type_index;
+    }
+    bool isSigned() const { return is_signed; }
+
+private:
+    size_t size;
+    bool is_signed;
+};
+
+class NumpyDataTypeFloat : public NumpyDataType
+{
+public:
+    NumpyDataTypeFloat(Endianness endianness, size_t size_) : NumpyDataType(endianness), size(size_)
+    {
+        switch (size)
+        {
+            case 4: type_index = NumpyDataTypeIndex::Float32; break;
+            case 8: type_index = NumpyDataTypeIndex::Float64; break;
+            default:
+                throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Numpy float type with size {} is not supported", size);
+        }
+    }
+
+    NumpyDataTypeIndex getTypeIndex() const override
+    {
+        return type_index;
+    }
+private:
+    size_t size;
+};
+
+class NumpyDataTypeString : public NumpyDataType
+{
+public:
+    NumpyDataTypeString(Endianness endianness, size_t size_) : NumpyDataType(endianness), size(size_)
+    {
+        type_index = NumpyDataTypeIndex::String;
+    }
+
+    NumpyDataTypeIndex getTypeIndex() const override { return type_index; }
+    size_t getSize() const { return size; }
+private:
+    size_t size;
+};
+
+class NumpyDataTypeUnicode : public NumpyDataType
+{
+public:
+    NumpyDataTypeUnicode(Endianness endianness, size_t size_) : NumpyDataType(endianness), size(size_)
+    {
+        type_index = NumpyDataTypeIndex::Unicode;
+    }
+
+    NumpyDataTypeIndex getTypeIndex() const override { return type_index; }
+    size_t getSize() const { return size * 4; }
+private:
+    size_t size;
+};

src/Formats/registerFormats.cpp

@@ -103,6 +103,7 @@ void registerInputFormatMySQLDump(FormatFactory & factory);
 void registerInputFormatParquetMetadata(FormatFactory & factory);
 void registerInputFormatDWARF(FormatFactory & factory);
 void registerInputFormatOne(FormatFactory & factory);
+void registerInputFormatNpy(FormatFactory & factory);
 
 #if USE_HIVE
 void registerInputFormatHiveText(FormatFactory & factory);
@@ -146,6 +147,7 @@ void registerBSONEachRowSchemaReader(FormatFactory & factory);
 void registerParquetMetadataSchemaReader(FormatFactory & factory);
 void registerDWARFSchemaReader(FormatFactory & factory);
 void registerOneSchemaReader(FormatFactory & factory);
+void registerNpySchemaReader(FormatFactory & factory);
 
 void registerFileExtensions(FormatFactory & factory);
 
@@ -249,6 +251,7 @@ void registerFormats()
     registerInputFormatParquetMetadata(factory);
     registerInputFormatDWARF(factory);
     registerInputFormatOne(factory);
+    registerInputFormatNpy(factory);
 
     registerNonTrivialPrefixAndSuffixCheckerJSONEachRow(factory);
     registerNonTrivialPrefixAndSuffixCheckerJSONAsString(factory);
@@ -287,6 +290,7 @@ void registerFormats()
     registerParquetMetadataSchemaReader(factory);
     registerDWARFSchemaReader(factory);
     registerOneSchemaReader(factory);
+    registerNpySchemaReader(factory);
 }
 
 }