New input format Npy

docs/en/interfaces/formats.md

@@ -74,6 +74,7 @@ The supported formats are:
 | [ArrowStream](#data-format-arrow-stream)                                                  | ✔    | ✔     |
 | [ORC](#data-format-orc)                                                                   | ✔    | ✔     |
 | [One](#data-format-one)                                                                   | ✔    | ✗     |
+| [Npy](#data-format-npy)                                                                   | ✔    | ✗     |
 | [RowBinary](#rowbinary)                                                                   | ✔    | ✔     |
 | [RowBinaryWithNames](#rowbinarywithnamesandtypes)                                         | ✔    | ✔     |
 | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes)                                 | ✔    | ✔     |
@@ -2445,6 +2446,41 @@ Result:
 └──────────────┘
 ```
 
+## Npy {#data-format-npy}
+
+This function is designed to load a NumPy array from a .npy file into ClickHouse. The NumPy file format is a binary format used for efficiently storing arrays of numerical data. During import, ClickHouse treats top level dimension as an array of rows with single column. Supported Npy data types and their corresponding type in ClickHouse: 
+| Npy type | ClickHouse type |
+|:--------:|:---------------:|
+| b1       |    Int8         |
+| i1       |    Int8         |
+| i2       |    Int16        |
+| i4       |    Int32        |
+| i8       |    Int64        |
+| u1       |    UInt8        |
+| u2       |    UInt16       |
+| u4       |    UInt32       |
+| u8       |    UInt64       |
+| f4       |    Float32      |
+| f8       |    Float64      |
+| S        |    String       |
+| U        |    String       |
+
+**Example**
+
+Query:
+```sql
+SELECT *
+FROM file('example_array.npy', Npy)
+```
+
+Result:
+```
+┌─array─────────┐
+│ [[1],[2],[3]] │
+│ [[4],[5],[6]] │
+└───────────────┘
+```
+
 ## LineAsString {#lineasstring}
 
 In this format, every line of input data is interpreted as a single string value. This format can only be parsed for table with a single field of type [String](/docs/en/sql-reference/data-types/string.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized), or omitted.

src/Formats/NumpyDataTypes.h

@@ -0,0 +1,108 @@
+#include <cstddef>
+#include <Storages/NamedCollectionsHelpers.h>
+
+enum class NumpyDataTypeIndex
+{
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Float32,
+    Float64,
+    String,
+    Unicode,
+};
+
+class NumpyDataType
+{
+public:
+    enum Endianness
+    {
+        LITTLE,
+        BIG,
+        NONE,
+    };
+
+    explicit NumpyDataType(Endianness endianess_) : endianess(endianess_) {}
+    virtual ~NumpyDataType() = default;
+
+    Endianness getEndianness() const { return endianess; }
+
+    virtual NumpyDataTypeIndex getTypeIndex() const = 0;
+    virtual size_t getSize() const = 0;
+
+private:
+    Endianness endianess;
+};
+
+class NumpyDataTypeInt : public NumpyDataType
+{
+public:
+    NumpyDataTypeInt(Endianness endianess, size_t size_, bool is_signed_) : NumpyDataType(endianess), size(size_), is_signed(is_signed_) {}
+
+    NumpyDataTypeIndex getTypeIndex() const override
+    {
+        switch (size)
+        {
+            case 1: return is_signed ? NumpyDataTypeIndex::Int8 : NumpyDataTypeIndex::UInt8;
+            case 2: return is_signed ? NumpyDataTypeIndex::Int16 : NumpyDataTypeIndex::UInt16;
+            case 4: return is_signed ? NumpyDataTypeIndex::Int32 : NumpyDataTypeIndex::UInt32;
+            case 8: return is_signed ? NumpyDataTypeIndex::Int64 : NumpyDataTypeIndex::UInt64;
+            default:
+                throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Incorrect int type with size {}", size);
+        }
+    }
+    size_t getSize() const override { return size; }
+    bool isSigned() const { return is_signed; }
+
+private:
+    size_t size;
+    bool is_signed;
+};
+
+class NumpyDataTypeFloat : public NumpyDataType
+{
+public:
+    NumpyDataTypeFloat(Endianness endianess, size_t size_) : NumpyDataType(endianess), size(size_) {}
+
+    NumpyDataTypeIndex getTypeIndex() const override
+    {
+        switch (size)
+        {
+            case 4: return NumpyDataTypeIndex::Float32;
+            case 8: return NumpyDataTypeIndex::Float64;
+            default:
+                throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Incorrect float type with size {}", size);
+        }
+    }
+    size_t getSize() const override { return size; }
+
+private:
+    size_t size;
+};
+
+class NumpyDataTypeString : public NumpyDataType
+{
+public:
+    NumpyDataTypeString(Endianness endianess, size_t size_) : NumpyDataType(endianess), size(size_) {}
+
+    NumpyDataTypeIndex getTypeIndex() const override { return NumpyDataTypeIndex::String; }
+    size_t getSize() const override { return size; }
+private:
+    size_t size;
+};
+
+class NumpyDataTypeUnicode : public NumpyDataType
+{
+public:
+    NumpyDataTypeUnicode(Endianness endianess, size_t size_) : NumpyDataType(endianess), size(size_) {}
+
+    NumpyDataTypeIndex getTypeIndex() const override { return NumpyDataTypeIndex::Unicode; }
+    size_t getSize() const override { return size; }
+private:
+    size_t size;
+};

src/Formats/registerFormats.cpp

@@ -102,6 +102,7 @@ void registerInputFormatLineAsString(FormatFactory & factory);
 void registerInputFormatMySQLDump(FormatFactory & factory);
 void registerInputFormatParquetMetadata(FormatFactory & factory);
 void registerInputFormatOne(FormatFactory & factory);
+void registerInputFormatNpy(FormatFactory & factory);
 
 #if USE_HIVE
 void registerInputFormatHiveText(FormatFactory & factory);
@@ -144,6 +145,7 @@ void registerMySQLSchemaReader(FormatFactory & factory);
 void registerBSONEachRowSchemaReader(FormatFactory & factory);
 void registerParquetMetadataSchemaReader(FormatFactory & factory);
 void registerOneSchemaReader(FormatFactory & factory);
+void registerNpySchemaReader(FormatFactory & factory);
 
 void registerFileExtensions(FormatFactory & factory);
 
@@ -246,6 +248,7 @@ void registerFormats()
 
     registerInputFormatParquetMetadata(factory);
     registerInputFormatOne(factory);
+    registerInputFormatNpy(factory);
 
     registerNonTrivialPrefixAndSuffixCheckerJSONEachRow(factory);
     registerNonTrivialPrefixAndSuffixCheckerJSONAsString(factory);
@@ -283,6 +286,7 @@ void registerFormats()
     registerBSONEachRowSchemaReader(factory);
     registerParquetMetadataSchemaReader(factory);
     registerOneSchemaReader(factory);
+    registerNpySchemaReader(factory);
 }
 
 }