Add annoy index

.gitmodules

@@ -259,6 +259,10 @@
 [submodule "contrib/minizip-ng"]
 	path = contrib/minizip-ng
 	url = https://github.com/zlib-ng/minizip-ng
+[submodule "contrib/annoy"]
+	path = contrib/annoy
+	url = https://github.com/ClickHouse/annoy.git
+	branch = ClickHouse-master
 [submodule "contrib/qpl"]
 	path = contrib/qpl
 	url = https://github.com/intel/qpl.git

contrib/CMakeLists.txt

@@ -159,6 +159,8 @@ add_contrib (s2geometry-cmake s2geometry)
 add_contrib (c-ares-cmake c-ares)
 add_contrib (qpl-cmake qpl)
 
+add_contrib(annoy-cmake annoy)
+
 # Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
 # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
 # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,

contrib/annoy-cmake/CMakeLists.txt

@@ -0,0 +1,24 @@
+option(ENABLE_ANNOY "Enable Annoy index support" ${ENABLE_LIBRARIES})
+
+# Annoy index should be disabled with undefined sanitizer. Because of memory storage optimizations
+# (https://github.com/ClickHouse/annoy/blob/9d8a603a4cd252448589e84c9846f94368d5a289/src/annoylib.h#L442-L463)
+# UBSan fails and leads to crash. Simmilar issue is already opened in Annoy repo
+# https://github.com/spotify/annoy/issues/456
+# Problem with aligment can lead to errors like
+# (https://stackoverflow.com/questions/46790550/c-undefined-behavior-strict-aliasing-rule-or-incorrect-alignment)
+# or will lead to crash on arm https://developer.arm.com/documentation/ka003038/latest
+# This issues should be resolved before annoy became non-experimental (--> setting "allow_experimental_annoy_index")
+if ((NOT ENABLE_ANNOY) OR (SANITIZE STREQUAL "undefined") OR (ARCH_AARCH64))
+    message (STATUS "Not using annoy")
+    return()
+endif()
+
+set(ANNOY_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/annoy")
+set(ANNOY_SOURCE_DIR "${ANNOY_PROJECT_DIR}/src")
+
+add_library(_annoy INTERFACE)
+target_include_directories(_annoy SYSTEM INTERFACE ${ANNOY_SOURCE_DIR})
+
+add_library(ch_contrib::annoy ALIAS _annoy)
+target_compile_definitions(_annoy INTERFACE ENABLE_ANNOY)
+target_compile_definitions(_annoy INTERFACE ANNOYLIB_MULTITHREADED_BUILD)

docs/en/engines/table-engines/mergetree-family/annindexes.md

@@ -0,0 +1,127 @@
+# Approximate Nearest Neighbor Search Indexes [experimental] {#table_engines-ANNIndex}
+
+The main task that indexes achieve is to quickly find nearest neighbors for multidimensional data. An example of such a problem can be finding similar pictures (texts) for a given picture (text). That problem can be reduced to finding the nearest [embeddings](https://cloud.google.com/architecture/overview-extracting-and-serving-feature-embeddings-for-machine-learning). They can be created from data using [UDF](../../../sql-reference/functions/index.md#executable-user-defined-functions).
+
+The next query finds the closest neighbors in N-dimensional space using the L2 (Euclidean) distance:
+``` sql 
+SELECT * 
+FROM table_name 
+WHERE L2Distance(Column, Point) < MaxDistance 
+LIMIT N
+```
+But it will take some time for execution because of the long calculation of the distance between `TargetEmbedding` and all other vectors. This is where ANN indexes can help. They store a compact approximation of the search space (e.g. using clustering, search trees, etc.) and are able to compute approximate neighbors quickly.
+
+## Indexes Structure
+
+Approximate Nearest Neighbor Search Indexes (`ANNIndexes`) are similar to skip indexes. They are constructed by some granules and determine which of them should be skipped. Compared to skip indices, ANN indices use their results not only to skip some group of granules, but also to select particular granules from a set of granules.
+
+`ANNIndexes` are designed to speed up two types of queries:
+
+- ######  Type 1: Where 
+   ``` sql 
+   SELECT * 
+   FROM table_name 
+   WHERE DistanceFunction(Column, Point) < MaxDistance 
+   LIMIT N
+   ```
+- ###### Type 2: Order by
+  ``` sql
+  SELECT * 
+  FROM table_name [WHERE ...] 
+  ORDER BY DistanceFunction(Column, Point) 
+  LIMIT N
+  ```
+
+In these queries, `DistanceFunction` is selected from [distance functions](../../../sql-reference/functions/distance-functions). `Point` is a known vector (something like `(0.1, 0.1, ... )`). To avoid writing large vectors, use [client parameters](../../../interfaces/cli.md#queries-with-parameters-cli-queries-with-parameters). `Value` - a float value that will bound the neighbourhood.
+
+!!! note "Note"
+    ANN index can't speed up query that satisfies both types(`where + order by`, only one of them). All queries must have the limit, as algorithms are used to find nearest neighbors and need a specific number of them.
+
+!!! note "Note"
+    Indexes are applied only to queries with a limit less than the `max_limit_for_ann_queries` setting. This helps to avoid memory overflows in queries with a large limit. `max_limit_for_ann_queries` setting can be changed if you know you can provide enough memory. The default value is `1000000`.
+
+Both types of queries are handled the same way. The indexes get `n` neighbors (where `n` is taken from the `LIMIT` clause) and work with them. In `ORDER BY` query they remember the numbers of all parts of the granule that have at least one of neighbor. In `WHERE` query they remember only those parts that satisfy the requirements.
+
+
+
+## Create table with ANNIndex
+
+This feature is disabled by default. To enable it, set `allow_experimental_annoy_index` to 1. Also, this feature is disabled for arm, due to likely problems with the algorithm.
+
+```sql
+CREATE TABLE t
+(
+  `id` Int64,
+  `number` Tuple(Float32, Float32, Float32),
+  INDEX x number TYPE annoy GRANULARITY N
+)
+ENGINE = MergeTree
+ORDER BY id;
+```
+
+```sql
+CREATE TABLE t
+(
+  `id` Int64,
+  `number` Array(Float32),
+  INDEX x number TYPE annoy GRANULARITY N
+)
+ENGINE = MergeTree
+ORDER BY id;
+```
+
+With greater `GRANULARITY` indexes remember the data structure better. The `GRANULARITY` indicates how many granules will be used to construct the index. The more data is provided for the index, the more of it can be handled by one index and the more chances that with the right hyperparameters the index will remember the data structure better. But some indexes can't be built if they don't have enough data, so this granule will always participate in the query. For more information, see the description of indexes.
+
+As the indexes are built only during insertions into table, `INSERT` and `OPTIMIZE` queries are slower than for ordinary table. At this stage indexes remember all the information about the given data. ANNIndexes should be used if you have immutable or rarely changed data and many read requests.
+
+You can create your table with index which uses certain algorithm. Now only indices based on the following algorithms are supported:
+
+# Index list
+- [Annoy](../../../engines/table-engines/mergetree-family/annindexes.md#annoy-annoy)
+
+# Annoy {#annoy}
+Implementation of the algorithm was taken from [this repository](https://github.com/spotify/annoy).
+
+Short description of the algorithm:
+The algorithm recursively divides in half all space by random linear surfaces (lines in 2D, planes in 3D e.t.c.). Thus it makes tree of polyhedrons and points that they contains. Repeating the operation several times for greater accuracy it creates a forest.
+To find K Nearest Neighbours it goes down through the trees and fills the buffer of closest points using the priority queue of polyhedrons. Next, it sorts buffer and return the nearest K points.
+
+__Examples__:
+```sql
+CREATE TABLE t
+(
+  id Int64,
+  number Tuple(Float32, Float32, Float32),
+  INDEX x number TYPE annoy(T) GRANULARITY N
+)
+ENGINE = MergeTree
+ORDER BY id;
+```
+
+```sql
+CREATE TABLE t
+(
+  id Int64,
+  number Array(Float32),
+  INDEX x number TYPE annoy(T) GRANULARITY N
+)
+ENGINE = MergeTree
+ORDER BY id;
+```
+!!! note "Note"
+    Table with array field will work faster, but all arrays **must** have same length. Use [CONSTRAINT](../../../sql-reference/statements/create/table.md#constraints) to avoid errors. For example, `CONSTRAINT constraint_name_1 CHECK length(number) = 256`.
+
+Parameter `T` is the number of trees which algorithm will create. The bigger it is, the slower (approximately linear) it works (in both `CREATE` and `SELECT` requests), but the better accuracy you get (adjusted for randomness). 
+
+Annoy supports only `L2Distance`.
+
+In the `SELECT` in the settings (`ann_index_select_query_params`) you can specify the size of the internal buffer (more details in the description above or in the [original repository](https://github.com/spotify/annoy)). During the query it will inspect up to `search_k` nodes which defaults to `n_trees * n` if not provided. `search_k` gives you a run-time tradeoff between better accuracy and speed.
+
+__Example__:
+``` sql
+SELECT * 
+FROM table_name [WHERE ...] 
+ORDER BY L2Distance(Column, Point) 
+LIMIT N
+SETTING ann_index_select_query_params=`k_search=100`
+```

docs/en/engines/table-engines/mergetree-family/mergetree.md

@@ -481,6 +481,10 @@ For example:
     -   `NOT startsWith(s, 'test')`
 :::
 
+
+## Approximate Nearest Neighbor Search Indexes [experimental] {#table_engines-ANNIndex}
+In addition to skip indices, there are also [Approximate Nearest Neighbor Search Indexes](../../../engines/table-engines/mergetree-family/annindexes.md).
+
 ## Projections {#projections}
 Projections are like [materialized views](../../../sql-reference/statements/create/view.md#materialized) but defined in part-level. It provides consistency guarantees along with automatic usage in queries.
 

src/CMakeLists.txt

@@ -579,6 +579,10 @@ endif()
 
 dbms_target_link_libraries(PUBLIC ch_contrib::consistent_hashing)
 
+if (TARGET ch_contrib::annoy)
+    dbms_target_link_libraries(PUBLIC ch_contrib::annoy)
+endif()
+
 include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake")
 
 if (ENABLE_TESTS)

src/Core/Settings.h

@@ -639,6 +639,9 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions (hashid, etc)", 0) \
     M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
     M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \
+    M(String, ann_index_select_query_params, "", "Parameters passed to ANN indexes in SELECT queries, the format is 'param1=x, param2=y, ...'", 0) \
+    M(UInt64, max_limit_for_ann_queries, 1000000, "Maximum limit value for using ANN indexes is used to prevent memory overflow in search queries for indexes", 0) \
+    M(Bool, allow_experimental_annoy_index, false, "Allows to use Annoy index. Disabled by default because this feature is experimental", 0) \
     M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \
     M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \
     M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \

src/Interpreters/InterpreterCreateQuery.cpp

@@ -668,8 +668,12 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
 
         if (create.columns_list->indices)
             for (const auto & index : create.columns_list->indices->children)
+            {
                 properties.indices.push_back(
                     IndexDescription::getIndexFromAST(index->clone(), properties.columns, getContext()));
+                    if (properties.indices.back().type == "annoy" && !getContext()->getSettingsRef().allow_experimental_annoy_index)
+                        throw Exception("Annoy index is disabled. Turn on allow_experimental_annoy_index", ErrorCodes::INCORRECT_QUERY);
+            }
 
         if (create.columns_list->projections)
             for (const auto & projection_ast : create.columns_list->projections->children)