Make cosine similarity faster by storing magnitude and normalizing vectors

docs/changelog/99445.yaml

@@ -0,0 +1,5 @@
+pr: 99445
+summary: Make cosine similarity faster by storing magnitude and normalizing vectors
+area: Vector Search
+type: enhancement
+issues: []

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml

@@ -135,3 +135,87 @@ setup:
   - match: {hits.hits.2._id: "1"}
   - gte: {hits.hits.2._score: 0.78}
   - lte: {hits.hits.2._score: 0.791}
+
+---
+"L2 similarity with indexed cosine similarity vector":
+  - skip:
+      features: close_to
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "l2norm(params.query_vector, 'indexed_vector')"
+                params:
+                  query_vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+
+  - match: {hits.total: 3}
+
+  - match: {hits.hits.0._id: "1"}
+  - close_to: {hits.hits.0._score: {value: 301.36, error: 0.01}}
+
+  - match: {hits.hits.1._id: "2"}
+  - close_to: {hits.hits.1._score: {value: 11.34, error: 0.01}}
+
+  - match: {hits.hits.2._id: "3"}
+  - close_to: {hits.hits.2._score: {value: 0.01, error: 0.01}}
+---
+"L1 similarity with indexed cosine similarity vector":
+  - skip:
+      features: close_to
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "l1norm(params.query_vector, 'indexed_vector')"
+                params:
+                  query_vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+
+  - match: {hits.total: 3}
+
+  - match: {hits.hits.0._id: "1"}
+  - close_to: {hits.hits.0._score: {value: 485.18, error: 0.01}}
+
+  - match: {hits.hits.1._id: "2"}
+  - close_to: {hits.hits.1._score: {value: 12.30, error: 0.01}}
+
+  - match: {hits.hits.2._id: "3"}
+  - close_to: {hits.hits.2._score: {value: 0.01, error: 0.01}}
+---
+"Test vector magnitude equality":
+  - skip:
+      features: close_to
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "doc['vector'].magnitude"
+
+  - match: {hits.total: 3}
+
+  - match: {hits.hits.0._id: "1"}
+  - close_to: {hits.hits.0._score: {value: 429.6021, error: 0.01}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value: 192.6447, error: 0.01}}
+
+  - match: {hits.hits.2._id: "2"}
+  - close_to: {hits.hits.2._score: {value: 186.34454, error: 0.01}}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search_cosine.yml

@@ -0,0 +1,253 @@
+setup:
+  - skip:
+      version: ' - 7.99.99'
+      reason: 'kNN search added in 8.0'
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 5
+                index: true
+                similarity: cosine
+              normalized_vector:
+                type: dense_vector
+                dims: 5
+                index: true
+                similarity: cosine
+              end_normalized:
+                type: dense_vector
+                dims: 5
+                index: true
+                similarity: cosine
+              first_normalized:
+                type: dense_vector
+                dims: 5
+                index: true
+                similarity: cosine
+              middle_normalized:
+                type: dense_vector
+                dims: 5
+                index: true
+                similarity: cosine
+
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          name: cow.jpg
+          vector: [230.0, 300.33, -34.8988, 15.555, -200.0]
+          middle_normalized: [230.0, 300.33, -34.8988, 15.555, -200.0]
+          normalized_vector: [0.5353791, 0.6990887, -0.08123516, 0.03620792, -0.46554706]
+          end_normalized: [230.0, 300.33, -34.8988, 15.555, -200.0]
+          first_normalized: [0.5353791, 0.6990887, -0.08123516, 0.03620792, -0.46554706]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          name: moose.jpg
+          vector: [-0.5, 100.0, -13, 14.8, -156.0]
+          first_normalized: [-0.5, 100.0, -13, 14.8, -156.0]
+          normalized_vector: [-0.0026832016, 0.53664035, -0.06976324, 0.07942277, -0.8371589]
+          middle_normalized: [-0.0026832016, 0.53664035, -0.06976324, 0.07942277, -0.8371589]
+          end_normalized: [-0.0026832016, 0.53664035, -0.06976324, 0.07942277, -0.8371589]
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          name: rabbit.jpg
+          vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+          first_normalized: [0.5, 111.3, -13.0, 14.8, -156.0]
+          middle_normalized: [0.5, 111.3, -13.0, 14.8, -156.0]
+          normalized_vector: [0.0025954517, 0.5777475, -0.06748174, 0.076825365, -0.8097809]
+          end_normalized: [0.0025954517, 0.5777475, -0.06748174, 0.076825365, -0.8097809]
+
+  - do:
+      indices.refresh: {}
+
+---
+"kNN search only regular query":
+  - skip:
+      version: ' - 8.3.99'
+      reason: 'kNN added to search endpoint in 8.4'
+      features: close_to
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: normalized_vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: first_normalized
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: middle_normalized
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: end_normalized
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+# With a normalized query vector, all should be the same
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: vector
+            query_vector: [-0.0027626718, 0.4972809, -0.055253435, 0.081775084, -0.86195356]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: normalized_vector
+            query_vector: [-0.0027626718, 0.4972809, -0.055253435, 0.081775084, -0.86195356]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: first_normalized
+            query_vector: [-0.0027626718, 0.4972809, -0.055253435, 0.081775084, -0.86195356]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: middle_normalized
+            query_vector: [-0.0027626718, 0.4972809, -0.055253435, 0.081775084, -0.86195356]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: end_normalized
+            query_vector: [-0.0027626718, 0.4972809, -0.055253435, 0.081775084, -0.86195356]
+            k: 3
+            num_candidates: 3
+
+  - match: {hits.hits.0._id: "2"}
+  - close_to: {hits.hits.0._score: {value: 0.999405, error: 0.0001}}
+
+  - match: {hits.hits.1._id: "3"}
+  - close_to: {hits.hits.1._score: {value:  0.9976501, error: 0.0001}}

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -87,6 +87,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion NEW_SPARSE_VECTOR = def(8_500_001, Version.LUCENE_9_7_0);
     public static final IndexVersion SPARSE_VECTOR_IN_FIELD_NAMES_SUPPORT = def(8_500_002, Version.LUCENE_9_7_0);
     public static final IndexVersion UPGRADE_LUCENE_9_8 = def(8_500_003, Version.LUCENE_9_8_0);
+    public static final IndexVersion NORMALIZED_VECTOR_COSINE = def(8_500_004, Version.LUCENE_9_8_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,