From 220dca3864dcdb4f9dbf65a2ee7c9e47652a45d3 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 8 Oct 2025 16:48:48 -0400 Subject: [PATCH 01/29] chore(deps): update redis-py requirement to >=6.4.0 Update minimum redis-py version from 6.2.0 to 6.4.0 to enable support for SVS-VAMANA vector indexing algorithm. --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3fc4202f..c70dcb6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "numpy>=1.26.0,<3", "pyyaml>=5.4,<7.0", - "redis>=5.0,<7.0", + "redis>=6.4.0,<7.0", "pydantic>=2,<3", "tenacity>=8.2.2", "ml-dtypes>=0.4.0,<1.0.0", diff --git a/uv.lock b/uv.lock index 09691c5e..548a33aa 100644 --- a/uv.lock +++ b/uv.lock @@ -3581,14 +3581,14 @@ wheels = [ [[package]] name = "redis" -version = "6.2.0" +version = "6.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/9a/0551e01ba52b944f97480721656578c8a7c46b51b99d66814f85fe3a4f3e/redis-6.2.0.tar.gz", hash = "sha256:e821f129b75dde6cb99dd35e5c76e8c49512a5a0d8dfdc560b2fbd44b85ca977", size = 4639129, upload-time = "2025-05-28T05:01:18.91Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/67/e60968d3b0e077495a8fee89cf3f2373db98e528288a48f1ee44967f6e8c/redis-6.2.0-py3-none-any.whl", hash = "sha256:c8ddf316ee0aab65f04a11229e94a64b2618451dab7a67cb2f77eb799d872d5e", size = 278659, upload-time = "2025-05-28T05:01:16.955Z" }, + { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" }, ] [[package]] @@ -3680,7 +3680,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2,<3" }, { name = "python-ulid", specifier = ">=3.0.0" }, { name = "pyyaml", specifier = ">=5.4,<7.0" }, - { name = "redis", specifier = ">=5.0,<7.0" }, + { name = "redis", specifier = ">=6.4.0,<7.0" }, { name = "sentence-transformers", marker = "extra == 'sentence-transformers'", specifier = ">=3.4.0,<4" }, { name = "tenacity", specifier = ">=8.2.2" }, { name = "urllib3", marker = "extra == 'bedrock'", specifier = "<2.2.0" }, From ac14a99a0f824ed53b8ee933fad43afc92635edc Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 8 Oct 2025 16:49:41 -0400 Subject: [PATCH 02/29] feat(schema): add basic SVS-VAMANA vector index support without compression --- redisvl/schema/__init__.py | 4 ++ redisvl/schema/fields.py | 97 +++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/redisvl/schema/__init__.py b/redisvl/schema/__init__.py index c835ccd5..a652e01a 100644 --- a/redisvl/schema/__init__.py +++ b/redisvl/schema/__init__.py @@ -1,10 +1,12 @@ from redisvl.schema.fields import ( BaseField, + CompressionType, FieldTypes, FlatVectorField, GeoField, HNSWVectorField, NumericField, + SVSVectorField, TagField, TextField, VectorDataType, @@ -24,6 +26,7 @@ "VectorDistanceMetric", "VectorDataType", "VectorIndexAlgorithm", + "CompressionType", "BaseField", "TextField", "TagField", @@ -31,5 +34,6 @@ "GeoField", "FlatVectorField", "HNSWVectorField", + "SVSVectorField", "validate_object", ] diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index 7533e8c4..bf9e030b 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Dict, Literal, Optional, Tuple, Type, Union -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from redis.commands.search.field import Field as RedisField from redis.commands.search.field import GeoField as RedisGeoField from redis.commands.search.field import NumericField as RedisNumericField @@ -51,6 +51,17 @@ class VectorDataType(str, Enum): class VectorIndexAlgorithm(str, Enum): FLAT = "FLAT" HNSW = "HNSW" + SVS_VAMANA = "SVS-VAMANA" + + +class CompressionType(str, Enum): + """Vector compression types for SVS-VAMANA algorithm""" + LVQ4 = "LVQ4" + LVQ4x4 = "LVQ4x4" + LVQ4x8 = "LVQ4x8" + LVQ8 = "LVQ8" + LeanVec4x8 = "LeanVec4x8" + LeanVec8x8 = "LeanVec8x8" ### Field Attributes ### @@ -171,6 +182,61 @@ class HNSWVectorFieldAttributes(BaseVectorFieldAttributes): """Relative factor that sets the boundaries in which a range query may search for candidates""" + +class SVSVectorFieldAttributes(BaseVectorFieldAttributes): + """SVS-VAMANA vector field attributes with optional compression support""" + algorithm: Literal[VectorIndexAlgorithm.SVS_VAMANA] = VectorIndexAlgorithm.SVS_VAMANA + """The indexing algorithm for the vector field""" + + # SVS-VAMANA graph parameters + graph_max_degree: int = Field(default=40) + """Maximum degree of the Vamana graph (number of edges per node)""" + construction_window_size: int = Field(default=250) + """Size of the candidate list during graph construction""" + search_window_size: int = Field(default=20) + """Size of the candidate list during search""" + epsilon: float = Field(default=0.01) + """Relative factor for range query boundaries""" + + # SVS-VAMANA compression parameters (optional, to be implemented) + compression: Optional[CompressionType] = None + """Vector compression type (LVQ or LeanVec)""" + reduce: Optional[int] = None + """Reduced dimensionality for LeanVec compression (must be < dims)""" + training_threshold: Optional[int] = None + """Minimum number of vectors required before compression training""" + + @model_validator(mode='after') + def validate_svs_params(self): + """Validate SVS-VAMANA specific constraints""" + # Datatype validation: SVS only supports FLOAT16 and FLOAT32 + if self.datatype not in (VectorDataType.FLOAT16, VectorDataType.FLOAT32): + raise ValueError( + f"SVS-VAMANA only supports FLOAT16 and FLOAT32 datatypes. " + f"Got: {self.datatype}. " + f"Unsupported types: BFLOAT16, FLOAT64, INT8, UINT8." + ) + + # Reduce validation: must be less than dims + if self.reduce is not None: + if self.reduce >= self.dims: + raise ValueError( + f"reduce ({self.reduce}) must be less than dims ({self.dims})" + ) + # Phase C: Add warning for reduce without LeanVec + # if not self.compression or not self.compression.value.startswith("LeanVec"): + # logger.warning( + # "reduce parameter is recommended with LeanVec compression" + # ) + + # Phase C: Add warning for LeanVec without reduce + # if self.compression and self.compression.value.startswith("LeanVec") and not self.reduce: + # logger.warning( + # f"LeanVec compression selected without 'reduce'. " + # f"Consider setting reduce={self.dims//2} for better performance" + # ) + + return self ### Field Classes ### @@ -352,7 +418,7 @@ def as_redis_field(self) -> RedisField: class FlatVectorField(BaseField): - "Vector field with a FLAT index (brute force nearest neighbors search)" + """Vector field with a FLAT index (brute force nearest neighbors search)""" type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR attrs: FlatVectorFieldAttributes @@ -387,6 +453,32 @@ def as_redis_field(self) -> RedisField: return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name) +class SVSVectorField(BaseField): + """Vector field with an SVS-VAMANA index""" + type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR + attrs: SVSVectorFieldAttributes + + def as_redis_field(self) -> RedisField: + name, as_name = self._handle_names() + field_data=self.attrs.field_data + field_data.update( + { + "GRAPH_MAX_DEGREE": self.attrs.graph_max_degree, + "CONSTRUCTION_WINDOW_SIZE": self.attrs.construction_window_size, + "SEARCH_WINDOW_SIZE": self.attrs.search_window_size, + "EPSILON": self.attrs.epsilon, + } + ) + # Add compression parameters if specified + if self.attrs.compression is not None: + field_data["COMPRESSION"] = self.attrs.compression + if self.attrs.reduce is not None: + field_data["REDUCE"] = self.attrs.reduce + if self.attrs.training_threshold is not None: + field_data["TRAINING_THRESHOLD"] = self.attrs.training_threshold + return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name) + + FIELD_TYPE_MAP = { "tag": TagField, "text": TextField, @@ -397,6 +489,7 @@ def as_redis_field(self) -> RedisField: VECTOR_FIELD_TYPE_MAP = { "flat": FlatVectorField, "hnsw": HNSWVectorField, + "svs-vamana": SVSVectorField, } From b2d7abb6c55798f3b946fbe54e962b27e93e03b4 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 8 Oct 2025 17:11:31 -0400 Subject: [PATCH 03/29] chore(fields) formatting --- redisvl/schema/fields.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index bf9e030b..15a18c9f 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -56,6 +56,7 @@ class VectorIndexAlgorithm(str, Enum): class CompressionType(str, Enum): """Vector compression types for SVS-VAMANA algorithm""" + LVQ4 = "LVQ4" LVQ4x4 = "LVQ4x4" LVQ4x8 = "LVQ4x8" @@ -182,10 +183,12 @@ class HNSWVectorFieldAttributes(BaseVectorFieldAttributes): """Relative factor that sets the boundaries in which a range query may search for candidates""" - class SVSVectorFieldAttributes(BaseVectorFieldAttributes): """SVS-VAMANA vector field attributes with optional compression support""" - algorithm: Literal[VectorIndexAlgorithm.SVS_VAMANA] = VectorIndexAlgorithm.SVS_VAMANA + + algorithm: Literal[VectorIndexAlgorithm.SVS_VAMANA] = ( + VectorIndexAlgorithm.SVS_VAMANA + ) """The indexing algorithm for the vector field""" # SVS-VAMANA graph parameters @@ -206,7 +209,7 @@ class SVSVectorFieldAttributes(BaseVectorFieldAttributes): training_threshold: Optional[int] = None """Minimum number of vectors required before compression training""" - @model_validator(mode='after') + @model_validator(mode="after") def validate_svs_params(self): """Validate SVS-VAMANA specific constraints""" # Datatype validation: SVS only supports FLOAT16 and FLOAT32 @@ -237,6 +240,8 @@ def validate_svs_params(self): # ) return self + + ### Field Classes ### @@ -455,12 +460,13 @@ def as_redis_field(self) -> RedisField: class SVSVectorField(BaseField): """Vector field with an SVS-VAMANA index""" + type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR attrs: SVSVectorFieldAttributes def as_redis_field(self) -> RedisField: name, as_name = self._handle_names() - field_data=self.attrs.field_data + field_data = self.attrs.field_data field_data.update( { "GRAPH_MAX_DEGREE": self.attrs.graph_max_degree, From 3e20983fb18d8ff233622fa197f0fd55e4c5d12f Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Thu, 9 Oct 2025 17:40:32 -0400 Subject: [PATCH 04/29] Add more validation after testing for lvq (no compression with lvg4) --- redisvl/schema/fields.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index 15a18c9f..f92e1690 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -220,17 +220,26 @@ def validate_svs_params(self): f"Unsupported types: BFLOAT16, FLOAT64, INT8, UINT8." ) - # Reduce validation: must be less than dims + # Reduce validation: must be less than dims and only valid with LeanVec if self.reduce is not None: if self.reduce >= self.dims: raise ValueError( f"reduce ({self.reduce}) must be less than dims ({self.dims})" ) - # Phase C: Add warning for reduce without LeanVec - # if not self.compression or not self.compression.value.startswith("LeanVec"): - # logger.warning( - # "reduce parameter is recommended with LeanVec compression" - # ) + + # Validate that reduce is only used with LeanVec compression + if self.compression is None: + raise ValueError( + "reduce parameter requires compression to be set. " + "Use LeanVec4x8 or LeanVec8x8 compression with reduce." + ) + + if not self.compression.value.startswith("LeanVec"): + raise ValueError( + f"reduce parameter is only supported with LeanVec compression types. " + f"Got compression={self.compression.value}. " + f"Either use LeanVec4x8/LeanVec8x8 or remove the reduce parameter." + ) # Phase C: Add warning for LeanVec without reduce # if self.compression and self.compression.value.startswith("LeanVec") and not self.reduce: From 5ee9761381e04a94015c5cc3d3425b83e2622b6b Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Thu, 9 Oct 2025 17:40:47 -0400 Subject: [PATCH 05/29] Update schema docs with SVS --- docs/api/schema.rst | 62 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 7f6ae174..7847f61f 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -97,16 +97,64 @@ Each field type supports specific attributes that customize its behavior. Below **Common Vector Field Attributes**: - `dims`: Dimensionality of the vector. -- `algorithm`: Indexing algorithm (`flat` or `hnsw`). -- `datatype`: Float datatype of the vector (`bfloat16`, `float16`, `float32`, `float64`). +- `algorithm`: Indexing algorithm (`flat`, `hnsw`, or `svs-vamana`). +- `datatype`: Float datatype of the vector (`bfloat16`, `float16`, `float32`, `float64`). Note: SVS-VAMANA only supports `float16` and `float32`. - `distance_metric`: Metric for measuring query relevance (`COSINE`, `L2`, `IP`). +- `initial_cap`: Initial capacity for the index (optional). +- `index_missing`: When True, allows searching for documents missing this field (optional). + +**FLAT Vector Field Specific Attributes**: + +- `block_size`: Block size for the FLAT index (optional). **HNSW Vector Field Specific Attributes**: -- `m`: Max outgoing edges per node in each layer. -- `ef_construction`: Max edge candidates during build time. -- `ef_runtime`: Max top candidates during search. -- `epsilon`: Range search boundary factor. +- `m`: Max outgoing edges per node in each layer (default: 16). +- `ef_construction`: Max edge candidates during build time (default: 200). +- `ef_runtime`: Max top candidates during search (default: 10). +- `epsilon`: Range search boundary factor (default: 0.01). + +**SVS-VAMANA Vector Field Specific Attributes**: + +SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. + +- `graph_max_degree`: Maximum degree of the Vamana graph, i.e., the number of edges per node (default: 40). +- `construction_window_size`: Size of the candidate list during graph construction. Higher values yield better quality graphs but increase construction time (default: 250). +- `search_window_size`: Size of the candidate list during search. Higher values increase accuracy but also increase search latency (default: 20). +- `epsilon`: Relative factor for range query boundaries (default: 0.01). +- `compression`: Optional vector compression algorithm. Supported types: + + - `LVQ4`: 4-bit Learned Vector Quantization + - `LVQ4x4`: 4-bit LVQ with 4x compression + - `LVQ4x8`: 4-bit LVQ with 8x compression + - `LVQ8`: 8-bit Learned Vector Quantization + - `LeanVec4x8`: 4-bit LeanVec with 8x compression and dimensionality reduction + - `LeanVec8x8`: 8-bit LeanVec with 8x compression and dimensionality reduction + +- `reduce`: Reduced dimensionality for LeanVec compression. Must be less than `dims`. Only valid with `LeanVec4x8` or `LeanVec8x8` compression types. Lowering this value can speed up search and reduce memory usage (optional). +- `training_threshold`: Minimum number of vectors required before compression training begins. Must be less than 100 * 1024 (default: 10 * 1024). + +**SVS-VAMANA Example**: + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: svs-vamana + dims: 768 + distance_metric: cosine + datatype: float32 + graph_max_degree: 64 + construction_window_size: 500 + search_window_size: 40 + compression: LeanVec4x8 + reduce: 384 + training_threshold: 1000 Note: - See fully documented Redis-supported fields and options here: https://redis.io/commands/ft.create/ \ No newline at end of file + - SVS-VAMANA requires Redis >= 8.2 with RediSearch >= 2.8.10. + - SVS-VAMANA only supports `float16` and `float32` datatypes. + - The `reduce` parameter is only valid with LeanVec compression types (`LeanVec4x8` or `LeanVec8x8`). + - Intel's proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. + - See fully documented Redis-supported fields and options here: https://redis.io/commands/ft.create/ \ No newline at end of file From 19f00f41ed9d010af0328caf6a3b217c785a58fb Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Thu, 9 Oct 2025 17:54:32 -0400 Subject: [PATCH 06/29] Unit tests for basic SVS Vamana addition --- tests/unit/test_fields.py | 214 +++++++++++++++++++++++++ tests/unit/test_validation.py | 293 ++++++++++++++++++++++++++++++++++ 2 files changed, 507 insertions(+) diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index bd8b41ac..6f5f706d 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -11,6 +11,7 @@ GeoField, HNSWVectorField, NumericField, + SVSVectorField, TagField, TextField, ) @@ -72,6 +73,24 @@ def create_hnsw_vector_field(**kwargs): return HNSWVectorField(**defaults) +def create_svs_vector_field(**kwargs): + defaults = { + "name": "example_svsvectorfield", + "attrs": { + "dims": 128, + "algorithm": "SVS-VAMANA", + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 40, + "construction_window_size": 250, + "search_window_size": 20, + "epsilon": 0.01, + }, + } + defaults["attrs"].update(kwargs) + return SVSVectorField(**defaults) + + # Tests for field schema creation and validation @pytest.mark.parametrize( "schema_func,field_class", @@ -422,3 +441,198 @@ def test_field_factory_with_new_attributes(): ) assert isinstance(vector_field, FlatVectorField) assert vector_field.attrs.index_missing == True + + +# ==================== SVS-VAMANA TESTS ==================== + + +def test_svs_vector_field_creation(): + """Test basic SVS-VAMANA vector field creation.""" + svs_field = create_svs_vector_field() + assert svs_field.name == "example_svsvectorfield" + assert svs_field.attrs.algorithm == "SVS-VAMANA" + assert svs_field.attrs.dims == 128 + assert svs_field.attrs.datatype.value == "FLOAT32" + assert svs_field.attrs.distance_metric.value == "COSINE" + assert svs_field.attrs.graph_max_degree == 40 + assert svs_field.attrs.construction_window_size == 250 + assert svs_field.attrs.search_window_size == 20 + assert svs_field.attrs.epsilon == 0.01 + + +def test_svs_vector_field_as_redis_field(): + """Test SVS-VAMANA field conversion to Redis field.""" + svs_field = create_svs_vector_field() + redis_field = svs_field.as_redis_field() + + assert isinstance(redis_field, RedisVectorField) + assert redis_field.name == "example_svsvectorfield" + + # Check that SVS-VAMANA specific parameters are in args + assert "GRAPH_MAX_DEGREE" in redis_field.args + assert "CONSTRUCTION_WINDOW_SIZE" in redis_field.args + assert "SEARCH_WINDOW_SIZE" in redis_field.args + assert "EPSILON" in redis_field.args + + +def test_svs_vector_field_default_params(): + """Test SVS-VAMANA field with default parameters.""" + svs_field = SVSVectorField( + name="test_vector", + attrs={ + "dims": 768, + "algorithm": "SVS-VAMANA", + "datatype": "float32", + "distance_metric": "cosine", + }, + ) + + # Check defaults are applied + assert svs_field.attrs.graph_max_degree == 40 + assert svs_field.attrs.construction_window_size == 250 + assert svs_field.attrs.search_window_size == 20 + assert svs_field.attrs.epsilon == 0.01 + assert svs_field.attrs.compression is None + assert svs_field.attrs.reduce is None + assert svs_field.attrs.training_threshold is None + + +def test_svs_vector_field_with_custom_graph_params(): + """Test SVS-VAMANA field with custom graph parameters.""" + svs_field = create_svs_vector_field( + graph_max_degree=64, + construction_window_size=500, + search_window_size=40, + epsilon=0.02, + ) + + redis_field = svs_field.as_redis_field() + + # Verify custom parameters are set + assert redis_field.args[redis_field.args.index("GRAPH_MAX_DEGREE") + 1] == 64 + assert ( + redis_field.args[redis_field.args.index("CONSTRUCTION_WINDOW_SIZE") + 1] == 500 + ) + assert redis_field.args[redis_field.args.index("SEARCH_WINDOW_SIZE") + 1] == 40 + assert redis_field.args[redis_field.args.index("EPSILON") + 1] == 0.02 + + +def test_svs_vector_field_with_lvq4_compression(): + """Test SVS-VAMANA field with LVQ4 compression.""" + svs_field = create_svs_vector_field(compression="LVQ4") + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LVQ4" + + +def test_svs_vector_field_with_lvq8_compression(): + """Test SVS-VAMANA field with LVQ8 compression.""" + svs_field = create_svs_vector_field(compression="LVQ8") + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LVQ8" + + +def test_svs_vector_field_with_leanvec_compression(): + """Test SVS-VAMANA field with LeanVec4x8 compression.""" + svs_field = create_svs_vector_field(compression="LeanVec4x8") + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec4x8" + + +def test_svs_vector_field_with_leanvec_and_reduce(): + """Test SVS-VAMANA field with LeanVec compression and reduce parameter.""" + svs_field = create_svs_vector_field(dims=768, compression="LeanVec4x8", reduce=384) + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec4x8" + assert "REDUCE" in redis_field.args + assert redis_field.args[redis_field.args.index("REDUCE") + 1] == 384 + + +def test_svs_vector_field_with_training_threshold(): + """Test SVS-VAMANA field with training_threshold parameter.""" + svs_field = create_svs_vector_field(compression="LVQ4", training_threshold=10000) + redis_field = svs_field.as_redis_field() + + assert "TRAINING_THRESHOLD" in redis_field.args + assert redis_field.args[redis_field.args.index("TRAINING_THRESHOLD") + 1] == 10000 + + +def test_svs_vector_field_reduce_with_lvq4_raises_error(): + """Test that reduce parameter with LVQ4 compression raises ValueError.""" + with pytest.raises( + ValueError, match="reduce parameter is only supported with LeanVec" + ): + create_svs_vector_field(dims=768, compression="LVQ4", reduce=384) + + +def test_svs_vector_field_reduce_with_lvq8_raises_error(): + """Test that reduce parameter with LVQ8 compression raises ValueError.""" + with pytest.raises( + ValueError, match="reduce parameter is only supported with LeanVec" + ): + create_svs_vector_field(dims=768, compression="LVQ8", reduce=384) + + +def test_svs_vector_field_reduce_without_compression_raises_error(): + """Test that reduce parameter without compression raises ValueError.""" + with pytest.raises(ValueError, match="reduce parameter requires compression"): + create_svs_vector_field(dims=768, reduce=384) + + +def test_svs_vector_field_reduce_greater_than_dims_raises_error(): + """Test that reduce >= dims raises ValueError.""" + with pytest.raises(ValueError, match="reduce.*must be less than dims"): + create_svs_vector_field(dims=768, compression="LeanVec4x8", reduce=768) + + +def test_svs_vector_field_reduce_equal_to_dims_raises_error(): + """Test that reduce == dims raises ValueError.""" + with pytest.raises(ValueError, match="reduce.*must be less than dims"): + create_svs_vector_field(dims=768, compression="LeanVec4x8", reduce=768) + + +def test_svs_vector_field_invalid_datatype_raises_error(): + """Test that invalid datatype (not float16/float32) raises ValueError.""" + with pytest.raises(Exception, match="SVS-VAMANA only supports FLOAT16 and FLOAT32"): + create_svs_vector_field(datatype="float64") + + +def test_svs_vector_field_float16_datatype(): + """Test SVS-VAMANA field with float16 datatype.""" + svs_field = create_svs_vector_field(datatype="float16") + redis_field = svs_field.as_redis_field() + + assert "TYPE" in redis_field.args + assert redis_field.args[redis_field.args.index("TYPE") + 1] == "FLOAT16" + + +def test_svs_vector_field_all_compression_types(): + """Test all valid compression types for SVS-VAMANA.""" + compression_types = ["LVQ4", "LVQ4x4", "LVQ4x8", "LVQ8", "LeanVec4x8", "LeanVec8x8"] + + for compression in compression_types: + svs_field = create_svs_vector_field(compression=compression) + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert ( + redis_field.args[redis_field.args.index("COMPRESSION") + 1] == compression + ) + + +def test_svs_vector_field_leanvec8x8_with_reduce(): + """Test SVS-VAMANA field with LeanVec8x8 compression and reduce.""" + svs_field = create_svs_vector_field(dims=1024, compression="LeanVec8x8", reduce=512) + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec8x8" + assert "REDUCE" in redis_field.args + assert redis_field.args[redis_field.args.index("REDUCE") + 1] == 512 diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index 68933938..3c843040 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -690,3 +690,296 @@ def test_explicit_none_fields_excluded(self, sample_hash_schema): assert "title" in validated assert "rating" not in validated assert "location" not in validated + + +# -------------------- SVS-VAMANA VALIDATION TESTS -------------------- + + +class TestSVSVamanaValidation: + """Tests for SVS-VAMANA specific validation rules.""" + + def test_svs_vamana_with_valid_float32(self): + """Test SVS-VAMANA field with valid float32 datatype.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float32", + "distance_metric": "cosine", + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + assert len(schema.fields) == 1 + + def test_svs_vamana_with_valid_float16(self): + """Test SVS-VAMANA field with valid float16 datatype.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float16", + "distance_metric": "cosine", + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + + def test_svs_vamana_with_invalid_float64(self): + """Test SVS-VAMANA field rejects float64 datatype.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float64", + "distance_metric": "cosine", + }, + } + ], + } + # Should raise validation error + with pytest.raises( + Exception, match="SVS-VAMANA only supports FLOAT16 and FLOAT32" + ): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_with_leanvec_and_reduce(self): + """Test SVS-VAMANA with LeanVec compression and reduce parameter.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 384, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + + def test_svs_vamana_reduce_with_lvq4_raises_error(self): + """Test SVS-VAMANA rejects reduce parameter with LVQ4 compression.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LVQ4", + "reduce": 384, + }, + } + ], + } + # Should raise validation error + with pytest.raises( + Exception, match="reduce parameter is only supported with LeanVec" + ): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_reduce_without_compression_raises_error(self): + """Test SVS-VAMANA rejects reduce parameter without compression.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "reduce": 384, + }, + } + ], + } + # Should raise validation error + with pytest.raises(Exception, match="reduce parameter requires compression"): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_reduce_greater_than_dims_raises_error(self): + """Test SVS-VAMANA rejects reduce >= dims.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 768, + }, + } + ], + } + # Should raise validation error + with pytest.raises(Exception, match="reduce.*must be less than dims"): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_with_all_compression_types(self): + """Test SVS-VAMANA with all valid compression types.""" + compression_types = [ + "LVQ4", + "LVQ4x4", + "LVQ4x8", + "LVQ8", + "LeanVec4x8", + "LeanVec8x8", + ] + + for compression in compression_types: + schema_dict = { + "index": { + "name": f"test-svs-{compression}", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float32", + "distance_metric": "cosine", + "compression": compression, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + + def test_svs_vamana_with_graph_parameters(self): + """Test SVS-VAMANA with custom graph parameters.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 64, + "construction_window_size": 500, + "search_window_size": 40, + "epsilon": 0.02, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + vector_field = schema.fields["embedding"] + assert vector_field.attrs.graph_max_degree == 64 + assert vector_field.attrs.construction_window_size == 500 + assert vector_field.attrs.search_window_size == 40 + assert vector_field.attrs.epsilon == 0.02 + + def test_svs_vamana_with_training_threshold(self): + """Test SVS-VAMANA with training_threshold parameter.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LVQ4", + "training_threshold": 10000, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + vector_field = schema.fields["embedding"] + assert vector_field.attrs.training_threshold == 10000 From 6c00ea279301675d5e00ffb936c769eb21bde98b Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 13:26:21 -0400 Subject: [PATCH 07/29] Updates: svs-vamana version capability validation --- redisvl/exceptions.py | 26 ++++++ redisvl/index/index.py | 42 ++++++++++ redisvl/redis/connection.py | 153 +++++++++++++++++++++++++++++++++++- redisvl/redis/constants.py | 9 +++ 4 files changed, 229 insertions(+), 1 deletion(-) diff --git a/redisvl/exceptions.py b/redisvl/exceptions.py index 2d8b5bf4..596a4cb5 100644 --- a/redisvl/exceptions.py +++ b/redisvl/exceptions.py @@ -31,3 +31,29 @@ class QueryValidationError(RedisVLError): """Error when validating a query.""" pass + + +class RedisModuleVersionError(RedisVLError): + """Error when Redis or module versions are incompatible with requested features.""" + + @classmethod + def for_svs_vamana(cls, capabilities, min_redis_version: str): + """Create error for unsupported SVS-VAMANA. + + Args: + capabilities: VectorSupport instance with version info + min_redis_version: Minimum required Redis version + + Returns: + RedisModuleVersionError with formatted message + """ + message = ( + f"SVS-VAMANA requires Redis >= {min_redis_version} with RediSearch >= 2.8.10. " + f"Current: Redis {capabilities.redis_version}, " + f"RediSearch {capabilities.search_version_str}, " + f"SearchLight {capabilities.searchlight_version_str}. " + f"Options: 1) Upgrade Redis Stack, " + f"2) Use algorithm='hnsw' or 'flat', " + f"3) Remove compression parameters" + ) + return cls(message) diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 4d7ee9b5..5405bc25 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -67,6 +67,7 @@ from redisvl.exceptions import ( QueryValidationError, + RedisModuleVersionError, RedisSearchError, RedisVLError, SchemaValidationError, @@ -82,11 +83,15 @@ from redisvl.query.filter import FilterExpression from redisvl.redis.connection import ( RedisConnectionFactory, + check_vector_capabilities, + check_vector_capabilities_async, convert_index_info_to_schema, ) +from redisvl.redis.constants import SVS_MIN_REDIS_VERSION from redisvl.schema import IndexSchema, StorageType from redisvl.schema.fields import ( VECTOR_NORM_MAP, + SVSVectorField, VectorDistanceMetric, VectorIndexAlgorithm, ) @@ -228,6 +233,12 @@ def _storage(self) -> BaseStorage: index_schema=self.schema ) + def _uses_svs_vamana(self) -> bool: + """Check if schema contains any SVS-VAMANA vector fields.""" + return any( + isinstance(field, SVSVectorField) for field in self.schema.fields.values() + ) + def _validate_query(self, query: BaseQuery) -> None: """Validate a query.""" if isinstance(query, VectorQuery): @@ -535,6 +546,17 @@ def set_client(self, redis_client: SyncRedisClient, **kwargs): self.__redis_client = redis_client return self + def _check_svs_support(self) -> None: + """Validate SVS-VAMANA support. + + Raises: + RedisModuleVersionError: If SVS-VAMANA requirements are not met. + """ + caps = check_vector_capabilities(self._redis_client) + + if not caps.svs_vamana_supported: + raise RedisModuleVersionError.for_svs_vamana(caps, SVS_MIN_REDIS_VERSION) + def create(self, overwrite: bool = False, drop: bool = False) -> None: """Create an index in Redis with the current schema and properties. @@ -566,6 +588,10 @@ def create(self, overwrite: bool = False, drop: bool = False) -> None: if not isinstance(overwrite, bool): raise TypeError("overwrite must be of type bool") + # Check if schema uses SVS-VAMANA and validate Redis capabilities + if self._uses_svs_vamana(): + self._check_svs_support() + if self.exists(): if not overwrite: logger.info("Index already exists, not overwriting.") @@ -1302,6 +1328,18 @@ async def _info(name: str, redis_client: AsyncRedisClient) -> Dict[str, Any]: f"Error while fetching {name} index info: {str(e)}" ) from e + async def _check_svs_support_async(self) -> None: + """Validate SVS-VAMANA support. + + Raises: + RedisModuleVersionError: If SVS-VAMANA requirements are not met. + """ + client = await self._get_client() + caps = await check_vector_capabilities_async(client) + + if not caps.svs_vamana_supported: + raise RedisModuleVersionError.for_svs_vamana(caps, SVS_MIN_REDIS_VERSION) + async def create(self, overwrite: bool = False, drop: bool = False) -> None: """Asynchronously create an index in Redis with the current schema and properties. @@ -1335,6 +1373,10 @@ async def create(self, overwrite: bool = False, drop: bool = False) -> None: if not isinstance(overwrite, bool): raise TypeError("overwrite must be of type bool") + # Check if schema uses SVS-VAMANA and validate Redis capabilities + if self._uses_svs_vamana(): + await self._check_svs_support_async() + if await self.exists(): if not overwrite: logger.info("Index already exists, not overwriting.") diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 0dce70c2..a3fe7c1b 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -1,4 +1,5 @@ import os +from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, overload from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from warnings import warn @@ -15,7 +16,11 @@ from redis.sentinel import Sentinel from redisvl import __version__ -from redisvl.redis.constants import REDIS_URL_ENV_VAR +from redisvl.redis.constants import ( + REDIS_URL_ENV_VAR, + SVS_MIN_REDIS_VERSION, + SVS_REQUIRED_MODULES, +) from redisvl.redis.utils import convert_bytes, is_cluster_url from redisvl.types import AsyncRedisClient, RedisClient, SyncRedisClient from redisvl.utils.utils import deprecated_function @@ -101,6 +106,152 @@ def unpack_redis_modules(module_list: List[Dict[str, Any]]) -> Dict[str, Any]: return {module["name"]: module["ver"] for module in module_list} +@dataclass +class VectorSupport: + """Redis server capabilities for vector operations.""" + + redis_version: str + search_version: int + searchlight_version: int + svs_vamana_supported: bool + + @property + def search_version_str(self) -> str: + """Format search module version as string.""" + return format_module_version(self.search_version) + + @property + def searchlight_version_str(self) -> str: + """Format searchlight module version as string.""" + return format_module_version(self.searchlight_version) + + +def format_module_version(version: int) -> str: + """Format module version from integer (20810) to string (2.8.10).""" + if version == 0: + return "not installed" + major = version // 10000 + minor = (version % 10000) // 100 + patch = version % 100 + return f"{major}.{minor}.{patch}" + + +def check_vector_capabilities(client: SyncRedisClient) -> VectorSupport: + """Check Redis server capabilities for vector features. + + Args: + client: Sync Redis client instance + + Returns: + VectorSupport with version info and supported features + """ + info = client.info("server") + redis_version = info.get("redis_version", "0.0.0") + + modules = RedisConnectionFactory.get_modules(client) + search_ver = modules.get("search", 0) + searchlight_ver = modules.get("searchlight", 0) + + # Check if SVS-VAMANA requirements are met + redis_ok = compare_versions(redis_version, SVS_MIN_REDIS_VERSION) + modules_ok = search_ver >= 20810 or searchlight_ver >= 20810 + + return VectorSupport( + redis_version=redis_version, + search_version=search_ver, + searchlight_version=searchlight_ver, + svs_vamana_supported=redis_ok and modules_ok, + ) + + +async def check_vector_capabilities_async(client: AsyncRedisClient) -> VectorSupport: + """Async version of check_vector_capabilities. + + Args: + client: Async Redis client instance + + Returns: + VectorSupport with version info and supported features + """ + info = await client.info("server") + redis_version = info.get("redis_version", "0.0.0") + + modules = await RedisConnectionFactory.get_modules_async(client) + search_ver = modules.get("search", 0) + searchlight_ver = modules.get("searchlight", 0) + + # Check if SVS-VAMANA requirements are met + redis_ok = compare_versions(redis_version, SVS_MIN_REDIS_VERSION) + modules_ok = search_ver >= 20810 or searchlight_ver >= 20810 + + return VectorSupport( + redis_version=redis_version, + search_version=search_ver, + searchlight_version=searchlight_ver, + svs_vamana_supported=redis_ok and modules_ok, + ) + + +def supports_svs_vamana(client: SyncRedisClient) -> bool: + """Check if Redis server supports SVS-VAMANA algorithm. + + SVS-Vamana requires: + - Redis version >= 8.2.0 + - RediSearch version >= 2.8.10 (20810) + + Args: + client: Sync Redis client instance + + Returns: + bool: True if SVS-VAMANA is supported, False otherwise + """ + info = client.info() + redis_version = info.get("redis_version", "0.0.0") + if not compare_versions(redis_version, SVS_MIN_REDIS_VERSION): + return False + + # Check module versions + modules = unpack_redis_modules(convert_bytes(client.module_list())) + for module in SVS_REQUIRED_MODULES: + module_name = module["name"] + required_version = module["ver"] + if module_name not in modules: + return False + if modules[module_name] < required_version: + return False + return True + + +async def async_supports_svs_vamana(client: AsyncRedisClient) -> bool: + """Check if Redis server supports SVS-VAMANA algorithm asynchronously. + + SVS-Vamana requires: + - Redis version >= 8.2.0 + - RediSearch version >= 2.8.10 (20810) + + Args: + client: Sync Redis client instance + + Returns: + bool: True if SVS-VAMANA is supported, False otherwise + """ + info = await client.info() + redis_version = info.get("redis_version", "0.0.0") + if not compare_versions(redis_version, SVS_MIN_REDIS_VERSION): + return False + + # Check module versions + modules = unpack_redis_modules(convert_bytes(await client.module_list())) + for module in SVS_REQUIRED_MODULES: + module_name = module["name"] + required_version = module["ver"] + if module_name not in modules: + return False + if modules[module_name] < required_version: + return False + return True + + def get_address_from_env() -> str: """Get Redis URL from environment variable.""" redis_url = os.getenv(REDIS_URL_ENV_VAR) diff --git a/redisvl/redis/constants.py b/redisvl/redis/constants.py index 434a2ff5..0a0737ed 100644 --- a/redisvl/redis/constants.py +++ b/redisvl/redis/constants.py @@ -4,6 +4,15 @@ {"name": "searchlight", "ver": 20600}, ] +# SVS-VAMANA requires Redis 8.2+ with RediSearch 2.8.10+ +SVS_REQUIRED_MODULES = [ + {"name": "search", "ver": 20810}, # RediSearch 2.8.10+ + {"name": "searchlight", "ver": 20810}, +] + +# Minimum Redis version for SVS-VAMANA +SVS_MIN_REDIS_VERSION = "8.2.0" + # default tag separator REDIS_TAG_SEPARATOR = "," From 6e8fc841780e0111d33a213972aeec327c38885d Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 13:26:31 -0400 Subject: [PATCH 08/29] Updates: svs-vamana version capability validation tests --- tests/unit/test_fields.py | 190 ++++++++++++++++++++ tests/unit/test_svs_capability_detection.py | 178 ++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 tests/unit/test_svs_capability_detection.py diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index 6f5f706d..0eb5c54e 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -1,3 +1,5 @@ +from unittest.mock import AsyncMock, Mock, patch + import pytest from redis.commands.search.field import GeoField as RedisGeoField from redis.commands.search.field import NumericField as RedisNumericField @@ -5,6 +7,9 @@ from redis.commands.search.field import TextField as RedisTextField from redis.commands.search.field import VectorField as RedisVectorField +from redisvl.exceptions import RedisModuleVersionError +from redisvl.index import AsyncSearchIndex, SearchIndex +from redisvl.schema import IndexSchema from redisvl.schema.fields import ( FieldFactory, FlatVectorField, @@ -636,3 +641,188 @@ def test_svs_vector_field_leanvec8x8_with_reduce(): assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec8x8" assert "REDUCE" in redis_field.args assert redis_field.args[redis_field.args.index("REDUCE") + 1] == 512 + + +# ==================== SVS-VAMANA INDEX VALIDATION TESTS ==================== + + +def test_uses_svs_vamana_true(): + """Test _uses_svs_vamana returns True for SVS schema.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = Mock() + + with patch.object(SearchIndex, "_redis_client", mock_client): + index = SearchIndex(schema=schema) + assert index._uses_svs_vamana() is True + + +def test_check_svs_support_raises_error(): + """Test _check_svs_support raises error when not supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "7.2.4"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20612, "searchlight": 20612} + + index = SearchIndex(schema=schema) + + # Mock the _redis_client property + with patch.object( + type(index), + "_redis_client", + new_callable=lambda: property(lambda self: mock_client), + ): + with pytest.raises(RedisModuleVersionError) as exc_info: + index._check_svs_support() + + error_msg = str(exc_info.value) + assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg + assert "Redis 7.2.4" in error_msg + + +def test_check_svs_support_passes(): + """Test _check_svs_support passes when supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + index = SearchIndex(schema=schema) + + # Mock the _redis_client property + with patch.object( + type(index), + "_redis_client", + new_callable=lambda: property(lambda self: mock_client), + ): + # Should not raise + index._check_svs_support() + + +@pytest.mark.asyncio +async def test_check_svs_support_async_raises_error(): + """Test _check_svs_support_async raises error when not supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = AsyncMock() + mock_client.info.return_value = {"redis_version": "7.2.4"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules_async" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20612, "searchlight": 20612} + + index = AsyncSearchIndex(schema=schema) + + with patch.object(index, "_get_client", return_value=mock_client): + with pytest.raises(RedisModuleVersionError) as exc_info: + await index._check_svs_support_async() + + error_msg = str(exc_info.value) + assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg + + +@pytest.mark.asyncio +async def test_check_svs_support_async_passes(): + """Test _check_svs_support_async passes when supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = AsyncMock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules_async" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + index = AsyncSearchIndex(schema=schema) + + with patch.object(index, "_get_client", return_value=mock_client): + # Should not raise + await index._check_svs_support_async() diff --git a/tests/unit/test_svs_capability_detection.py b/tests/unit/test_svs_capability_detection.py new file mode 100644 index 00000000..9a52db62 --- /dev/null +++ b/tests/unit/test_svs_capability_detection.py @@ -0,0 +1,178 @@ +""" +Unit tests for SVS-VAMANA capability detection. + +Tests the core functionality that determines if SVS-VAMANA vector indexing +is supported on the connected Redis instance. +""" + +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +from redisvl.exceptions import RedisModuleVersionError +from redisvl.redis.connection import ( + VectorSupport, + async_supports_svs_vamana, + check_vector_capabilities, + check_vector_capabilities_async, + compare_versions, + format_module_version, + supports_svs_vamana, +) + +# ============================================================================ +# Helper Function Tests +# ============================================================================ + + +def test_format_version_20810(): + """Test formatting version 20810 -> 2.8.10""" + assert format_module_version(20810) == "2.8.10" + + +def test_compare_greater_version(): + """Test version comparison: greater version returns True.""" + assert compare_versions("8.2.0", "8.1.0") is True + assert compare_versions("8.2.1", "8.2.0") is True + assert compare_versions("9.0.0", "8.2.0") is True + + +def test_compare_lesser_version(): + """Test version comparison: lesser version returns False.""" + assert compare_versions("7.2.4", "8.2.0") is False + assert compare_versions("8.1.9", "8.2.0") is False + assert compare_versions("8.2.0", "8.2.1") is False + + +# ============================================================================ +# check_vector_capabilities Tests +# ============================================================================ + + +def test_check_vector_capabilities_supported(): + """Test check_vector_capabilities when SVS is supported.""" + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + caps = check_vector_capabilities(mock_client) + + assert caps.redis_version == "8.2.0" + assert caps.search_version == 20810 + assert caps.searchlight_version == 20810 + assert caps.svs_vamana_supported is True + + +def test_check_vector_capabilities_old_redis(): + """Test check_vector_capabilities with old Redis version.""" + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "7.2.4"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + caps = check_vector_capabilities(mock_client) + + assert caps.redis_version == "7.2.4" + assert caps.svs_vamana_supported is False + + +def test_check_vector_capabilities_old_modules(): + """Test check_vector_capabilities with old module versions.""" + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20612, "searchlight": 20612} + + caps = check_vector_capabilities(mock_client) + + assert caps.search_version == 20612 + assert caps.searchlight_version == 20612 + assert caps.svs_vamana_supported is False + + +@pytest.mark.asyncio +async def test_check_vector_capabilities_async_supported(): + """Test check_vector_capabilities_async when SVS is supported.""" + mock_client = AsyncMock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules_async" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + caps = await check_vector_capabilities_async(mock_client) + + assert caps.redis_version == "8.2.0" + assert caps.search_version == 20810 + assert caps.svs_vamana_supported is True + + +# ============================================================================ +# supports_svs_vamana Tests +# ============================================================================ + + +def test_supports_svs_vamana_true(): + """Test supports_svs_vamana returns True when supported.""" + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + mock_client.module_list.return_value = [ + {"name": b"search", "ver": 20810}, + {"name": b"searchlight", "ver": 20810}, + ] + + assert supports_svs_vamana(mock_client) is True + + +def test_supports_svs_vamana_false_old_redis(): + """Test supports_svs_vamana returns False with old Redis.""" + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "7.2.4"} + mock_client.module_list.return_value = [ + {"name": b"search", "ver": 20810}, + ] + + assert supports_svs_vamana(mock_client) is False + + +def test_supports_svs_vamana_exception_handling(): + """Test supports_svs_vamana handles exceptions gracefully.""" + mock_client = Mock() + mock_client.info.side_effect = Exception("Connection error") + + assert supports_svs_vamana(mock_client) is False + + +# ============================================================================ +# Exception Tests +# ============================================================================ + + +def test_for_svs_vamana_error_message(): + """Test RedisModuleVersionError.for_svs_vamana creates proper exception.""" + caps = VectorSupport( + redis_version="7.2.4", + search_version=20612, + searchlight_version=20612, + svs_vamana_supported=False, + ) + + error = RedisModuleVersionError.for_svs_vamana(caps, "8.2.0") + + error_msg = str(error) + assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg + assert "RediSearch >= 2.8.10" in error_msg + assert "Redis 7.2.4" in error_msg + assert "RediSearch 2.6.12" in error_msg + assert "SearchLight 2.6.12" in error_msg From dcf62c572cf544f969bf521110b7f59e407bddf3 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 13:47:50 -0400 Subject: [PATCH 09/29] Lint passes --- redisvl/redis/connection.py | 69 ++------------------- tests/unit/test_fields.py | 6 -- tests/unit/test_svs_capability_detection.py | 52 ---------------- 3 files changed, 5 insertions(+), 122 deletions(-) diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index a3fe7c1b..aa3afeb8 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -145,8 +145,8 @@ def check_vector_capabilities(client: SyncRedisClient) -> VectorSupport: Returns: VectorSupport with version info and supported features """ - info = client.info("server") - redis_version = info.get("redis_version", "0.0.0") + info = client.info("server") # type: ignore[union-attr] + redis_version = info.get("redis_version", "0.0.0") # type: ignore[union-attr] modules = RedisConnectionFactory.get_modules(client) search_ver = modules.get("search", 0) @@ -173,8 +173,8 @@ async def check_vector_capabilities_async(client: AsyncRedisClient) -> VectorSup Returns: VectorSupport with version info and supported features """ - info = await client.info("server") - redis_version = info.get("redis_version", "0.0.0") + info = await client.info("server") # type: ignore[union-attr] + redis_version = info.get("redis_version", "0.0.0") # type: ignore[union-attr] modules = await RedisConnectionFactory.get_modules_async(client) search_ver = modules.get("search", 0) @@ -192,66 +192,6 @@ async def check_vector_capabilities_async(client: AsyncRedisClient) -> VectorSup ) -def supports_svs_vamana(client: SyncRedisClient) -> bool: - """Check if Redis server supports SVS-VAMANA algorithm. - - SVS-Vamana requires: - - Redis version >= 8.2.0 - - RediSearch version >= 2.8.10 (20810) - - Args: - client: Sync Redis client instance - - Returns: - bool: True if SVS-VAMANA is supported, False otherwise - """ - info = client.info() - redis_version = info.get("redis_version", "0.0.0") - if not compare_versions(redis_version, SVS_MIN_REDIS_VERSION): - return False - - # Check module versions - modules = unpack_redis_modules(convert_bytes(client.module_list())) - for module in SVS_REQUIRED_MODULES: - module_name = module["name"] - required_version = module["ver"] - if module_name not in modules: - return False - if modules[module_name] < required_version: - return False - return True - - -async def async_supports_svs_vamana(client: AsyncRedisClient) -> bool: - """Check if Redis server supports SVS-VAMANA algorithm asynchronously. - - SVS-Vamana requires: - - Redis version >= 8.2.0 - - RediSearch version >= 2.8.10 (20810) - - Args: - client: Sync Redis client instance - - Returns: - bool: True if SVS-VAMANA is supported, False otherwise - """ - info = await client.info() - redis_version = info.get("redis_version", "0.0.0") - if not compare_versions(redis_version, SVS_MIN_REDIS_VERSION): - return False - - # Check module versions - modules = unpack_redis_modules(convert_bytes(await client.module_list())) - for module in SVS_REQUIRED_MODULES: - module_name = module["name"] - required_version = module["ver"] - if module_name not in modules: - return False - if modules[module_name] < required_version: - return False - return True - - def get_address_from_env() -> str: """Get Redis URL from environment variable.""" redis_url = os.getenv(REDIS_URL_ENV_VAR) @@ -558,6 +498,7 @@ async def _get_aredis_connection( """ url = url or get_address_from_env() + client: AsyncRedisClient if url.startswith("redis+sentinel"): client = RedisConnectionFactory._redis_sentinel_client( url, AsyncRedis, **kwargs diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index 0eb5c54e..b6d41b69 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -448,9 +448,6 @@ def test_field_factory_with_new_attributes(): assert vector_field.attrs.index_missing == True -# ==================== SVS-VAMANA TESTS ==================== - - def test_svs_vector_field_creation(): """Test basic SVS-VAMANA vector field creation.""" svs_field = create_svs_vector_field() @@ -643,9 +640,6 @@ def test_svs_vector_field_leanvec8x8_with_reduce(): assert redis_field.args[redis_field.args.index("REDUCE") + 1] == 512 -# ==================== SVS-VAMANA INDEX VALIDATION TESTS ==================== - - def test_uses_svs_vamana_true(): """Test _uses_svs_vamana returns True for SVS schema.""" schema_dict = { diff --git a/tests/unit/test_svs_capability_detection.py b/tests/unit/test_svs_capability_detection.py index 9a52db62..0296fcb6 100644 --- a/tests/unit/test_svs_capability_detection.py +++ b/tests/unit/test_svs_capability_detection.py @@ -12,18 +12,12 @@ from redisvl.exceptions import RedisModuleVersionError from redisvl.redis.connection import ( VectorSupport, - async_supports_svs_vamana, check_vector_capabilities, check_vector_capabilities_async, compare_versions, format_module_version, - supports_svs_vamana, ) -# ============================================================================ -# Helper Function Tests -# ============================================================================ - def test_format_version_20810(): """Test formatting version 20810 -> 2.8.10""" @@ -44,11 +38,6 @@ def test_compare_lesser_version(): assert compare_versions("8.2.0", "8.2.1") is False -# ============================================================================ -# check_vector_capabilities Tests -# ============================================================================ - - def test_check_vector_capabilities_supported(): """Test check_vector_capabilities when SVS is supported.""" mock_client = Mock() @@ -118,47 +107,6 @@ async def test_check_vector_capabilities_async_supported(): assert caps.svs_vamana_supported is True -# ============================================================================ -# supports_svs_vamana Tests -# ============================================================================ - - -def test_supports_svs_vamana_true(): - """Test supports_svs_vamana returns True when supported.""" - mock_client = Mock() - mock_client.info.return_value = {"redis_version": "8.2.0"} - mock_client.module_list.return_value = [ - {"name": b"search", "ver": 20810}, - {"name": b"searchlight", "ver": 20810}, - ] - - assert supports_svs_vamana(mock_client) is True - - -def test_supports_svs_vamana_false_old_redis(): - """Test supports_svs_vamana returns False with old Redis.""" - mock_client = Mock() - mock_client.info.return_value = {"redis_version": "7.2.4"} - mock_client.module_list.return_value = [ - {"name": b"search", "ver": 20810}, - ] - - assert supports_svs_vamana(mock_client) is False - - -def test_supports_svs_vamana_exception_handling(): - """Test supports_svs_vamana handles exceptions gracefully.""" - mock_client = Mock() - mock_client.info.side_effect = Exception("Connection error") - - assert supports_svs_vamana(mock_client) is False - - -# ============================================================================ -# Exception Tests -# ============================================================================ - - def test_for_svs_vamana_error_message(): """Test RedisModuleVersionError.for_svs_vamana creates proper exception.""" caps = VectorSupport( From 2238c0364a407487244175eeaaa4de67b4a0240f Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 14:05:42 -0400 Subject: [PATCH 10/29] remove comment --- tests/unit/test_validation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index 3c843040..fed3d13a 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -692,9 +692,6 @@ def test_explicit_none_fields_excluded(self, sample_hash_schema): assert "location" not in validated -# -------------------- SVS-VAMANA VALIDATION TESTS -------------------- - - class TestSVSVamanaValidation: """Tests for SVS-VAMANA specific validation rules.""" From 35bae2af865441ec38c804ad1bf7baaff5a44a4e Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 20:09:40 -0400 Subject: [PATCH 11/29] Compression and validation wip --- redisvl/schema/fields.py | 31 ++- redisvl/utils/__init__.py | 3 + redisvl/utils/compression.py | 216 +++++++++++++++++++++ tests/unit/test_compression_advisor.py | 250 +++++++++++++++++++++++++ tests/unit/test_fields.py | 158 ++++++++++++++++ 5 files changed, 652 insertions(+), 6 deletions(-) create mode 100644 redisvl/utils/compression.py create mode 100644 tests/unit/test_compression_advisor.py diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index f92e1690..b2735479 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -16,8 +16,11 @@ from redis.commands.search.field import TextField as RedisTextField from redis.commands.search.field import VectorField as RedisVectorField +from redisvl.utils.log import get_logger from redisvl.utils.utils import norm_cosine_distance, norm_l2_distance +logger = get_logger(__name__) + VECTOR_NORM_MAP = { "COSINE": norm_cosine_distance, "L2": norm_l2_distance, @@ -241,12 +244,28 @@ def validate_svs_params(self): f"Either use LeanVec4x8/LeanVec8x8 or remove the reduce parameter." ) - # Phase C: Add warning for LeanVec without reduce - # if self.compression and self.compression.value.startswith("LeanVec") and not self.reduce: - # logger.warning( - # f"LeanVec compression selected without 'reduce'. " - # f"Consider setting reduce={self.dims//2} for better performance" - # ) + # LeanVec without reduce is not recommended + if ( + self.compression + and self.compression.value.startswith("LeanVec") + and not self.reduce + ): + logger.warning( + f"LeanVec compression selected without 'reduce'. " + f"Consider setting reduce={self.dims//2} for better performance" + ) + + if self.graph_max_degree and self.graph_max_degree < 32: + logger.warning( + f"graph_max_degree={self.graph_max_degree} is low. " + f"Consider values between 32-64 for better recall." + ) + + if self.search_window_size and self.search_window_size > 100: + logger.warning( + f"search_window_size={self.search_window_size} is high. " + f"This may impact query latency. Consider values between 20-50." + ) return self diff --git a/redisvl/utils/__init__.py b/redisvl/utils/__init__.py index e69de29b..639e6e2a 100644 --- a/redisvl/utils/__init__.py +++ b/redisvl/utils/__init__.py @@ -0,0 +1,3 @@ +from redisvl.utils.compression import CompressionAdvisor + +__all__ = ["CompressionAdvisor"] diff --git a/redisvl/utils/compression.py b/redisvl/utils/compression.py new file mode 100644 index 00000000..fcee3a22 --- /dev/null +++ b/redisvl/utils/compression.py @@ -0,0 +1,216 @@ +"""SVS-VAMANA compression configuration utilities.""" + +from typing import Literal, Optional, TypedDict, cast + + +class SVSConfig(TypedDict, total=False): + """SVS-VAMANA configuration dictionary. + + Attributes: + algorithm: Always "svs-vamana" + datatype: Vector datatype (float16, float32) + compression: Compression type (LVQ4, LeanVec4x8, etc.) + reduce: Reduced dimensionality (only for LeanVec) + graph_max_degree: Max edges per node + construction_window_size: Build-time candidates + search_window_size: Query-time candidates + """ + + algorithm: Literal["svs-vamana"] + datatype: str + compression: str + reduce: int # only for LeanVec + graph_max_degree: int + construction_window_size: int + search_window_size: int + + +class CompressionAdvisor: + """Helper to recommend compression settings based on vector characteristics. + + This class provides utilities to: + - Recommend optimal SVS-VAMANA configurations based on vector dimensions and priorities + - Estimate memory savings from compression and dimensionality reduction + + Examples: + >>> # Get recommendations for high-dimensional vectors + >>> config = CompressionAdvisor.recommend(dims=1536, priority="balanced") + >>> config["compression"] + 'LeanVec4x8' + >>> config["reduce"] + 768 + + >>> # Estimate memory savings + >>> savings = CompressionAdvisor.estimate_memory_savings( + ... compression="LeanVec4x8", + ... dims=1536, + ... reduce=768 + ... ) + >>> savings + 81.2 + """ + + # Dimension thresholds + HIGH_DIM_THRESHOLD = 1024 + + # Compression bit rates (bits per dimension) + COMPRESSION_BITS = { + "LVQ4": 4, + "LVQ4x4": 8, + "LVQ4x8": 12, + "LVQ8": 8, + "LeanVec4x8": 12, + "LeanVec8x8": 16, + } + + @staticmethod + def recommend( + dims: int, + priority: Literal["speed", "memory", "balanced"] = "balanced", + datatype: Optional[str] = None, + ) -> SVSConfig: + """Recommend compression settings based on dimensions and priorities. + + Args: + dims: Vector dimensionality (must be > 0) + priority: Optimization priority: + - "memory": Maximize memory savings + - "speed": Optimize for query speed + - "balanced": Balance between memory and speed + datatype: Override datatype (default: float16 for high-dim, float32 for low-dim) + + Returns: + dict: Complete SVS-VAMANA configuration including: + - algorithm: "svs-vamana" + - datatype: Recommended datatype + - compression: Compression type + - reduce: Dimensionality reduction (for LeanVec only) + - graph_max_degree: Graph connectivity + - construction_window_size: Build-time candidates + - search_window_size: Query-time candidates + + Raises: + ValueError: If dims <= 0 + + Examples: + >>> # High-dimensional embeddings (e.g., OpenAI ada-002) + >>> config = CompressionAdvisor.recommend(dims=1536, priority="memory") + >>> config["compression"] + 'LeanVec4x8' + >>> config["reduce"] + 768 + + >>> # Lower-dimensional embeddings + >>> config = CompressionAdvisor.recommend(dims=384, priority="speed") + >>> config["compression"] + 'LVQ4x8' + """ + if dims <= 0: + raise ValueError(f"dims must be positive, got {dims}") + + # High-dimensional vectors (>= 1024) - use LeanVec + if dims >= CompressionAdvisor.HIGH_DIM_THRESHOLD: + base = { + "algorithm": "svs-vamana", + "datatype": datatype or "float16", + "graph_max_degree": 64, + "construction_window_size": 300, + } + + if priority == "memory": + return cast( + SVSConfig, + { + **base, + "compression": "LeanVec4x8", + "reduce": dims // 2, + "search_window_size": 20, + }, + ) + elif priority == "speed": + return cast( + SVSConfig, + { + **base, + "compression": "LeanVec4x8", + "reduce": max(256, dims // 4), + "search_window_size": 40, + }, + ) + else: # balanced + return cast( + SVSConfig, + { + **base, + "compression": "LeanVec4x8", + "reduce": dims // 2, + "search_window_size": 30, + }, + ) + + # Lower-dimensional vectors - use LVQ + else: + base = { + "algorithm": "svs-vamana", + "datatype": datatype or "float32", + "graph_max_degree": 40, + "construction_window_size": 250, + "search_window_size": 20, + } + + if priority == "memory": + return cast(SVSConfig, {**base, "compression": "LVQ4"}) + elif priority == "speed": + return cast(SVSConfig, {**base, "compression": "LVQ4x8"}) + else: # balanced + return cast(SVSConfig, {**base, "compression": "LVQ4x4"}) + + @staticmethod + def estimate_memory_savings( + compression: str, dims: int, reduce: Optional[int] = None + ) -> float: + """Estimate memory savings percentage from compression. + + Calculates the percentage of memory saved compared to uncompressed float32 vectors. + + Args: + compression: Compression type (e.g., "LVQ4", "LeanVec4x8") + dims: Original vector dimensionality + reduce: Reduced dimensionality (for LeanVec compression) + + Returns: + float: Memory savings percentage (0-100) + + Examples: + >>> # LeanVec with dimensionality reduction + >>> CompressionAdvisor.estimate_memory_savings( + ... compression="LeanVec4x8", + ... dims=1536, + ... reduce=768 + ... ) + 81.2 + + >>> # LVQ without dimensionality reduction + >>> CompressionAdvisor.estimate_memory_savings( + ... compression="LVQ4", + ... dims=384 + ... ) + 87.5 + """ + # Base bits per dimension (float32) + base_bits = 32 + + # Compressed bits per dimension + compression_bits = CompressionAdvisor.COMPRESSION_BITS.get( + compression, base_bits + ) + + # Account for dimensionality reduction + effective_dims = reduce if reduce else dims + + # Calculate savings + original_size = dims * base_bits + compressed_size = effective_dims * compression_bits + savings = (1 - compressed_size / original_size) * 100 + + return round(savings, 1) diff --git a/tests/unit/test_compression_advisor.py b/tests/unit/test_compression_advisor.py new file mode 100644 index 00000000..6226d543 --- /dev/null +++ b/tests/unit/test_compression_advisor.py @@ -0,0 +1,250 @@ +"""Unit tests for CompressionAdvisor utility.""" + +import pytest + +from redisvl.utils.compression import CompressionAdvisor, SVSConfig + + +class TestCompressionAdvisorRecommend: + """Tests for CompressionAdvisor.recommend() method.""" + + def test_recommend_high_dim_memory_priority(self): + """Test memory-optimized config for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=1536, priority="memory") + + assert config["algorithm"] == "svs-vamana" + assert config["datatype"] == "float16" + assert config["compression"] == "LeanVec4x8" + assert config["reduce"] == 768 # dims // 2 + assert config["graph_max_degree"] == 64 + assert config["construction_window_size"] == 300 + assert config["search_window_size"] == 20 + + def test_recommend_high_dim_speed_priority(self): + """Test speed-optimized config for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=1536, priority="speed") + + assert config["algorithm"] == "svs-vamana" + assert config["datatype"] == "float16" + assert config["compression"] == "LeanVec4x8" + assert config["reduce"] == 384 # dims // 4 + assert config["graph_max_degree"] == 64 + assert config["construction_window_size"] == 300 + assert config["search_window_size"] == 40 + + def test_recommend_high_dim_balanced_priority(self): + """Test balanced config for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=1536, priority="balanced") + + assert config["algorithm"] == "svs-vamana" + assert config["datatype"] == "float16" + assert config["compression"] == "LeanVec4x8" + assert config["reduce"] == 768 # dims // 2 + assert config["graph_max_degree"] == 64 + assert config["construction_window_size"] == 300 + assert config["search_window_size"] == 30 + + def test_recommend_high_dim_default_priority(self): + """Test default priority (balanced) for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=2048) + + assert config["compression"] == "LeanVec4x8" + assert config["reduce"] == 1024 + assert config["search_window_size"] == 30 + + def test_recommend_low_dim_memory_priority(self): + """Test memory-optimized config for low-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=384, priority="memory") + + assert config["algorithm"] == "svs-vamana" + assert config["datatype"] == "float32" + assert config["compression"] == "LVQ4" + assert "reduce" not in config # LVQ doesn't use reduce + assert config["graph_max_degree"] == 40 + assert config["construction_window_size"] == 250 + assert config["search_window_size"] == 20 + + def test_recommend_low_dim_speed_priority(self): + """Test speed-optimized config for low-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=384, priority="speed") + + assert config["algorithm"] == "svs-vamana" + assert config["datatype"] == "float32" + assert config["compression"] == "LVQ4x8" + assert "reduce" not in config + assert config["graph_max_degree"] == 40 + assert config["construction_window_size"] == 250 + assert config["search_window_size"] == 20 + + def test_recommend_low_dim_balanced_priority(self): + """Test balanced config for low-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=768, priority="balanced") + + assert config["algorithm"] == "svs-vamana" + assert config["datatype"] == "float32" + assert config["compression"] == "LVQ4x4" + assert "reduce" not in config + assert config["graph_max_degree"] == 40 + assert config["construction_window_size"] == 250 + assert config["search_window_size"] == 20 + + def test_recommend_threshold_boundary_low(self): + """Test recommendation at threshold boundary (1023 dims).""" + config = CompressionAdvisor.recommend(dims=1023) + + # Should use LVQ (below threshold) + assert config["compression"] in ["LVQ4", "LVQ4x4", "LVQ4x8"] + assert config["datatype"] == "float32" + assert "reduce" not in config + + def test_recommend_threshold_boundary_high(self): + """Test recommendation at threshold boundary (1024 dims).""" + config = CompressionAdvisor.recommend(dims=1024) + + # Should use LeanVec (at threshold) + assert config["compression"] == "LeanVec4x8" + assert config["datatype"] == "float16" + assert "reduce" in config + + def test_recommend_custom_datatype(self): + """Test custom datatype override.""" + config = CompressionAdvisor.recommend(dims=1536, datatype="float32") + + assert config["datatype"] == "float32" + + def test_recommend_speed_reduce_minimum(self): + """Test that speed priority respects minimum reduce value.""" + config = CompressionAdvisor.recommend(dims=1024, priority="speed") + + # dims // 4 = 256, max(256, 256) = 256 + assert config["reduce"] == 256 + + config = CompressionAdvisor.recommend(dims=512, priority="speed") + # Below threshold, should use LVQ + assert "reduce" not in config + + def test_recommend_invalid_dims_zero(self): + """Test that zero dims raises ValueError.""" + with pytest.raises(ValueError, match="dims must be positive"): + CompressionAdvisor.recommend(dims=0) + + def test_recommend_invalid_dims_negative(self): + """Test that negative dims raises ValueError.""" + with pytest.raises(ValueError, match="dims must be positive"): + CompressionAdvisor.recommend(dims=-100) + + +class TestCompressionAdvisorEstimateMemorySavings: + """Tests for CompressionAdvisor.estimate_memory_savings() method.""" + + def test_estimate_lvq4_no_reduce(self): + """Test memory savings for LVQ4 without dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LVQ4", dims=384 + ) + + # Original: 384 * 32 = 12,288 bits + # Compressed: 384 * 4 = 1,536 bits + # Savings: (1 - 1536/12288) * 100 = 87.5% + assert savings == 87.5 + + def test_estimate_lvq4x4_no_reduce(self): + """Test memory savings for LVQ4x4 without dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LVQ4x4", dims=768 + ) + + # Original: 768 * 32 = 24,576 bits + # Compressed: 768 * 8 = 6,144 bits + # Savings: (1 - 6144/24576) * 100 = 75.0% + assert savings == 75.0 + + def test_estimate_leanvec4x8_with_reduce(self): + """Test memory savings for LeanVec4x8 with dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LeanVec4x8", dims=1536, reduce=768 + ) + + # Original: 1536 * 32 = 49,152 bits + # Compressed: 768 * 12 = 9,216 bits + # Savings: (1 - 9216/49152) * 100 = 81.25% -> 81.2% (rounded) + assert savings == 81.2 + + def test_estimate_leanvec4x8_no_reduce(self): + """Test memory savings for LeanVec4x8 without dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LeanVec4x8", dims=1536 + ) + + # Original: 1536 * 32 = 49,152 bits + # Compressed: 1536 * 12 = 18,432 bits + # Savings: (1 - 18432/49152) * 100 = 62.5% + assert savings == 62.5 + + def test_estimate_leanvec8x8_with_reduce(self): + """Test memory savings for LeanVec8x8 with dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LeanVec8x8", dims=2048, reduce=1024 + ) + + # Original: 2048 * 32 = 65,536 bits + # Compressed: 1024 * 16 = 16,384 bits + # Savings: (1 - 16384/65536) * 100 = 75.0% + assert savings == 75.0 + + def test_estimate_unknown_compression(self): + """Test that unknown compression type defaults to no savings.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="UNKNOWN", dims=512 + ) + + # Should default to base_bits (32), so no savings + # Original: 512 * 32 = 16,384 bits + # Compressed: 512 * 32 = 16,384 bits + # Savings: 0% + assert savings == 0.0 + + def test_estimate_rounding(self): + """Test that savings are rounded to 1 decimal place.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LVQ4", dims=333 + ) + + # Original: 333 * 32 = 10,656 bits + # Compressed: 333 * 4 = 1,332 bits + # Savings: (1 - 1332/10656) * 100 = 87.5% + assert savings == 87.5 + assert isinstance(savings, float) + + +class TestSVSConfigTypedDict: + """Tests for SVSConfig TypedDict structure.""" + + def test_svs_config_structure(self): + """Test that SVSConfig can be constructed with all fields.""" + config: SVSConfig = { + "algorithm": "svs-vamana", + "datatype": "float16", + "compression": "LeanVec4x8", + "reduce": 768, + "graph_max_degree": 64, + "construction_window_size": 300, + "search_window_size": 30, + } + + assert config["algorithm"] == "svs-vamana" + assert config["reduce"] == 768 + + def test_svs_config_without_reduce(self): + """Test that SVSConfig can be constructed without reduce field.""" + config: SVSConfig = { + "algorithm": "svs-vamana", + "datatype": "float32", + "compression": "LVQ4", + "graph_max_degree": 40, + "construction_window_size": 250, + "search_window_size": 20, + } + + assert "reduce" not in config + assert config["compression"] == "LVQ4" diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index b6d41b69..7211102f 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -820,3 +820,161 @@ async def test_check_svs_support_async_passes(): with patch.object(index, "_get_client", return_value=mock_client): # Should not raise await index._check_svs_support_async() + + +# Phase C: Warning Tests + + +def test_leanvec_without_reduce_warning(caplog): + """Test warning when LeanVec compression is used without reduce parameter.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + # No reduce parameter + }, + ) + + # Check warning was logged + assert len(caplog.records) == 1 + assert "LeanVec compression selected without 'reduce'" in caplog.records[0].message + assert "Consider setting reduce=768" in caplog.records[0].message + + +def test_leanvec_with_reduce_no_warning(caplog): + """Test no warning when LeanVec compression is used with reduce parameter.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 768, + }, + ) + + # No warnings should be logged + assert len(caplog.records) == 0 + + +def test_low_graph_max_degree_warning(caplog): + """Test warning when graph_max_degree is too low.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 16, # Too low + }, + ) + + # Check warning was logged + assert len(caplog.records) == 1 + assert "graph_max_degree=16 is low" in caplog.records[0].message + assert "Consider values between 32-64" in caplog.records[0].message + + +def test_normal_graph_max_degree_no_warning(caplog): + """Test no warning when graph_max_degree is in normal range.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 40, # Normal value + }, + ) + + # No warnings should be logged + assert len(caplog.records) == 0 + + +def test_high_search_window_size_warning(caplog): + """Test warning when search_window_size is too high.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "search_window_size": 150, # Too high + }, + ) + + # Check warning was logged + assert len(caplog.records) == 1 + assert "search_window_size=150 is high" in caplog.records[0].message + assert "This may impact query latency" in caplog.records[0].message + + +def test_normal_search_window_size_no_warning(caplog): + """Test no warning when search_window_size is in normal range.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "search_window_size": 30, # Normal value + }, + ) + + # No warnings should be logged + assert len(caplog.records) == 0 + + +def test_multiple_warnings(caplog): + """Test multiple warnings are logged when multiple issues exist.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + # No reduce parameter - warning 1 + "graph_max_degree": 20, # Too low - warning 2 + "search_window_size": 120, # Too high - warning 3 + }, + ) + + # Check all three warnings were logged + assert len(caplog.records) == 3 + messages = [record.message for record in caplog.records] + assert any("LeanVec compression" in msg for msg in messages) + assert any("graph_max_degree=20" in msg for msg in messages) + assert any("search_window_size=120" in msg for msg in messages) From 5b928e6c7fa5a64b7261217a3899b835ce43ca62 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 20:11:07 -0400 Subject: [PATCH 12/29] Revert redis version dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c70dcb6a..3fc4202f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "numpy>=1.26.0,<3", "pyyaml>=5.4,<7.0", - "redis>=6.4.0,<7.0", + "redis>=5.0,<7.0", "pydantic>=2,<3", "tenacity>=8.2.2", "ml-dtypes>=0.4.0,<1.0.0", From 6d964599d16d723089c43ac6af4e3a786ae83ab8 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Fri, 10 Oct 2025 22:02:43 -0400 Subject: [PATCH 13/29] Add migration and integration test scripts --- redisvl/redis/connection.py | 51 +++ redisvl/utils/__init__.py | 3 +- redisvl/utils/migration.py | 433 +++++++++++++++++++ tests/integration/test_svs_integration.py | 498 ++++++++++++++++++++++ 4 files changed, 984 insertions(+), 1 deletion(-) create mode 100644 redisvl/utils/migration.py create mode 100644 tests/integration/test_svs_integration.py diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index aa3afeb8..0b74d95d 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -318,6 +318,57 @@ def parse_vector_attrs(attrs): # Default to float32 if missing normalized["datatype"] = "float32" + # Handle SVS-VAMANA specific parameters + # Compression - Redis uses different internal names, so we need to map them + if "compression" in vector_attrs: + compression_value = vector_attrs["compression"] + # Map Redis internal names to our enum values + compression_mapping = { + "GlobalSQ8": "LVQ4x4", # Default mapping + "GlobalSQ4": "LVQ4", + # Add more mappings as we discover them + } + # Try to map, otherwise use the value as-is + normalized["compression"] = compression_mapping.get( + compression_value, compression_value + ) + + # Dimensionality reduction (reduce parameter) + if "reduce" in vector_attrs: + try: + normalized["reduce"] = int(vector_attrs["reduce"]) + except (ValueError, TypeError): + pass + + # Graph parameters + if "graph_max_degree" in vector_attrs: + try: + normalized["graph_max_degree"] = int(vector_attrs["graph_max_degree"]) + except (ValueError, TypeError): + pass + + if "construction_window_size" in vector_attrs: + try: + normalized["construction_window_size"] = int( + vector_attrs["construction_window_size"] + ) + except (ValueError, TypeError): + pass + + if "search_window_size" in vector_attrs: + try: + normalized["search_window_size"] = int( + vector_attrs["search_window_size"] + ) + except (ValueError, TypeError): + pass + + if "epsilon" in vector_attrs: + try: + normalized["epsilon"] = float(vector_attrs["epsilon"]) + except (ValueError, TypeError): + pass + # Validate that we have required dims if "dims" not in normalized: # Could not parse dims - this field is not properly supported diff --git a/redisvl/utils/__init__.py b/redisvl/utils/__init__.py index 639e6e2a..8add03e3 100644 --- a/redisvl/utils/__init__.py +++ b/redisvl/utils/__init__.py @@ -1,3 +1,4 @@ from redisvl.utils.compression import CompressionAdvisor +from redisvl.utils.migration import IndexMigrator -__all__ = ["CompressionAdvisor"] +__all__ = ["CompressionAdvisor", "IndexMigrator"] diff --git a/redisvl/utils/migration.py b/redisvl/utils/migration.py new file mode 100644 index 00000000..50382909 --- /dev/null +++ b/redisvl/utils/migration.py @@ -0,0 +1,433 @@ +""" +Utilities for migrating Redis indices to SVS-VAMANA with compression. + +This module provides tools to migrate existing FLAT or HNSW indices to +SVS-VAMANA indices with compression, enabling significant memory savings +while maintaining search quality. +""" + +import logging +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +from redisvl.exceptions import RedisModuleVersionError +from redisvl.redis.connection import check_vector_capabilities +from redisvl.schema import IndexSchema +from redisvl.utils.compression import CompressionAdvisor +from redisvl.utils.log import get_logger + +# Avoid circular imports by using TYPE_CHECKING +if TYPE_CHECKING: + from redisvl.index import AsyncSearchIndex, SearchIndex + from redisvl.query import FilterQuery + from redisvl.query.filter import FilterExpression + +logger = get_logger(__name__) + + +class IndexMigrator: + """Helper class to migrate indices to SVS-VAMANA with compression. + + This class provides utilities to migrate existing FLAT or HNSW vector indices + to SVS-VAMANA indices with compression, enabling significant memory savings. + + Example: + .. code-block:: python + + from redisvl.index import SearchIndex + from redisvl.utils import IndexMigrator + + # Load existing index + old_index = SearchIndex.from_existing("my_flat_index") + + # Migrate to SVS-VAMANA with LVQ compression + new_index = IndexMigrator.migrate_to_svs( + old_index, + compression="LVQ4x4", + batch_size=1000 + ) + + print(f"Migrated {new_index.info()['num_docs']} documents") + """ + + @staticmethod + def migrate_to_svs( + old_index: "SearchIndex", + new_index_name: Optional[str] = None, + compression: Optional[str] = None, + reduce: Optional[int] = None, + batch_size: int = 1000, + overwrite: bool = False, + progress_callback: Optional[Callable[[int, int], None]] = None, + ) -> "SearchIndex": + """Migrate an existing index to SVS-VAMANA with compression. + + This method creates a new SVS-VAMANA index and copies all data from the + old index in batches. The old index is not modified or deleted. + + Args: + old_index: The existing SearchIndex to migrate from. + new_index_name: Name for the new index. If None, uses "{old_name}_svs". + compression: Compression type (LVQ4, LVQ4x4, LVQ4x8, LVQ8, LeanVec4x8, LeanVec8x8). + If None, uses CompressionAdvisor to recommend based on dimensions. + reduce: Dimensionality reduction parameter for LeanVec compression. + Required for LeanVec compression types. + batch_size: Number of documents to migrate per batch. Default: 1000. + overwrite: Whether to overwrite the new index if it exists. Default: False. + progress_callback: Optional callback function(current, total) for progress tracking. + + Returns: + SearchIndex: The new SVS-VAMANA index with migrated data. + + Raises: + RedisModuleVersionError: If Redis version doesn't support SVS-VAMANA. + ValueError: If the old index has no vector fields or invalid parameters. + + Example: + .. code-block:: python + + def progress(current, total): + print(f"Migrated {current}/{total} documents") + + new_index = IndexMigrator.migrate_to_svs( + old_index, + compression="LVQ4x4", + batch_size=500, + progress_callback=progress + ) + """ + # Import here to avoid circular imports + from redisvl.index import SearchIndex + from redisvl.query import FilterQuery + from redisvl.query.filter import FilterExpression + + # Check SVS-VAMANA support + caps = check_vector_capabilities(old_index._redis_client) + if not caps.svs_vamana_supported: + raise RedisModuleVersionError.for_svs_vamana( + caps.redis_version, caps.search_version + ) + + # Find vector fields in the old schema + vector_fields = [ + (name, field) + for name, field in old_index.schema.fields.items() + if hasattr(field, "attrs") and hasattr(field.attrs, "algorithm") + ] + + if not vector_fields: + raise ValueError("Old index has no vector fields to migrate") + + # Create new schema based on old schema + new_schema_dict = old_index.schema.to_dict() + + # Update index name + if new_index_name is None: + new_index_name = f"{old_index.name}_svs" + new_schema_dict["index"]["name"] = new_index_name + new_schema_dict["index"]["prefix"] = new_index_name + + # Update vector fields to use SVS-VAMANA + for field_dict in new_schema_dict["fields"]: + if field_dict["type"] == "vector": + attrs = field_dict.get("attrs", {}) + dims = attrs.get("dims") + + if dims is None: + raise ValueError(f"Vector field '{field_dict['name']}' has no dims") + + # Use CompressionAdvisor if compression not specified + if compression is None: + config = CompressionAdvisor.recommend( + dims=dims, + priority="balanced", + datatype=attrs.get("datatype", "float32"), + ) + compression = config["compression"] + if "reduce" in config and reduce is None: + reduce = config["reduce"] + logger.info( + f"CompressionAdvisor recommended: {compression} " + f"(reduce={reduce}) for {dims} dims" + ) + + # Update to SVS-VAMANA + attrs["algorithm"] = "svs-vamana" + attrs["compression"] = compression + + if reduce is not None: + attrs["reduce"] = reduce + + # Set default SVS parameters if not present + if "graph_max_degree" not in attrs: + attrs["graph_max_degree"] = 40 + if "construction_window_size" not in attrs: + attrs["construction_window_size"] = 250 + if "search_window_size" not in attrs: + attrs["search_window_size"] = 20 + # Set a low training threshold for small datasets + # Default is 10240, minimum is 1024 (DEFAULT_BLOCK_SIZE) + if "training_threshold" not in attrs: + attrs["training_threshold"] = 1024 + + # Create new index + new_schema = IndexSchema.from_dict(new_schema_dict) + new_index = SearchIndex(schema=new_schema, redis_client=old_index._redis_client) + new_index.create(overwrite=overwrite) + + logger.info(f"Created new SVS-VAMANA index: {new_index_name}") + + # Get total document count + old_info = old_index.info() + total_docs = int(old_info.get("num_docs", 0)) + + if total_docs == 0: + logger.warning("Old index has no documents to migrate") + return new_index + + logger.info(f"Migrating {total_docs} documents in batches of {batch_size}") + + # Migrate data in batches using pagination + migrated_count = 0 + query = FilterQuery( + filter_expression=FilterExpression("*"), + return_fields=list(old_index.schema.fields.keys()), + ) + + for batch in old_index.paginate(query, page_size=batch_size): + if batch: + # The 'id' field contains the full Redis key (e.g., "prefix:ulid") + # We need to preserve the document ID part for the new index + batch_keys = [] + batch_docs = [] + + for doc in batch: + # Get the full Redis key from the id field + full_key = doc.get("id", "") + # Extract the document ID (everything after the prefix) + # Split by the key separator and take the last part + doc_id = full_key.split(old_index.schema.index.key_separator)[-1] + + # Create a copy of the document without the id field + # (the id field is metadata, not actual document data) + doc_copy = {k: v for k, v in doc.items() if k != "id"} + + batch_keys.append(new_index.key(doc_id)) + batch_docs.append(doc_copy) + + # Load batch to new index with explicit keys to preserve IDs + new_index.load(batch_docs, keys=batch_keys) + migrated_count += len(batch) + + # Call progress callback if provided + if progress_callback: + progress_callback(migrated_count, total_docs) + + logger.debug(f"Migrated {migrated_count}/{total_docs} documents") + + logger.info(f"Migration complete: {migrated_count} documents migrated") + + # Verify migration by checking Redis keys (not index count) + # Note: SVS-VAMANA indices have a training_threshold (default 10240) + # Documents are written to Redis but may not be indexed until threshold is reached + new_info = new_index.info() + new_doc_count = int(new_info.get("num_docs", 0)) + + if new_doc_count != total_docs: + # Check if documents exist in Redis even if not indexed yet + client = new_index._redis_client + actual_keys = client.keys(f"{new_index.schema.index.prefix}:*") + actual_count = len(actual_keys) + + if actual_count == total_docs: + logger.info( + f"Documents written to Redis: {actual_count}/{total_docs}. " + f"Index shows {new_doc_count} (may be below training_threshold)" + ) + else: + logger.warning( + f"Document count mismatch: expected {total_docs}, " + f"got {actual_count} in Redis, {new_doc_count} in index" + ) + + return new_index + + @staticmethod + async def migrate_to_svs_async( + old_index: "AsyncSearchIndex", + new_index_name: Optional[str] = None, + compression: Optional[str] = None, + reduce: Optional[int] = None, + batch_size: int = 1000, + overwrite: bool = False, + progress_callback: Optional[Callable[[int, int], None]] = None, + ) -> "AsyncSearchIndex": + """Asynchronously migrate an existing index to SVS-VAMANA with compression. + + This is the async version of migrate_to_svs(). See migrate_to_svs() for + detailed documentation. + + Args: + old_index: The existing AsyncSearchIndex to migrate from. + new_index_name: Name for the new index. If None, uses "{old_name}_svs". + compression: Compression type. If None, uses CompressionAdvisor. + reduce: Dimensionality reduction parameter for LeanVec. + batch_size: Number of documents to migrate per batch. Default: 1000. + overwrite: Whether to overwrite the new index if it exists. Default: False. + progress_callback: Optional callback function(current, total) for progress. + + Returns: + AsyncSearchIndex: The new SVS-VAMANA index with migrated data. + + Example: + .. code-block:: python + + async def progress(current, total): + print(f"Migrated {current}/{total} documents") + + new_index = await IndexMigrator.migrate_to_svs_async( + old_index, + compression="LVQ4x4", + progress_callback=progress + ) + """ + # Import here to avoid circular imports + from redisvl.index import AsyncSearchIndex + from redisvl.query import FilterQuery + from redisvl.query.filter import FilterExpression + from redisvl.redis.connection import check_vector_capabilities_async + + # Check SVS-VAMANA support + client = await old_index._get_client() + caps = await check_vector_capabilities_async(client) + if not caps.svs_vamana_supported: + raise RedisModuleVersionError.for_svs_vamana( + caps.redis_version, caps.search_version + ) + + # Find vector fields + vector_fields = [ + (name, field) + for name, field in old_index.schema.fields.items() + if hasattr(field, "attrs") and hasattr(field.attrs, "algorithm") + ] + + if not vector_fields: + raise ValueError("Old index has no vector fields to migrate") + + # Create new schema + new_schema_dict = old_index.schema.to_dict() + + if new_index_name is None: + new_index_name = f"{old_index.name}_svs" + new_schema_dict["index"]["name"] = new_index_name + new_schema_dict["index"]["prefix"] = new_index_name + + # Update vector fields + for field_dict in new_schema_dict["fields"]: + if field_dict["type"] == "vector": + attrs = field_dict.get("attrs", {}) + dims = attrs.get("dims") + + if dims is None: + raise ValueError(f"Vector field '{field_dict['name']}' has no dims") + + if compression is None: + config = CompressionAdvisor.recommend( + dims=dims, + priority="balanced", + datatype=attrs.get("datatype", "float32"), + ) + compression = config["compression"] + if "reduce" in config and reduce is None: + reduce = config["reduce"] + logger.info( + f"CompressionAdvisor recommended: {compression} " + f"(reduce={reduce}) for {dims} dims" + ) + + attrs["algorithm"] = "svs-vamana" + attrs["compression"] = compression + + if reduce is not None: + attrs["reduce"] = reduce + + if "graph_max_degree" not in attrs: + attrs["graph_max_degree"] = 40 + if "construction_window_size" not in attrs: + attrs["construction_window_size"] = 250 + if "search_window_size" not in attrs: + attrs["search_window_size"] = 20 + if "training_threshold" not in attrs: + attrs["training_threshold"] = 1024 + + # Create new index + new_schema = IndexSchema.from_dict(new_schema_dict) + new_index = AsyncSearchIndex(schema=new_schema, redis_client=client) + await new_index.create(overwrite=overwrite) + + logger.info(f"Created new SVS-VAMANA index: {new_index_name}") + + # Get total document count + old_info = await old_index.info() + total_docs = int(old_info.get("num_docs", 0)) + + if total_docs == 0: + logger.warning("Old index has no documents to migrate") + return new_index + + logger.info(f"Migrating {total_docs} documents in batches of {batch_size}") + + # Migrate data in batches + migrated_count = 0 + query = FilterQuery( + filter_expression=FilterExpression("*"), + return_fields=list(old_index.schema.fields.keys()), + ) + + async for batch in old_index.paginate(query, page_size=batch_size): + if batch: + # Extract document IDs from full Redis keys + batch_keys = [] + batch_docs = [] + + for doc in batch: + full_key = doc.get("id", "") + doc_id = full_key.split(old_index.schema.index.key_separator)[-1] + doc_copy = {k: v for k, v in doc.items() if k != "id"} + + batch_keys.append(new_index.key(doc_id)) + batch_docs.append(doc_copy) + + # Load batch to new index with explicit keys + await new_index.load(batch_docs, keys=batch_keys) + migrated_count += len(batch) + + if progress_callback: + progress_callback(migrated_count, total_docs) + + logger.debug(f"Migrated {migrated_count}/{total_docs} documents") + + logger.info(f"Migration complete: {migrated_count} documents migrated") + + # Verify migration by checking Redis keys + new_info = await new_index.info() + new_doc_count = int(new_info.get("num_docs", 0)) + + if new_doc_count != total_docs: + # Check if documents exist in Redis even if not indexed yet + actual_keys = await client.keys(f"{new_index.schema.index.prefix}:*") + actual_count = len(actual_keys) + + if actual_count == total_docs: + logger.info( + f"Documents written to Redis: {actual_count}/{total_docs}. " + f"Index shows {new_doc_count} (may be below training_threshold)" + ) + else: + logger.warning( + f"Document count mismatch: expected {total_docs}, " + f"got {actual_count} in Redis, {new_doc_count} in index" + ) + + return new_index + diff --git a/tests/integration/test_svs_integration.py b/tests/integration/test_svs_integration.py new file mode 100644 index 00000000..31d6327a --- /dev/null +++ b/tests/integration/test_svs_integration.py @@ -0,0 +1,498 @@ +""" +Integration tests for SVS-VAMANA vector indexing. + +These tests require: +- Redis >= 8.2.0 +- RediSearch >= 2.8.10 +- Environment variable: REDISVL_TEST_SVS=1 + +Run with: + REDISVL_TEST_SVS=1 pytest tests/integration/test_svs_integration.py + +To run with Redis 8.2+ in Docker: + docker run -d -p 6379:6379 redis/redis-stack-server:8.2.2-v0 + REDISVL_TEST_SVS=1 pytest tests/integration/test_svs_integration.py + +Or set the Redis image for the test suite: + REDIS_IMAGE=redis/redis-stack-server:8.2.2-v0 REDISVL_TEST_SVS=1 pytest tests/integration/test_svs_integration.py +""" + +import os + +import numpy as np +import pytest + +from redisvl.exceptions import RedisModuleVersionError +from redisvl.index import SearchIndex +from redisvl.query import VectorQuery +from redisvl.redis.connection import check_vector_capabilities +from redisvl.redis.utils import array_to_buffer +from redisvl.schema import IndexSchema +from redisvl.utils import CompressionAdvisor + +# Skip all tests in this module if REDISVL_TEST_SVS is not set +pytestmark = pytest.mark.skipif( + not os.getenv("REDISVL_TEST_SVS"), + reason="SVS tests require REDISVL_TEST_SVS=1 and Redis 8.2+", +) + + +@pytest.fixture +def svs_schema_lvq(worker_id): + """Create SVS-VAMANA schema with LVQ compression.""" + return IndexSchema.from_dict( + { + "index": { + "name": f"svs_lvq_{worker_id}", + "prefix": f"svs_lvq_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "content", "type": "text"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 384, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LVQ4x4", + "graph_max_degree": 40, + "construction_window_size": 250, + "search_window_size": 20, + }, + }, + ], + } + ) + + +@pytest.fixture +def svs_schema_leanvec(worker_id): + """Create SVS-VAMANA schema with LeanVec compression and dimensionality reduction.""" + return IndexSchema.from_dict( + { + "index": { + "name": f"svs_leanvec_{worker_id}", + "prefix": f"svs_leanvec_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "content", "type": "text"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 768, + "graph_max_degree": 64, + "construction_window_size": 300, + "search_window_size": 30, + }, + }, + ], + } + ) + + +@pytest.fixture +def svs_index_lvq(svs_schema_lvq, client): + """Create SVS-VAMANA index with LVQ compression.""" + index = SearchIndex(schema=svs_schema_lvq, redis_client=client) + index.create(overwrite=True) + yield index + index.delete(drop=True) + + +@pytest.fixture +def svs_index_leanvec(svs_schema_leanvec, client): + """Create SVS-VAMANA index with LeanVec compression.""" + index = SearchIndex(schema=svs_schema_leanvec, redis_client=client) + index.create(overwrite=True) + yield index + index.delete(drop=True) + + +def generate_test_vectors(dims, count=100, dtype="float32"): + """Generate random test vectors.""" + vectors = [] + for i in range(count): + vector = np.random.random(dims).astype(dtype) + vectors.append( + { + "id": f"doc_{i}", + "content": f"This is test document {i}", + "embedding": array_to_buffer(vector, dtype=dtype), + } + ) + return vectors + + +class TestSVSCapabilityDetection: + """Test SVS-VAMANA capability detection.""" + + def test_check_svs_capabilities(self, client): + """Test that SVS-VAMANA is supported on the test Redis instance.""" + caps = check_vector_capabilities(client) + + # These tests require Redis 8.2+ with RediSearch 2.8.10+ + assert caps.svs_vamana_supported is True, ( + f"SVS-VAMANA not supported. " + f"Redis: {caps.redis_version}, " + f"RediSearch: {caps.search_version}" + ) + + +class TestSVSIndexCreation: + """Test creating SVS-VAMANA indices with various configurations.""" + + def test_create_svs_index_lvq(self, svs_index_lvq): + """Test creating SVS-VAMANA index with LVQ compression.""" + assert svs_index_lvq.exists() + + # Verify index info + info = svs_index_lvq.info() + assert info["num_docs"] == 0 + + def test_create_svs_index_leanvec(self, svs_index_leanvec): + """Test creating SVS-VAMANA index with LeanVec compression.""" + assert svs_index_leanvec.exists() + + # Verify index info + info = svs_index_leanvec.info() + assert info["num_docs"] == 0 + + def test_create_svs_with_compression_advisor(self, client, worker_id): + """Test creating SVS-VAMANA index using CompressionAdvisor.""" + dims = 768 + config = CompressionAdvisor.recommend(dims=dims, priority="balanced") + + schema = IndexSchema.from_dict( + { + "index": { + "name": f"svs_advisor_{worker_id}", + "prefix": f"svs_advisor_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + **config, + "distance_metric": "cosine", + }, + }, + ], + } + ) + + index = SearchIndex(schema=schema, redis_client=client) + index.create(overwrite=True) + + try: + assert index.exists() + info = index.info() + assert info["num_docs"] == 0 + finally: + index.delete(drop=True) + + +class TestSVSDataIngestion: + """Test loading data into SVS-VAMANA indices.""" + + def test_load_data_lvq(self, svs_index_lvq): + """Test loading data into SVS-VAMANA index with LVQ compression.""" + vectors = generate_test_vectors(dims=384, count=50, dtype="float32") + svs_index_lvq.load(vectors) + + # Verify data was loaded + info = svs_index_lvq.info() + assert info["num_docs"] == 50 + + def test_load_data_leanvec(self, svs_index_leanvec): + """Test loading data into SVS-VAMANA index with LeanVec compression.""" + vectors = generate_test_vectors(dims=1536, count=50, dtype="float32") + svs_index_leanvec.load(vectors) + + # Verify data was loaded + info = svs_index_leanvec.info() + assert info["num_docs"] == 50 + + def test_load_large_batch(self, svs_index_lvq): + """Test loading larger batch of data.""" + vectors = generate_test_vectors(dims=384, count=200, dtype="float32") + svs_index_lvq.load(vectors) + + # Verify data was loaded + info = svs_index_lvq.info() + assert info["num_docs"] == 200 + + +class TestSVSQuerying: + """Test querying SVS-VAMANA indices.""" + + def test_vector_query_lvq(self, svs_index_lvq): + """Test vector similarity search on SVS-VAMANA index with LVQ.""" + # Load test data + vectors = generate_test_vectors(dims=384, count=100, dtype="float32") + svs_index_lvq.load(vectors) + + # Create query vector + query_vector = np.random.random(384).astype(np.float32) + + # Execute query + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id", "content"], + num_results=10, + ) + + results = svs_index_lvq.query(query) + + # Verify results + assert len(results) <= 10 + assert all("id" in result for result in results) + assert all("content" in result for result in results) + + def test_vector_query_leanvec(self, svs_index_leanvec): + """Test vector similarity search on SVS-VAMANA index with LeanVec.""" + # Load test data + vectors = generate_test_vectors(dims=1536, count=100, dtype="float32") + svs_index_leanvec.load(vectors) + + # Create query vector + query_vector = np.random.random(1536).astype(np.float32) + + # Execute query + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id", "content"], + num_results=5, + ) + + results = svs_index_leanvec.query(query) + + # Verify results + assert len(results) <= 5 + assert all("id" in result for result in results) + + def test_query_with_filters(self, svs_index_lvq): + """Test vector query with filters on SVS-VAMANA index.""" + # Load test data with specific IDs + vectors = [] + for i in range(50): + vector = np.random.random(384).astype(np.float32) + vectors.append( + { + "id": f"category_a_{i}" if i < 25 else f"category_b_{i}", + "content": f"Document {i}", + "embedding": array_to_buffer(vector, dtype="float32"), + } + ) + svs_index_lvq.load(vectors) + + # Query with filter + query_vector = np.random.random(384).astype(np.float32) + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id", "content"], + num_results=10, + filter_expression="@id:{category_a*}", + ) + + results = svs_index_lvq.query(query) + + # Verify all results match filter + assert len(results) <= 10 + assert all(result["id"].startswith("category_a") for result in results) + + +class TestSVSFromExisting: + """Test loading existing SVS-VAMANA indices.""" + + def test_from_existing_lvq(self, svs_index_lvq, client): + """Test loading existing SVS-VAMANA index with LVQ compression.""" + # Load some data + vectors = generate_test_vectors(dims=384, count=20, dtype="float32") + svs_index_lvq.load(vectors) + + # Load the index from existing + loaded_index = SearchIndex.from_existing( + svs_index_lvq.name, redis_client=client + ) + + # Verify the loaded index + assert loaded_index.exists() + assert loaded_index.name == svs_index_lvq.name + + # Verify schema was loaded correctly + embedding_field = loaded_index.schema.fields["embedding"] + assert embedding_field.attrs.algorithm.value == "SVS-VAMANA" + assert embedding_field.attrs.compression.value == "LVQ4x4" + assert embedding_field.attrs.dims == 384 + + # Verify data is accessible + info = loaded_index.info() + assert info["num_docs"] == 20 + + def test_from_existing_leanvec(self, svs_index_leanvec, client): + """Test loading existing SVS-VAMANA index with LeanVec compression.""" + # Load some data + vectors = generate_test_vectors(dims=1536, count=20, dtype="float32") + svs_index_leanvec.load(vectors) + + # Load the index from existing + loaded_index = SearchIndex.from_existing( + svs_index_leanvec.name, redis_client=client + ) + + # Verify the loaded index + assert loaded_index.exists() + assert loaded_index.name == svs_index_leanvec.name + + # Verify schema was loaded correctly + embedding_field = loaded_index.schema.fields["embedding"] + assert embedding_field.attrs.algorithm.value == "SVS-VAMANA" + assert embedding_field.attrs.compression.value == "LeanVec4x8" + assert embedding_field.attrs.dims == 1536 + assert embedding_field.attrs.reduce == 768 + + # Verify data is accessible + info = loaded_index.info() + assert info["num_docs"] == 20 + + +class TestSVSCompressionTypes: + """Test different compression types for SVS-VAMANA.""" + + @pytest.mark.parametrize( + "compression,dims,dtype", + [ + ("LVQ4", 384, "float32"), + ("LVQ4x4", 384, "float32"), + ("LVQ4x8", 384, "float32"), + ("LVQ8", 384, "float32"), + ], + ) + def test_lvq_compression_types(self, client, worker_id, compression, dims, dtype): + """Test various LVQ compression types.""" + schema = IndexSchema.from_dict( + { + "index": { + "name": f"svs_{compression.lower()}_{worker_id}", + "prefix": f"svs_{compression.lower()}_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + "algorithm": "svs-vamana", + "datatype": dtype, + "distance_metric": "cosine", + "compression": compression, + }, + }, + ], + } + ) + + index = SearchIndex(schema=schema, redis_client=client) + index.create(overwrite=True) + + try: + # Load data + vectors = generate_test_vectors(dims=dims, count=50, dtype=dtype) + index.load(vectors) + + # Verify + assert index.exists() + info = index.info() + assert info["num_docs"] == 50 + + # Query + query_vector = np.random.random(dims).astype(dtype) + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id"], + num_results=5, + ) + results = index.query(query) + assert len(results) <= 5 + finally: + index.delete(drop=True) + + @pytest.mark.parametrize( + "compression,dims,reduce,dtype", + [ + ("LeanVec4x8", 1024, 512, "float16"), + ("LeanVec4x8", 1536, 768, "float16"), + ("LeanVec8x8", 1536, 768, "float16"), + ], + ) + def test_leanvec_compression_types( + self, client, worker_id, compression, dims, reduce, dtype + ): + """Test various LeanVec compression types with dimensionality reduction.""" + schema = IndexSchema.from_dict( + { + "index": { + "name": f"svs_{compression.lower()}_{reduce}_{worker_id}", + "prefix": f"svs_{compression.lower()}_{reduce}_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + "algorithm": "svs-vamana", + "datatype": dtype, + "distance_metric": "cosine", + "compression": compression, + "reduce": reduce, + }, + }, + ], + } + ) + + index = SearchIndex(schema=schema, redis_client=client) + index.create(overwrite=True) + + try: + # Load data + vectors = generate_test_vectors(dims=dims, count=50, dtype="float32") + index.load(vectors) + + # Verify + assert index.exists() + info = index.info() + assert info["num_docs"] == 50 + + # Query + query_vector = np.random.random(dims).astype(np.float32) + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id"], + num_results=5, + ) + results = index.query(query) + assert len(results) <= 5 + finally: + index.delete(drop=True) From 6263c9468752d0f6bd4b9e04aaea57d062a6544c Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Mon, 13 Oct 2025 14:11:01 -0400 Subject: [PATCH 14/29] Formatting --- redisvl/utils/migration.py | 109 ++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 55 deletions(-) diff --git a/redisvl/utils/migration.py b/redisvl/utils/migration.py index 50382909..adb9992c 100644 --- a/redisvl/utils/migration.py +++ b/redisvl/utils/migration.py @@ -26,26 +26,26 @@ class IndexMigrator: """Helper class to migrate indices to SVS-VAMANA with compression. - + This class provides utilities to migrate existing FLAT or HNSW vector indices to SVS-VAMANA indices with compression, enabling significant memory savings. - + Example: .. code-block:: python - + from redisvl.index import SearchIndex from redisvl.utils import IndexMigrator - + # Load existing index old_index = SearchIndex.from_existing("my_flat_index") - + # Migrate to SVS-VAMANA with LVQ compression new_index = IndexMigrator.migrate_to_svs( old_index, compression="LVQ4x4", batch_size=1000 ) - + print(f"Migrated {new_index.info()['num_docs']} documents") """ @@ -60,10 +60,10 @@ def migrate_to_svs( progress_callback: Optional[Callable[[int, int], None]] = None, ) -> "SearchIndex": """Migrate an existing index to SVS-VAMANA with compression. - + This method creates a new SVS-VAMANA index and copies all data from the old index in batches. The old index is not modified or deleted. - + Args: old_index: The existing SearchIndex to migrate from. new_index_name: Name for the new index. If None, uses "{old_name}_svs". @@ -74,20 +74,20 @@ def migrate_to_svs( batch_size: Number of documents to migrate per batch. Default: 1000. overwrite: Whether to overwrite the new index if it exists. Default: False. progress_callback: Optional callback function(current, total) for progress tracking. - + Returns: SearchIndex: The new SVS-VAMANA index with migrated data. - + Raises: RedisModuleVersionError: If Redis version doesn't support SVS-VAMANA. ValueError: If the old index has no vector fields or invalid parameters. - + Example: .. code-block:: python - + def progress(current, total): print(f"Migrated {current}/{total} documents") - + new_index = IndexMigrator.migrate_to_svs( old_index, compression="LVQ4x4", @@ -106,35 +106,35 @@ def progress(current, total): raise RedisModuleVersionError.for_svs_vamana( caps.redis_version, caps.search_version ) - + # Find vector fields in the old schema vector_fields = [ (name, field) for name, field in old_index.schema.fields.items() if hasattr(field, "attrs") and hasattr(field.attrs, "algorithm") ] - + if not vector_fields: raise ValueError("Old index has no vector fields to migrate") - + # Create new schema based on old schema new_schema_dict = old_index.schema.to_dict() - + # Update index name if new_index_name is None: new_index_name = f"{old_index.name}_svs" new_schema_dict["index"]["name"] = new_index_name new_schema_dict["index"]["prefix"] = new_index_name - + # Update vector fields to use SVS-VAMANA for field_dict in new_schema_dict["fields"]: if field_dict["type"] == "vector": attrs = field_dict.get("attrs", {}) dims = attrs.get("dims") - + if dims is None: raise ValueError(f"Vector field '{field_dict['name']}' has no dims") - + # Use CompressionAdvisor if compression not specified if compression is None: config = CompressionAdvisor.recommend( @@ -149,14 +149,14 @@ def progress(current, total): f"CompressionAdvisor recommended: {compression} " f"(reduce={reduce}) for {dims} dims" ) - + # Update to SVS-VAMANA attrs["algorithm"] = "svs-vamana" attrs["compression"] = compression - + if reduce is not None: attrs["reduce"] = reduce - + # Set default SVS parameters if not present if "graph_max_degree" not in attrs: attrs["graph_max_degree"] = 40 @@ -168,31 +168,31 @@ def progress(current, total): # Default is 10240, minimum is 1024 (DEFAULT_BLOCK_SIZE) if "training_threshold" not in attrs: attrs["training_threshold"] = 1024 - + # Create new index new_schema = IndexSchema.from_dict(new_schema_dict) new_index = SearchIndex(schema=new_schema, redis_client=old_index._redis_client) new_index.create(overwrite=overwrite) - + logger.info(f"Created new SVS-VAMANA index: {new_index_name}") - + # Get total document count old_info = old_index.info() total_docs = int(old_info.get("num_docs", 0)) - + if total_docs == 0: logger.warning("Old index has no documents to migrate") return new_index - + logger.info(f"Migrating {total_docs} documents in batches of {batch_size}") - + # Migrate data in batches using pagination migrated_count = 0 query = FilterQuery( filter_expression=FilterExpression("*"), return_fields=list(old_index.schema.fields.keys()), ) - + for batch in old_index.paginate(query, page_size=batch_size): if batch: # The 'id' field contains the full Redis key (e.g., "prefix:ulid") @@ -223,7 +223,7 @@ def progress(current, total): progress_callback(migrated_count, total_docs) logger.debug(f"Migrated {migrated_count}/{total_docs} documents") - + logger.info(f"Migration complete: {migrated_count} documents migrated") # Verify migration by checking Redis keys (not index count) @@ -262,10 +262,10 @@ async def migrate_to_svs_async( progress_callback: Optional[Callable[[int, int], None]] = None, ) -> "AsyncSearchIndex": """Asynchronously migrate an existing index to SVS-VAMANA with compression. - + This is the async version of migrate_to_svs(). See migrate_to_svs() for detailed documentation. - + Args: old_index: The existing AsyncSearchIndex to migrate from. new_index_name: Name for the new index. If None, uses "{old_name}_svs". @@ -274,16 +274,16 @@ async def migrate_to_svs_async( batch_size: Number of documents to migrate per batch. Default: 1000. overwrite: Whether to overwrite the new index if it exists. Default: False. progress_callback: Optional callback function(current, total) for progress. - + Returns: AsyncSearchIndex: The new SVS-VAMANA index with migrated data. - + Example: .. code-block:: python - + async def progress(current, total): print(f"Migrated {current}/{total} documents") - + new_index = await IndexMigrator.migrate_to_svs_async( old_index, compression="LVQ4x4", @@ -303,34 +303,34 @@ async def progress(current, total): raise RedisModuleVersionError.for_svs_vamana( caps.redis_version, caps.search_version ) - + # Find vector fields vector_fields = [ (name, field) for name, field in old_index.schema.fields.items() if hasattr(field, "attrs") and hasattr(field.attrs, "algorithm") ] - + if not vector_fields: raise ValueError("Old index has no vector fields to migrate") - + # Create new schema new_schema_dict = old_index.schema.to_dict() - + if new_index_name is None: new_index_name = f"{old_index.name}_svs" new_schema_dict["index"]["name"] = new_index_name new_schema_dict["index"]["prefix"] = new_index_name - + # Update vector fields for field_dict in new_schema_dict["fields"]: if field_dict["type"] == "vector": attrs = field_dict.get("attrs", {}) dims = attrs.get("dims") - + if dims is None: raise ValueError(f"Vector field '{field_dict['name']}' has no dims") - + if compression is None: config = CompressionAdvisor.recommend( dims=dims, @@ -344,13 +344,13 @@ async def progress(current, total): f"CompressionAdvisor recommended: {compression} " f"(reduce={reduce}) for {dims} dims" ) - + attrs["algorithm"] = "svs-vamana" attrs["compression"] = compression - + if reduce is not None: attrs["reduce"] = reduce - + if "graph_max_degree" not in attrs: attrs["graph_max_degree"] = 40 if "construction_window_size" not in attrs: @@ -364,26 +364,26 @@ async def progress(current, total): new_schema = IndexSchema.from_dict(new_schema_dict) new_index = AsyncSearchIndex(schema=new_schema, redis_client=client) await new_index.create(overwrite=overwrite) - + logger.info(f"Created new SVS-VAMANA index: {new_index_name}") - + # Get total document count old_info = await old_index.info() total_docs = int(old_info.get("num_docs", 0)) - + if total_docs == 0: logger.warning("Old index has no documents to migrate") return new_index - + logger.info(f"Migrating {total_docs} documents in batches of {batch_size}") - + # Migrate data in batches migrated_count = 0 query = FilterQuery( filter_expression=FilterExpression("*"), return_fields=list(old_index.schema.fields.keys()), ) - + async for batch in old_index.paginate(query, page_size=batch_size): if batch: # Extract document IDs from full Redis keys @@ -406,7 +406,7 @@ async def progress(current, total): progress_callback(migrated_count, total_docs) logger.debug(f"Migrated {migrated_count}/{total_docs} documents") - + logger.info(f"Migration complete: {migrated_count} documents migrated") # Verify migration by checking Redis keys @@ -430,4 +430,3 @@ async def progress(current, total): ) return new_index - From 0724f7d1ba748c117c83c5f2b85e8d9216732bb7 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 14 Oct 2025 19:05:50 -0400 Subject: [PATCH 15/29] Simplify version validation --- docs/user_guide/01_getting_started.ipynb | 9 +- docs/user_guide/index.md | 1 + redisvl/exceptions.py | 6 +- redisvl/index/index.py | 16 +-- redisvl/redis/connection.py | 88 +++++--------- redisvl/redis/constants.py | 2 + redisvl/utils/migration.py | 19 ++- tests/conftest.py | 6 +- tests/integration/test_semantic_router.py | 4 +- tests/integration/test_svs_integration.py | 11 +- tests/unit/test_svs_capability_detection.py | 126 -------------------- 11 files changed, 60 insertions(+), 228 deletions(-) delete mode 100644 tests/unit/test_svs_capability_detection.py diff --git a/docs/user_guide/01_getting_started.ipynb b/docs/user_guide/01_getting_started.ipynb index 7b8b60df..9f5034c9 100644 --- a/docs/user_guide/01_getting_started.ipynb +++ b/docs/user_guide/01_getting_started.ipynb @@ -761,7 +761,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -775,10 +775,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" - }, - "orig_nbformat": 4 + "version": "3.12.6" + } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index ca900d63..53c6a8ae 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -23,3 +23,4 @@ User guides provide helpful resources for using RedisVL and its different compon 08_semantic_router 11_advanced_queries ``` +# TODO: Nitin Add the new user guide for SVS-VAMANA \ No newline at end of file diff --git a/redisvl/exceptions.py b/redisvl/exceptions.py index 596a4cb5..79748c67 100644 --- a/redisvl/exceptions.py +++ b/redisvl/exceptions.py @@ -37,11 +37,10 @@ class RedisModuleVersionError(RedisVLError): """Error when Redis or module versions are incompatible with requested features.""" @classmethod - def for_svs_vamana(cls, capabilities, min_redis_version: str): + def for_svs_vamana(cls, min_redis_version: str): """Create error for unsupported SVS-VAMANA. Args: - capabilities: VectorSupport instance with version info min_redis_version: Minimum required Redis version Returns: @@ -49,9 +48,6 @@ def for_svs_vamana(cls, capabilities, min_redis_version: str): """ message = ( f"SVS-VAMANA requires Redis >= {min_redis_version} with RediSearch >= 2.8.10. " - f"Current: Redis {capabilities.redis_version}, " - f"RediSearch {capabilities.search_version_str}, " - f"SearchLight {capabilities.searchlight_version_str}. " f"Options: 1) Upgrade Redis Stack, " f"2) Use algorithm='hnsw' or 'flat', " f"3) Remove compression parameters" diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 5405bc25..e9916518 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -83,9 +83,9 @@ from redisvl.query.filter import FilterExpression from redisvl.redis.connection import ( RedisConnectionFactory, - check_vector_capabilities, - check_vector_capabilities_async, convert_index_info_to_schema, + supports_svs, + supports_svs_async, ) from redisvl.redis.constants import SVS_MIN_REDIS_VERSION from redisvl.schema import IndexSchema, StorageType @@ -552,10 +552,8 @@ def _check_svs_support(self) -> None: Raises: RedisModuleVersionError: If SVS-VAMANA requirements are not met. """ - caps = check_vector_capabilities(self._redis_client) - - if not caps.svs_vamana_supported: - raise RedisModuleVersionError.for_svs_vamana(caps, SVS_MIN_REDIS_VERSION) + if not supports_svs(self._redis_client): + raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) def create(self, overwrite: bool = False, drop: bool = False) -> None: """Create an index in Redis with the current schema and properties. @@ -1335,10 +1333,8 @@ async def _check_svs_support_async(self) -> None: RedisModuleVersionError: If SVS-VAMANA requirements are not met. """ client = await self._get_client() - caps = await check_vector_capabilities_async(client) - - if not caps.svs_vamana_supported: - raise RedisModuleVersionError.for_svs_vamana(caps, SVS_MIN_REDIS_VERSION) + if not await supports_svs_async(client): + raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) async def create(self, overwrite: bool = False, drop: bool = False) -> None: """Asynchronously create an index in Redis with the current schema diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 0b74d95d..2f2cf994 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -19,7 +19,7 @@ from redisvl.redis.constants import ( REDIS_URL_ENV_VAR, SVS_MIN_REDIS_VERSION, - SVS_REQUIRED_MODULES, + SVS_MIN_SEARCH_VERSION, ) from redisvl.redis.utils import convert_bytes, is_cluster_url from redisvl.types import AsyncRedisClient, RedisClient, SyncRedisClient @@ -72,16 +72,16 @@ def _strip_cluster_from_url_and_kwargs( return cleaned_url, cleaned_kwargs -def compare_versions(version1: str, version2: str): +def is_version_gte(version1: str, version2: str) -> bool: """ - Compare two Redis version strings numerically. + Check if version1 >= version2. Parameters: - version1 (str): The first version string (e.g., "7.2.4"). - version2 (str): The second version string (e.g., "6.2.1"). + version1 (str): The first version string (e.g., "7.2.4"). + version2 (str): The second version string (e.g., "6.2.1"). Returns: - int: -1 if version1 < version2, 0 if version1 == version2, 1 if version1 > version2. + bool: True if version1 >= version2, False otherwise. """ v1_parts = list(map(int, version1.split("."))) v2_parts = list(map(int, version2.split("."))) @@ -106,44 +106,14 @@ def unpack_redis_modules(module_list: List[Dict[str, Any]]) -> Dict[str, Any]: return {module["name"]: module["ver"] for module in module_list} -@dataclass -class VectorSupport: - """Redis server capabilities for vector operations.""" - - redis_version: str - search_version: int - searchlight_version: int - svs_vamana_supported: bool - - @property - def search_version_str(self) -> str: - """Format search module version as string.""" - return format_module_version(self.search_version) - - @property - def searchlight_version_str(self) -> str: - """Format searchlight module version as string.""" - return format_module_version(self.searchlight_version) - - -def format_module_version(version: int) -> str: - """Format module version from integer (20810) to string (2.8.10).""" - if version == 0: - return "not installed" - major = version // 10000 - minor = (version % 10000) // 100 - patch = version % 100 - return f"{major}.{minor}.{patch}" - - -def check_vector_capabilities(client: SyncRedisClient) -> VectorSupport: - """Check Redis server capabilities for vector features. +def supports_svs(client: SyncRedisClient) -> bool: + """Check if Redis server supports SVS-VAMANA. Args: client: Sync Redis client instance Returns: - VectorSupport with version info and supported features + True if SVS-VAMANA is supported, False otherwise """ info = client.info("server") # type: ignore[union-attr] redis_version = info.get("redis_version", "0.0.0") # type: ignore[union-attr] @@ -153,25 +123,26 @@ def check_vector_capabilities(client: SyncRedisClient) -> VectorSupport: searchlight_ver = modules.get("searchlight", 0) # Check if SVS-VAMANA requirements are met - redis_ok = compare_versions(redis_version, SVS_MIN_REDIS_VERSION) - modules_ok = search_ver >= 20810 or searchlight_ver >= 20810 - - return VectorSupport( - redis_version=redis_version, - search_version=search_ver, - searchlight_version=searchlight_ver, - svs_vamana_supported=redis_ok and modules_ok, + redis_ok = is_version_gte(redis_version, SVS_MIN_REDIS_VERSION) + + # Check either search or searchlight module (only one is typically installed) + # RediSearch is the open-source module, SearchLight is the enterprise version + modules_ok = ( + search_ver >= SVS_MIN_SEARCH_VERSION + or searchlight_ver >= SVS_MIN_SEARCH_VERSION ) + return redis_ok and modules_ok + -async def check_vector_capabilities_async(client: AsyncRedisClient) -> VectorSupport: - """Async version of check_vector_capabilities. +async def supports_svs_async(client: AsyncRedisClient) -> bool: + """Async version of _supports_svs. Args: client: Async Redis client instance Returns: - VectorSupport with version info and supported features + True if SVS-VAMANA is supported, False otherwise """ info = await client.info("server") # type: ignore[union-attr] redis_version = info.get("redis_version", "0.0.0") # type: ignore[union-attr] @@ -181,16 +152,17 @@ async def check_vector_capabilities_async(client: AsyncRedisClient) -> VectorSup searchlight_ver = modules.get("searchlight", 0) # Check if SVS-VAMANA requirements are met - redis_ok = compare_versions(redis_version, SVS_MIN_REDIS_VERSION) - modules_ok = search_ver >= 20810 or searchlight_ver >= 20810 - - return VectorSupport( - redis_version=redis_version, - search_version=search_ver, - searchlight_version=searchlight_ver, - svs_vamana_supported=redis_ok and modules_ok, + redis_ok = is_version_gte(redis_version, SVS_MIN_REDIS_VERSION) + + # Check either search or searchlight module (only one is typically installed) + # RediSearch is the open-source module, SearchLight is the enterprise version + modules_ok = ( + search_ver >= SVS_MIN_SEARCH_VERSION + or searchlight_ver >= SVS_MIN_SEARCH_VERSION ) + return redis_ok and modules_ok + def get_address_from_env() -> str: """Get Redis URL from environment variable.""" diff --git a/redisvl/redis/constants.py b/redisvl/redis/constants.py index 0a0737ed..fa6861d1 100644 --- a/redisvl/redis/constants.py +++ b/redisvl/redis/constants.py @@ -12,6 +12,8 @@ # Minimum Redis version for SVS-VAMANA SVS_MIN_REDIS_VERSION = "8.2.0" +# Minimum search module version for SVS-VAMANA (2.8.10) +SVS_MIN_SEARCH_VERSION = 20810 # default tag separator REDIS_TAG_SEPARATOR = "," diff --git a/redisvl/utils/migration.py b/redisvl/utils/migration.py index adb9992c..fe7f621f 100644 --- a/redisvl/utils/migration.py +++ b/redisvl/utils/migration.py @@ -10,7 +10,8 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union from redisvl.exceptions import RedisModuleVersionError -from redisvl.redis.connection import check_vector_capabilities +from redisvl.redis.connection import supports_svs +from redisvl.redis.constants import SVS_MIN_REDIS_VERSION from redisvl.schema import IndexSchema from redisvl.utils.compression import CompressionAdvisor from redisvl.utils.log import get_logger @@ -101,11 +102,8 @@ def progress(current, total): from redisvl.query.filter import FilterExpression # Check SVS-VAMANA support - caps = check_vector_capabilities(old_index._redis_client) - if not caps.svs_vamana_supported: - raise RedisModuleVersionError.for_svs_vamana( - caps.redis_version, caps.search_version - ) + if not supports_svs(old_index._redis_client): + raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) # Find vector fields in the old schema vector_fields = [ @@ -294,15 +292,12 @@ async def progress(current, total): from redisvl.index import AsyncSearchIndex from redisvl.query import FilterQuery from redisvl.query.filter import FilterExpression - from redisvl.redis.connection import check_vector_capabilities_async + from redisvl.redis.connection import supports_svs_async # Check SVS-VAMANA support client = await old_index._get_client() - caps = await check_vector_capabilities_async(client) - if not caps.svs_vamana_supported: - raise RedisModuleVersionError.for_svs_vamana( - caps.redis_version, caps.search_version - ) + if not await supports_svs_async(client): + raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) # Find vector fields vector_fields = [ diff --git a/tests/conftest.py b/tests/conftest.py index 692ce77d..9ff9795b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ from testcontainers.compose import DockerCompose from redisvl.index.index import AsyncSearchIndex, SearchIndex -from redisvl.redis.connection import RedisConnectionFactory, compare_versions +from redisvl.redis.connection import RedisConnectionFactory, is_version_gte from redisvl.redis.utils import array_to_buffer from redisvl.utils.vectorize import HFTextVectorizer @@ -699,7 +699,7 @@ def skip_if_redis_version_below(client, min_version: str, message: str = None): message: Custom skip message """ redis_version = get_redis_version(client) - if not compare_versions(redis_version, min_version): + if not is_version_gte(redis_version, min_version): skip_msg = message or f"Redis version {redis_version} < {min_version} required" pytest.skip(skip_msg) @@ -716,7 +716,7 @@ async def skip_if_redis_version_below_async( message: Custom skip message """ redis_version = await get_redis_version_async(client) - if not compare_versions(redis_version, min_version): + if not is_version_gte(redis_version, min_version): skip_msg = message or f"Redis version {redis_version} < {min_version} required" pytest.skip(skip_msg) diff --git a/tests/integration/test_semantic_router.py b/tests/integration/test_semantic_router.py index d082ceaa..47e7c451 100644 --- a/tests/integration/test_semantic_router.py +++ b/tests/integration/test_semantic_router.py @@ -11,7 +11,7 @@ Route, RoutingConfig, ) -from redisvl.redis.connection import compare_versions +from redisvl.redis.connection import is_version_gte from tests.conftest import skip_if_no_redisearch, skip_if_redis_version_below @@ -159,7 +159,7 @@ def test_add_route(semantic_router): assert "political speech" in route.references redis_version = semantic_router._index.client.info()["redis_version"] - if compare_versions(redis_version, "7.0.0"): + if is_version_gte(redis_version, "7.0.0"): match = semantic_router("political speech") print(match, flush=True) assert match is not None diff --git a/tests/integration/test_svs_integration.py b/tests/integration/test_svs_integration.py index 31d6327a..2cccebc5 100644 --- a/tests/integration/test_svs_integration.py +++ b/tests/integration/test_svs_integration.py @@ -25,7 +25,7 @@ from redisvl.exceptions import RedisModuleVersionError from redisvl.index import SearchIndex from redisvl.query import VectorQuery -from redisvl.redis.connection import check_vector_capabilities +from redisvl.redis.connection import supports_svs from redisvl.redis.utils import array_to_buffer from redisvl.schema import IndexSchema from redisvl.utils import CompressionAdvisor @@ -138,13 +138,10 @@ class TestSVSCapabilityDetection: def test_check_svs_capabilities(self, client): """Test that SVS-VAMANA is supported on the test Redis instance.""" - caps = check_vector_capabilities(client) - # These tests require Redis 8.2+ with RediSearch 2.8.10+ - assert caps.svs_vamana_supported is True, ( - f"SVS-VAMANA not supported. " - f"Redis: {caps.redis_version}, " - f"RediSearch: {caps.search_version}" + assert supports_svs(client) is True, ( + "SVS-VAMANA not supported. " + "Requires Redis >= 8.2.0 with RediSearch >= 2.8.10" ) diff --git a/tests/unit/test_svs_capability_detection.py b/tests/unit/test_svs_capability_detection.py deleted file mode 100644 index 0296fcb6..00000000 --- a/tests/unit/test_svs_capability_detection.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Unit tests for SVS-VAMANA capability detection. - -Tests the core functionality that determines if SVS-VAMANA vector indexing -is supported on the connected Redis instance. -""" - -from unittest.mock import AsyncMock, Mock, patch - -import pytest - -from redisvl.exceptions import RedisModuleVersionError -from redisvl.redis.connection import ( - VectorSupport, - check_vector_capabilities, - check_vector_capabilities_async, - compare_versions, - format_module_version, -) - - -def test_format_version_20810(): - """Test formatting version 20810 -> 2.8.10""" - assert format_module_version(20810) == "2.8.10" - - -def test_compare_greater_version(): - """Test version comparison: greater version returns True.""" - assert compare_versions("8.2.0", "8.1.0") is True - assert compare_versions("8.2.1", "8.2.0") is True - assert compare_versions("9.0.0", "8.2.0") is True - - -def test_compare_lesser_version(): - """Test version comparison: lesser version returns False.""" - assert compare_versions("7.2.4", "8.2.0") is False - assert compare_versions("8.1.9", "8.2.0") is False - assert compare_versions("8.2.0", "8.2.1") is False - - -def test_check_vector_capabilities_supported(): - """Test check_vector_capabilities when SVS is supported.""" - mock_client = Mock() - mock_client.info.return_value = {"redis_version": "8.2.0"} - - with patch( - "redisvl.redis.connection.RedisConnectionFactory.get_modules" - ) as mock_get_modules: - mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} - - caps = check_vector_capabilities(mock_client) - - assert caps.redis_version == "8.2.0" - assert caps.search_version == 20810 - assert caps.searchlight_version == 20810 - assert caps.svs_vamana_supported is True - - -def test_check_vector_capabilities_old_redis(): - """Test check_vector_capabilities with old Redis version.""" - mock_client = Mock() - mock_client.info.return_value = {"redis_version": "7.2.4"} - - with patch( - "redisvl.redis.connection.RedisConnectionFactory.get_modules" - ) as mock_get_modules: - mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} - - caps = check_vector_capabilities(mock_client) - - assert caps.redis_version == "7.2.4" - assert caps.svs_vamana_supported is False - - -def test_check_vector_capabilities_old_modules(): - """Test check_vector_capabilities with old module versions.""" - mock_client = Mock() - mock_client.info.return_value = {"redis_version": "8.2.0"} - - with patch( - "redisvl.redis.connection.RedisConnectionFactory.get_modules" - ) as mock_get_modules: - mock_get_modules.return_value = {"search": 20612, "searchlight": 20612} - - caps = check_vector_capabilities(mock_client) - - assert caps.search_version == 20612 - assert caps.searchlight_version == 20612 - assert caps.svs_vamana_supported is False - - -@pytest.mark.asyncio -async def test_check_vector_capabilities_async_supported(): - """Test check_vector_capabilities_async when SVS is supported.""" - mock_client = AsyncMock() - mock_client.info.return_value = {"redis_version": "8.2.0"} - - with patch( - "redisvl.redis.connection.RedisConnectionFactory.get_modules_async" - ) as mock_get_modules: - mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} - - caps = await check_vector_capabilities_async(mock_client) - - assert caps.redis_version == "8.2.0" - assert caps.search_version == 20810 - assert caps.svs_vamana_supported is True - - -def test_for_svs_vamana_error_message(): - """Test RedisModuleVersionError.for_svs_vamana creates proper exception.""" - caps = VectorSupport( - redis_version="7.2.4", - search_version=20612, - searchlight_version=20612, - svs_vamana_supported=False, - ) - - error = RedisModuleVersionError.for_svs_vamana(caps, "8.2.0") - - error_msg = str(error) - assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg - assert "RediSearch >= 2.8.10" in error_msg - assert "Redis 7.2.4" in error_msg - assert "RediSearch 2.6.12" in error_msg - assert "SearchLight 2.6.12" in error_msg From 67140540cd858f1296778837b6c55930dbea9727 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 15 Oct 2025 12:33:11 -0400 Subject: [PATCH 16/29] Update docs and info --- docs/api/schema.rst | 333 +++++++++++++++++++++++++++++++++------ redisvl/schema/fields.py | 93 +++++++---- 2 files changed, 347 insertions(+), 79 deletions(-) diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 7847f61f..18809bed 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -67,74 +67,166 @@ Each field type supports specific attributes that customize its behavior. Below **Text Field Attributes**: -- `weight`: Importance of the field in result calculation. -- `no_stem`: Disables stemming during indexing. -- `withsuffixtrie`: Optimizes queries by maintaining a suffix trie. -- `phonetic_matcher`: Enables phonetic matching. -- `sortable`: Allows sorting on this field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). -- `unf`: Un-normalized form. When True, preserves original case for sorting (requires `sortable=True`). +.. autoclass:: TextFieldAttributes + :members: + :undoc-members: **Tag Field Attributes**: -- `separator`: Character for splitting text into individual tags. -- `case_sensitive`: Case sensitivity in tag matching. -- `withsuffixtrie`: Suffix trie optimization for queries. -- `sortable`: Enables sorting based on the tag field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). +.. autoclass:: TagFieldAttributes + :members: + :undoc-members: **Numeric Field Attributes**: -- `sortable`: Enables sorting on the numeric field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). -- `unf`: Un-normalized form. When True, maintains original numeric representation for sorting (requires `sortable=True`). +.. autoclass:: NumericFieldAttributes + :members: + :undoc-members: **Geo Field Attributes**: -- `sortable`: Enables sorting based on the geo field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). +.. autoclass:: GeoFieldAttributes + :members: + :undoc-members: **Common Vector Field Attributes**: -- `dims`: Dimensionality of the vector. -- `algorithm`: Indexing algorithm (`flat`, `hnsw`, or `svs-vamana`). -- `datatype`: Float datatype of the vector (`bfloat16`, `float16`, `float32`, `float64`). Note: SVS-VAMANA only supports `float16` and `float32`. -- `distance_metric`: Metric for measuring query relevance (`COSINE`, `L2`, `IP`). -- `initial_cap`: Initial capacity for the index (optional). -- `index_missing`: When True, allows searching for documents missing this field (optional). +- `dims`: Dimensionality of the vector (e.g., 768, 1536). +- `algorithm`: Indexing algorithm for vector search: -**FLAT Vector Field Specific Attributes**: + - `flat`: Brute-force exact search. 100% recall, slower for large datasets. Best for <10K vectors. + - `hnsw`: Graph-based approximate search. Fast with high recall (95-99%). Best for general use. + - `svs-vamana`: Compressed approximate search. Memory-efficient with compression. Best for large datasets on Intel hardware. -- `block_size`: Block size for the FLAT index (optional). + .. note:: + For detailed algorithm comparison and selection guidance, see :ref:`vector-algorithm-comparison`. + +- `datatype`: Float precision (`bfloat16`, `float16`, `float32`, `float64`). Note: SVS-VAMANA only supports `float16` and `float32`. +- `distance_metric`: Similarity metric (`COSINE`, `L2`, `IP`). +- `initial_cap`: Initial capacity hint for memory allocation (optional). +- `index_missing`: When True, allows searching for documents missing this field (optional). **HNSW Vector Field Specific Attributes**: -- `m`: Max outgoing edges per node in each layer (default: 16). -- `ef_construction`: Max edge candidates during build time (default: 200). -- `ef_runtime`: Max top candidates during search (default: 10). -- `epsilon`: Range search boundary factor (default: 0.01). +HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with excellent recall. **Best for general-purpose vector search (10K-1M+ vectors).** + +.. dropdown:: When to use HNSW & Performance Details + :color: info + + **Use HNSW when:** + + - Medium to large datasets (10K-1M+ vectors) requiring high recall rates + - Search accuracy is more important than memory usage + - Need general-purpose vector search with balanced performance + - Cross-platform deployments where hardware-specific optimizations aren't available + + **Performance characteristics:** + + - **Search speed**: Very fast approximate search with tunable accuracy + - **Memory usage**: Higher than compressed SVS-VAMANA but reasonable for most applications + - **Recall quality**: Excellent recall rates (95-99%), often better than other approximate methods + - **Build time**: Moderate construction time, faster than SVS-VAMANA for smaller datasets + +.. currentmodule:: redisvl.schema.fields + +.. autoclass:: HNSWVectorFieldAttributes + :members: + :undoc-members: + +**HNSW Examples:** + +**Balanced configuration (recommended starting point):** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + distance_metric: cosine + datatype: float32 + # Balanced settings for good recall and performance + m: 16 + ef_construction: 200 + ef_runtime: 10 + +**High-recall configuration:** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + distance_metric: cosine + datatype: float32 + # Tuned for maximum accuracy + m: 32 + ef_construction: 400 + ef_runtime: 50 **SVS-VAMANA Vector Field Specific Attributes**: -SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. +SVS-VAMANA - High-performance search with optional compression. **Best for large datasets (>100K vectors) on Intel hardware with memory constraints.** + +.. dropdown:: When to use SVS-VAMANA & Detailed Guide + :color: info + + **Requirements:** + - Redis >= 8.2.0 with RediSearch >= 2.8.10 + - datatype must be 'float16' or 'float32' (float64/bfloat16 not supported) + + **Use SVS-VAMANA when:** + - Large datasets where memory is expensive + - Cloud deployments with memory-based pricing + - When 90-95% recall is acceptable + - High-dimensional vectors (>1024 dims) with LeanVec compression + + **Performance vs other algorithms:** + - **vs FLAT**: Much faster search, significantly lower memory usage with compression, but approximate results + + - **vs HNSW**: Better memory efficiency with compression, similar or better recall, Intel-optimized + + **Compression selection guide:** + + - **No compression**: Best performance, standard memory usage -- `graph_max_degree`: Maximum degree of the Vamana graph, i.e., the number of edges per node (default: 40). -- `construction_window_size`: Size of the candidate list during graph construction. Higher values yield better quality graphs but increase construction time (default: 250). -- `search_window_size`: Size of the candidate list during search. Higher values increase accuracy but also increase search latency (default: 20). -- `epsilon`: Relative factor for range query boundaries (default: 0.01). -- `compression`: Optional vector compression algorithm. Supported types: + - **LVQ4/LVQ8**: Good balance of compression (2x-4x) and performance - - `LVQ4`: 4-bit Learned Vector Quantization - - `LVQ4x4`: 4-bit LVQ with 4x compression - - `LVQ4x8`: 4-bit LVQ with 8x compression - - `LVQ8`: 8-bit Learned Vector Quantization - - `LeanVec4x8`: 4-bit LeanVec with 8x compression and dimensionality reduction - - `LeanVec8x8`: 8-bit LeanVec with 8x compression and dimensionality reduction + - **LeanVec4x8/LeanVec8x8**: Maximum compression (up to 8x) with dimensionality reduction -- `reduce`: Reduced dimensionality for LeanVec compression. Must be less than `dims`. Only valid with `LeanVec4x8` or `LeanVec8x8` compression types. Lowering this value can speed up search and reduce memory usage (optional). -- `training_threshold`: Minimum number of vectors required before compression training begins. Must be less than 100 * 1024 (default: 10 * 1024). + **Memory Savings Examples (1M vectors, 768 dims):** + - No compression (float32): 3.1 GB -**SVS-VAMANA Example**: + - LVQ4x4 compression: 1.6 GB (~48% savings) + + - LeanVec4x8 + reduce to 384: 580 MB (~81% savings) + +.. autoclass:: SVSVectorFieldAttributes + :members: + :undoc-members: + +**SVS-VAMANA Examples:** + +**Basic configuration (no compression):** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: svs-vamana + dims: 768 + distance_metric: cosine + datatype: float32 + # Standard settings for balanced performance + graph_max_degree: 40 + construction_window_size: 250 + search_window_size: 20 + +**High-performance configuration with compression:** .. code-block:: yaml @@ -145,16 +237,155 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap dims: 768 distance_metric: cosine datatype: float32 + # Tuned for better recall graph_max_degree: 64 construction_window_size: 500 search_window_size: 40 + # Maximum compression with dimensionality reduction compression: LeanVec4x8 - reduce: 384 + reduce: 384 # 50% dimensionality reduction training_threshold: 1000 -Note: - - SVS-VAMANA requires Redis >= 8.2 with RediSearch >= 2.8.10. - - SVS-VAMANA only supports `float16` and `float32` datatypes. - - The `reduce` parameter is only valid with LeanVec compression types (`LeanVec4x8` or `LeanVec8x8`). - - Intel's proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. - - See fully documented Redis-supported fields and options here: https://redis.io/commands/ft.create/ \ No newline at end of file +**Important Notes:** + +- **Requirements**: SVS-VAMANA requires Redis >= 8.2 with RediSearch >= 2.8.10. +- **Datatype limitations**: SVS-VAMANA only supports `float16` and `float32` datatypes (not `bfloat16` or `float64`). +- **Compression compatibility**: The `reduce` parameter is only valid with LeanVec compression types (`LeanVec4x8` or `LeanVec8x8`). +- **Platform considerations**: Intel's proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. +- **Performance tip**: Start with default parameters and tune `search_window_size` first for your speed vs accuracy requirements. + +**FLAT Vector Field Specific Attributes**: + +FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requiring 100% accuracy.** + +.. dropdown:: When to use FLAT & Performance Details + :color: info + + **Use FLAT when:** + - Small datasets (<10K vectors) where exact results are required + - Search accuracy is critical and approximate results are not acceptable + - Baseline comparisons when evaluating approximate algorithms + - Simple use cases where setup simplicity is more important than performance + + **Performance characteristics:** + - **Search accuracy**: 100% exact results (no approximation) + - **Search speed**: Linear time O(n) - slower as dataset grows + - **Memory usage**: Minimal overhead, stores vectors as-is + - **Build time**: Fastest index construction (no preprocessing) + + **Trade-offs vs other algorithms:** + - **vs HNSW**: Much slower search but exact results, faster index building + - **vs SVS-VAMANA**: Slower search and higher memory usage, but exact results + +.. autoclass:: FlatVectorFieldAttributes + :members: + :undoc-members: + +**FLAT Example:** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: flat + dims: 768 + distance_metric: cosine + datatype: float32 + # Optional: tune for batch processing + block_size: 1024 + +**Note**: FLAT is recommended for small datasets or when exact results are mandatory. For larger datasets, consider HNSW or SVS-VAMANA for better performance. + +.. _vector-algorithm-comparison: + +Vector Algorithm Comparison +=========================== + +This section provides detailed guidance for choosing between vector search algorithms. + +Algorithm Selection Guide +------------------------- + +.. list-table:: Vector Algorithm Comparison + :header-rows: 1 + :widths: 15 20 20 20 25 + + * - Algorithm + - Best For + - Performance + - Memory Usage + - Trade-offs + * - **FLAT** + - Small datasets (<10K vectors) + - 100% recall, O(n) search + - Minimal overhead + - Exact but slow for large data + * - **HNSW** + - General purpose (10K-1M+ vectors) + - 95-99% recall, O(log n) search + - Moderate (graph overhead) + - Fast approximate search + * - **SVS-VAMANA** + - Large datasets with memory constraints + - 90-95% recall, O(log n) search + - Low (with compression) + - Intel-optimized, requires newer Redis + +When to Use Each Algorithm +-------------------------- + +**Choose FLAT when:** + - Dataset size < 10,000 vectors + - Exact results are mandatory + - Simple setup is preferred + - Query latency is not critical + +**Choose HNSW when:** + - Dataset size 10K - 1M+ vectors + - Need balanced speed and accuracy + - Cross-platform compatibility required + - Most common choice for production + +**Choose SVS-VAMANA when:** + - Dataset size > 100K vectors + - Memory usage is a primary concern + - Running on Intel hardware + - Can accept 90-95% recall for memory savings + +Performance Characteristics +--------------------------- + +**Search Speed:** + - FLAT: Linear time O(n) - gets slower as data grows + - HNSW: Logarithmic time O(log n) - scales well + - SVS-VAMANA: Logarithmic time O(log n) - scales well + +**Memory Usage (1M vectors, 768 dims, float32):** + - FLAT: ~3.1 GB (baseline) + - HNSW: ~3.7 GB (20% overhead for graph) + - SVS-VAMANA: 1.6-3.1 GB (depends on compression) + +**Recall Quality:** + - FLAT: 100% (exact search) + - HNSW: 95-99% (tunable via ef_runtime) + - SVS-VAMANA: 90-95% (depends on compression) + +Migration Considerations +------------------------ + +**From FLAT to HNSW:** + - Straightforward migration + - Expect slight recall reduction but major speed improvement + - Tune ef_runtime to balance speed vs accuracy + +**From HNSW to SVS-VAMANA:** + - Requires Redis >= 8.2 with RediSearch >= 2.8.10 + - Change datatype to float16 or float32 if using others + - Consider compression options for memory savings + +**From SVS-VAMANA to others:** + - May need to change datatype back if using float64/bfloat16 + - HNSW provides similar performance with broader compatibility + +For complete Redis field documentation, see: https://redis.io/commands/ft.create/ \ No newline at end of file diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index b2735479..13a39507 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -1,8 +1,36 @@ """ -RedisVL Fields, FieldAttributes, and Enums - -Reference Redis search source documentation as needed: https://redis.io/commands/ft.create/ -Reference Redis vector search documentation as needed: https://redis.io/docs/interact/search-and-query/advanced-concepts/vectors/ +RedisVL Schema Fields and Attributes + +This module defines field types and their attributes for creating Redis search indices. + +Field Types: + - TextField: Full-text search with stemming, phonetic matching + - TagField: Exact-match categorical data (tags, categories, IDs) + - NumericField: Numeric values for range queries and sorting + - GeoField: Geographic coordinates for location-based search + - VectorField: Vector embeddings for semantic similarity search + - FlatVectorField: Brute-force exact search (100% recall) + - HNSWVectorField: Approximate nearest neighbor search (fast, high recall) + - SVSVectorField: Compressed vector search with memory savings + +Common Vector Field Attributes (all algorithms): + - dims: Number of dimensions in the vector (e.g., 768, 1536) + - algorithm: Indexing algorithm ('flat', 'hnsw', or 'svs-vamana') + - datatype: Float precision ('float16', 'float32', 'float64', 'bfloat16') + Note: SVS-VAMANA only supports 'float16' and 'float32' + - distance_metric: Similarity metric ('COSINE', 'L2', 'IP') + - initial_cap: Initial capacity hint for memory allocation (optional) + - index_missing: Allow searching for documents without this field (optional) + +Algorithm-Specific Parameters: + - FLAT: block_size (memory management for dynamic indices) + - HNSW: m, ef_construction, ef_runtime, epsilon (graph tuning) + - SVS-VAMANA: graph_max_degree, construction_window_size, search_window_size, + compression, reduce, training_threshold (graph + compression) + +References: + - Redis FT.CREATE: https://redis.io/commands/ft.create/ + - Vector Search: https://redis.io/docs/interact/search-and-query/advanced-concepts/vectors/ """ from enum import Enum @@ -126,12 +154,12 @@ class GeoFieldAttributes(BaseFieldAttributes): class BaseVectorFieldAttributes(BaseModel): - """Base vector field attributes shared by both FLAT and HNSW fields""" + """Base vector field attributes shared by FLAT, HNSW, and SVS-VAMANA fields""" dims: int """Dimensionality of the vector embeddings field""" algorithm: VectorIndexAlgorithm - """The indexing algorithm for the field: HNSW or FLAT""" + """The indexing algorithm for the field: FLAT, HNSW, or SVS-VAMANA""" datatype: VectorDataType = Field(default=VectorDataType.FLOAT32) """The float datatype for the vector embeddings""" distance_metric: VectorDistanceMetric = Field(default=VectorDistanceMetric.COSINE) @@ -163,54 +191,63 @@ def field_data(self) -> Dict[str, Any]: class FlatVectorFieldAttributes(BaseVectorFieldAttributes): - """FLAT vector field attributes""" + """FLAT vector field attributes for exact nearest neighbor search.""" algorithm: Literal[VectorIndexAlgorithm.FLAT] = VectorIndexAlgorithm.FLAT - """The indexing algorithm for the vector field""" + """The indexing algorithm (fixed as 'flat')""" + block_size: Optional[int] = None - """Block size to hold amount of vectors in a contiguous array. This is useful when the index is dynamic with respect to addition and deletion""" + """Block size for processing (optional) - improves batch operation throughput""" class HNSWVectorFieldAttributes(BaseVectorFieldAttributes): - """HNSW vector field attributes""" + """HNSW vector field attributes for approximate nearest neighbor search.""" algorithm: Literal[VectorIndexAlgorithm.HNSW] = VectorIndexAlgorithm.HNSW - """The indexing algorithm for the vector field""" + """The indexing algorithm (fixed as 'hnsw')""" m: int = Field(default=16) - """Number of max outgoing edges for each graph node in each layer""" + """Max outgoing edges per node in each layer (default: 16, range: 8-64)""" + ef_construction: int = Field(default=200) - """Number of max allowed potential outgoing edges candidates for each node in the graph during build time""" + """Max edge candidates during build time (default: 200, range: 100-800)""" + ef_runtime: int = Field(default=10) - """Number of maximum top candidates to hold during KNN search""" + """Max top candidates during search (default: 10) - primary tuning parameter""" + epsilon: float = Field(default=0.01) - """Relative factor that sets the boundaries in which a range query may search for candidates""" + """Range search boundary factor (default: 0.01)""" class SVSVectorFieldAttributes(BaseVectorFieldAttributes): - """SVS-VAMANA vector field attributes with optional compression support""" + """SVS-VAMANA vector field attributes with compression support.""" algorithm: Literal[VectorIndexAlgorithm.SVS_VAMANA] = ( VectorIndexAlgorithm.SVS_VAMANA ) """The indexing algorithm for the vector field""" - # SVS-VAMANA graph parameters + # Graph Construction Parameters graph_max_degree: int = Field(default=40) - """Maximum degree of the Vamana graph (number of edges per node)""" + """Max edges per node (default: 40) - affects recall vs memory""" + construction_window_size: int = Field(default=250) - """Size of the candidate list during graph construction""" + """Build-time candidates (default: 250) - affects quality vs build time""" + search_window_size: int = Field(default=20) - """Size of the candidate list during search""" + """Search candidates (default: 20) - primary tuning parameter""" + epsilon: float = Field(default=0.01) - """Relative factor for range query boundaries""" + """Range query boundary factor (default: 0.01)""" - # SVS-VAMANA compression parameters (optional, to be implemented) + # Compression Parameters compression: Optional[CompressionType] = None - """Vector compression type (LVQ or LeanVec)""" + """Vector compression: LVQ4, LVQ8, LeanVec4x8, LeanVec8x8""" + reduce: Optional[int] = None - """Reduced dimensionality for LeanVec compression (must be < dims)""" + """Dimensionality reduction for LeanVec types (must be < dims)""" + training_threshold: Optional[int] = None - """Minimum number of vectors required before compression training""" + """Min vectors before compression training (default: 10,240)""" @model_validator(mode="after") def validate_svs_params(self): @@ -451,7 +488,7 @@ def as_redis_field(self) -> RedisField: class FlatVectorField(BaseField): - """Vector field with a FLAT index (brute force nearest neighbors search)""" + """Vector field with FLAT (brute-force) indexing for exact nearest neighbor search.""" type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR attrs: FlatVectorFieldAttributes @@ -466,7 +503,7 @@ def as_redis_field(self) -> RedisField: class HNSWVectorField(BaseField): - """Vector field with an HNSW index (approximate nearest neighbors search)""" + """Vector field with HNSW (Hierarchical Navigable Small World) indexing for approximate nearest neighbor search.""" type: Literal["vector"] = "vector" attrs: HNSWVectorFieldAttributes @@ -487,7 +524,7 @@ def as_redis_field(self) -> RedisField: class SVSVectorField(BaseField): - """Vector field with an SVS-VAMANA index""" + """Vector field with SVS-VAMANA indexing and compression for memory-efficient approximate nearest neighbor search.""" type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR attrs: SVSVectorFieldAttributes From c316bd34d110cb444ee4f022267dee301900e380 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 15 Oct 2025 12:33:34 -0400 Subject: [PATCH 17/29] Remove migration class, will move it to resources later. --- redisvl/utils/__init__.py | 3 +- redisvl/utils/migration.py | 427 ------------------------------------- 2 files changed, 1 insertion(+), 429 deletions(-) delete mode 100644 redisvl/utils/migration.py diff --git a/redisvl/utils/__init__.py b/redisvl/utils/__init__.py index 8add03e3..639e6e2a 100644 --- a/redisvl/utils/__init__.py +++ b/redisvl/utils/__init__.py @@ -1,4 +1,3 @@ from redisvl.utils.compression import CompressionAdvisor -from redisvl.utils.migration import IndexMigrator -__all__ = ["CompressionAdvisor", "IndexMigrator"] +__all__ = ["CompressionAdvisor"] diff --git a/redisvl/utils/migration.py b/redisvl/utils/migration.py deleted file mode 100644 index fe7f621f..00000000 --- a/redisvl/utils/migration.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -Utilities for migrating Redis indices to SVS-VAMANA with compression. - -This module provides tools to migrate existing FLAT or HNSW indices to -SVS-VAMANA indices with compression, enabling significant memory savings -while maintaining search quality. -""" - -import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union - -from redisvl.exceptions import RedisModuleVersionError -from redisvl.redis.connection import supports_svs -from redisvl.redis.constants import SVS_MIN_REDIS_VERSION -from redisvl.schema import IndexSchema -from redisvl.utils.compression import CompressionAdvisor -from redisvl.utils.log import get_logger - -# Avoid circular imports by using TYPE_CHECKING -if TYPE_CHECKING: - from redisvl.index import AsyncSearchIndex, SearchIndex - from redisvl.query import FilterQuery - from redisvl.query.filter import FilterExpression - -logger = get_logger(__name__) - - -class IndexMigrator: - """Helper class to migrate indices to SVS-VAMANA with compression. - - This class provides utilities to migrate existing FLAT or HNSW vector indices - to SVS-VAMANA indices with compression, enabling significant memory savings. - - Example: - .. code-block:: python - - from redisvl.index import SearchIndex - from redisvl.utils import IndexMigrator - - # Load existing index - old_index = SearchIndex.from_existing("my_flat_index") - - # Migrate to SVS-VAMANA with LVQ compression - new_index = IndexMigrator.migrate_to_svs( - old_index, - compression="LVQ4x4", - batch_size=1000 - ) - - print(f"Migrated {new_index.info()['num_docs']} documents") - """ - - @staticmethod - def migrate_to_svs( - old_index: "SearchIndex", - new_index_name: Optional[str] = None, - compression: Optional[str] = None, - reduce: Optional[int] = None, - batch_size: int = 1000, - overwrite: bool = False, - progress_callback: Optional[Callable[[int, int], None]] = None, - ) -> "SearchIndex": - """Migrate an existing index to SVS-VAMANA with compression. - - This method creates a new SVS-VAMANA index and copies all data from the - old index in batches. The old index is not modified or deleted. - - Args: - old_index: The existing SearchIndex to migrate from. - new_index_name: Name for the new index. If None, uses "{old_name}_svs". - compression: Compression type (LVQ4, LVQ4x4, LVQ4x8, LVQ8, LeanVec4x8, LeanVec8x8). - If None, uses CompressionAdvisor to recommend based on dimensions. - reduce: Dimensionality reduction parameter for LeanVec compression. - Required for LeanVec compression types. - batch_size: Number of documents to migrate per batch. Default: 1000. - overwrite: Whether to overwrite the new index if it exists. Default: False. - progress_callback: Optional callback function(current, total) for progress tracking. - - Returns: - SearchIndex: The new SVS-VAMANA index with migrated data. - - Raises: - RedisModuleVersionError: If Redis version doesn't support SVS-VAMANA. - ValueError: If the old index has no vector fields or invalid parameters. - - Example: - .. code-block:: python - - def progress(current, total): - print(f"Migrated {current}/{total} documents") - - new_index = IndexMigrator.migrate_to_svs( - old_index, - compression="LVQ4x4", - batch_size=500, - progress_callback=progress - ) - """ - # Import here to avoid circular imports - from redisvl.index import SearchIndex - from redisvl.query import FilterQuery - from redisvl.query.filter import FilterExpression - - # Check SVS-VAMANA support - if not supports_svs(old_index._redis_client): - raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) - - # Find vector fields in the old schema - vector_fields = [ - (name, field) - for name, field in old_index.schema.fields.items() - if hasattr(field, "attrs") and hasattr(field.attrs, "algorithm") - ] - - if not vector_fields: - raise ValueError("Old index has no vector fields to migrate") - - # Create new schema based on old schema - new_schema_dict = old_index.schema.to_dict() - - # Update index name - if new_index_name is None: - new_index_name = f"{old_index.name}_svs" - new_schema_dict["index"]["name"] = new_index_name - new_schema_dict["index"]["prefix"] = new_index_name - - # Update vector fields to use SVS-VAMANA - for field_dict in new_schema_dict["fields"]: - if field_dict["type"] == "vector": - attrs = field_dict.get("attrs", {}) - dims = attrs.get("dims") - - if dims is None: - raise ValueError(f"Vector field '{field_dict['name']}' has no dims") - - # Use CompressionAdvisor if compression not specified - if compression is None: - config = CompressionAdvisor.recommend( - dims=dims, - priority="balanced", - datatype=attrs.get("datatype", "float32"), - ) - compression = config["compression"] - if "reduce" in config and reduce is None: - reduce = config["reduce"] - logger.info( - f"CompressionAdvisor recommended: {compression} " - f"(reduce={reduce}) for {dims} dims" - ) - - # Update to SVS-VAMANA - attrs["algorithm"] = "svs-vamana" - attrs["compression"] = compression - - if reduce is not None: - attrs["reduce"] = reduce - - # Set default SVS parameters if not present - if "graph_max_degree" not in attrs: - attrs["graph_max_degree"] = 40 - if "construction_window_size" not in attrs: - attrs["construction_window_size"] = 250 - if "search_window_size" not in attrs: - attrs["search_window_size"] = 20 - # Set a low training threshold for small datasets - # Default is 10240, minimum is 1024 (DEFAULT_BLOCK_SIZE) - if "training_threshold" not in attrs: - attrs["training_threshold"] = 1024 - - # Create new index - new_schema = IndexSchema.from_dict(new_schema_dict) - new_index = SearchIndex(schema=new_schema, redis_client=old_index._redis_client) - new_index.create(overwrite=overwrite) - - logger.info(f"Created new SVS-VAMANA index: {new_index_name}") - - # Get total document count - old_info = old_index.info() - total_docs = int(old_info.get("num_docs", 0)) - - if total_docs == 0: - logger.warning("Old index has no documents to migrate") - return new_index - - logger.info(f"Migrating {total_docs} documents in batches of {batch_size}") - - # Migrate data in batches using pagination - migrated_count = 0 - query = FilterQuery( - filter_expression=FilterExpression("*"), - return_fields=list(old_index.schema.fields.keys()), - ) - - for batch in old_index.paginate(query, page_size=batch_size): - if batch: - # The 'id' field contains the full Redis key (e.g., "prefix:ulid") - # We need to preserve the document ID part for the new index - batch_keys = [] - batch_docs = [] - - for doc in batch: - # Get the full Redis key from the id field - full_key = doc.get("id", "") - # Extract the document ID (everything after the prefix) - # Split by the key separator and take the last part - doc_id = full_key.split(old_index.schema.index.key_separator)[-1] - - # Create a copy of the document without the id field - # (the id field is metadata, not actual document data) - doc_copy = {k: v for k, v in doc.items() if k != "id"} - - batch_keys.append(new_index.key(doc_id)) - batch_docs.append(doc_copy) - - # Load batch to new index with explicit keys to preserve IDs - new_index.load(batch_docs, keys=batch_keys) - migrated_count += len(batch) - - # Call progress callback if provided - if progress_callback: - progress_callback(migrated_count, total_docs) - - logger.debug(f"Migrated {migrated_count}/{total_docs} documents") - - logger.info(f"Migration complete: {migrated_count} documents migrated") - - # Verify migration by checking Redis keys (not index count) - # Note: SVS-VAMANA indices have a training_threshold (default 10240) - # Documents are written to Redis but may not be indexed until threshold is reached - new_info = new_index.info() - new_doc_count = int(new_info.get("num_docs", 0)) - - if new_doc_count != total_docs: - # Check if documents exist in Redis even if not indexed yet - client = new_index._redis_client - actual_keys = client.keys(f"{new_index.schema.index.prefix}:*") - actual_count = len(actual_keys) - - if actual_count == total_docs: - logger.info( - f"Documents written to Redis: {actual_count}/{total_docs}. " - f"Index shows {new_doc_count} (may be below training_threshold)" - ) - else: - logger.warning( - f"Document count mismatch: expected {total_docs}, " - f"got {actual_count} in Redis, {new_doc_count} in index" - ) - - return new_index - - @staticmethod - async def migrate_to_svs_async( - old_index: "AsyncSearchIndex", - new_index_name: Optional[str] = None, - compression: Optional[str] = None, - reduce: Optional[int] = None, - batch_size: int = 1000, - overwrite: bool = False, - progress_callback: Optional[Callable[[int, int], None]] = None, - ) -> "AsyncSearchIndex": - """Asynchronously migrate an existing index to SVS-VAMANA with compression. - - This is the async version of migrate_to_svs(). See migrate_to_svs() for - detailed documentation. - - Args: - old_index: The existing AsyncSearchIndex to migrate from. - new_index_name: Name for the new index. If None, uses "{old_name}_svs". - compression: Compression type. If None, uses CompressionAdvisor. - reduce: Dimensionality reduction parameter for LeanVec. - batch_size: Number of documents to migrate per batch. Default: 1000. - overwrite: Whether to overwrite the new index if it exists. Default: False. - progress_callback: Optional callback function(current, total) for progress. - - Returns: - AsyncSearchIndex: The new SVS-VAMANA index with migrated data. - - Example: - .. code-block:: python - - async def progress(current, total): - print(f"Migrated {current}/{total} documents") - - new_index = await IndexMigrator.migrate_to_svs_async( - old_index, - compression="LVQ4x4", - progress_callback=progress - ) - """ - # Import here to avoid circular imports - from redisvl.index import AsyncSearchIndex - from redisvl.query import FilterQuery - from redisvl.query.filter import FilterExpression - from redisvl.redis.connection import supports_svs_async - - # Check SVS-VAMANA support - client = await old_index._get_client() - if not await supports_svs_async(client): - raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) - - # Find vector fields - vector_fields = [ - (name, field) - for name, field in old_index.schema.fields.items() - if hasattr(field, "attrs") and hasattr(field.attrs, "algorithm") - ] - - if not vector_fields: - raise ValueError("Old index has no vector fields to migrate") - - # Create new schema - new_schema_dict = old_index.schema.to_dict() - - if new_index_name is None: - new_index_name = f"{old_index.name}_svs" - new_schema_dict["index"]["name"] = new_index_name - new_schema_dict["index"]["prefix"] = new_index_name - - # Update vector fields - for field_dict in new_schema_dict["fields"]: - if field_dict["type"] == "vector": - attrs = field_dict.get("attrs", {}) - dims = attrs.get("dims") - - if dims is None: - raise ValueError(f"Vector field '{field_dict['name']}' has no dims") - - if compression is None: - config = CompressionAdvisor.recommend( - dims=dims, - priority="balanced", - datatype=attrs.get("datatype", "float32"), - ) - compression = config["compression"] - if "reduce" in config and reduce is None: - reduce = config["reduce"] - logger.info( - f"CompressionAdvisor recommended: {compression} " - f"(reduce={reduce}) for {dims} dims" - ) - - attrs["algorithm"] = "svs-vamana" - attrs["compression"] = compression - - if reduce is not None: - attrs["reduce"] = reduce - - if "graph_max_degree" not in attrs: - attrs["graph_max_degree"] = 40 - if "construction_window_size" not in attrs: - attrs["construction_window_size"] = 250 - if "search_window_size" not in attrs: - attrs["search_window_size"] = 20 - if "training_threshold" not in attrs: - attrs["training_threshold"] = 1024 - - # Create new index - new_schema = IndexSchema.from_dict(new_schema_dict) - new_index = AsyncSearchIndex(schema=new_schema, redis_client=client) - await new_index.create(overwrite=overwrite) - - logger.info(f"Created new SVS-VAMANA index: {new_index_name}") - - # Get total document count - old_info = await old_index.info() - total_docs = int(old_info.get("num_docs", 0)) - - if total_docs == 0: - logger.warning("Old index has no documents to migrate") - return new_index - - logger.info(f"Migrating {total_docs} documents in batches of {batch_size}") - - # Migrate data in batches - migrated_count = 0 - query = FilterQuery( - filter_expression=FilterExpression("*"), - return_fields=list(old_index.schema.fields.keys()), - ) - - async for batch in old_index.paginate(query, page_size=batch_size): - if batch: - # Extract document IDs from full Redis keys - batch_keys = [] - batch_docs = [] - - for doc in batch: - full_key = doc.get("id", "") - doc_id = full_key.split(old_index.schema.index.key_separator)[-1] - doc_copy = {k: v for k, v in doc.items() if k != "id"} - - batch_keys.append(new_index.key(doc_id)) - batch_docs.append(doc_copy) - - # Load batch to new index with explicit keys - await new_index.load(batch_docs, keys=batch_keys) - migrated_count += len(batch) - - if progress_callback: - progress_callback(migrated_count, total_docs) - - logger.debug(f"Migrated {migrated_count}/{total_docs} documents") - - logger.info(f"Migration complete: {migrated_count} documents migrated") - - # Verify migration by checking Redis keys - new_info = await new_index.info() - new_doc_count = int(new_info.get("num_docs", 0)) - - if new_doc_count != total_docs: - # Check if documents exist in Redis even if not indexed yet - actual_keys = await client.keys(f"{new_index.schema.index.prefix}:*") - actual_count = len(actual_keys) - - if actual_count == total_docs: - logger.info( - f"Documents written to Redis: {actual_count}/{total_docs}. " - f"Index shows {new_doc_count} (may be below training_threshold)" - ) - else: - logger.warning( - f"Document count mismatch: expected {total_docs}, " - f"got {actual_count} in Redis, {new_doc_count} in index" - ) - - return new_index From 5f50454829993ae08ca7f7c82e76d29a082fd646 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 15 Oct 2025 12:50:38 -0400 Subject: [PATCH 18/29] Update --- docs/api/schema.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 18809bed..113af2b0 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -96,7 +96,7 @@ Each field type supports specific attributes that customize its behavior. Below - `flat`: Brute-force exact search. 100% recall, slower for large datasets. Best for <10K vectors. - `hnsw`: Graph-based approximate search. Fast with high recall (95-99%). Best for general use. - - `svs-vamana`: Compressed approximate search. Memory-efficient with compression. Best for large datasets on Intel hardware. + - `svs-vamana`: SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. .. note:: For detailed algorithm comparison and selection guidance, see :ref:`vector-algorithm-comparison`. @@ -169,7 +169,7 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with **SVS-VAMANA Vector Field Specific Attributes**: -SVS-VAMANA - High-performance search with optional compression. **Best for large datasets (>100K vectors) on Intel hardware with memory constraints.** +SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. **Best for large datasets (>100K vectors) on Intel hardware with memory constraints.** .. dropdown:: When to use SVS-VAMANA & Detailed Guide :color: info From 6273f350bca26ba3e0c45c57e2ae81b9641eed7b Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 15 Oct 2025 13:38:26 -0400 Subject: [PATCH 19/29] Fix breaking tests --- redisvl/schema/fields.py | 3 ++- tests/unit/test_error_handling.py | 2 ++ tests/unit/test_fields.py | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index 13a39507..27444130 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -26,7 +26,8 @@ - FLAT: block_size (memory management for dynamic indices) - HNSW: m, ef_construction, ef_runtime, epsilon (graph tuning) - SVS-VAMANA: graph_max_degree, construction_window_size, search_window_size, - compression, reduce, training_threshold (graph + compression) + compression, reduce, training_threshold (VAMANA graph algorithm + with Intel hardware optimization and vector compression) References: - Redis FT.CREATE: https://redis.io/commands/ft.create/ diff --git a/tests/unit/test_error_handling.py b/tests/unit/test_error_handling.py index 8c31e8ac..478743f9 100644 --- a/tests/unit/test_error_handling.py +++ b/tests/unit/test_error_handling.py @@ -38,6 +38,7 @@ def test_redis_error_in_create_method(self, mock_validate): # Create a mock schema schema = Mock(spec=IndexSchema) schema.redis_fields = ["test_field"] + schema.fields = {} # Dict[str, BaseField] for compatibility schema.index = Mock() schema.index.name = "test_index" schema.index.prefix = "test:" @@ -68,6 +69,7 @@ def test_unexpected_error_in_create_method(self, mock_validate): # Create a mock schema schema = Mock(spec=IndexSchema) schema.redis_fields = ["test_field"] + schema.fields = {} # Dict[str, BaseField] for compatibility schema.index = Mock() schema.index.name = "test_index" schema.index.prefix = "test:" diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index 7211102f..eeeda604 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -707,7 +707,6 @@ def test_check_svs_support_raises_error(): error_msg = str(exc_info.value) assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg - assert "Redis 7.2.4" in error_msg def test_check_svs_support_passes(): From 7996aa90c315aea41c5c01c2e5ace8912e91dceb Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 19:47:51 -0400 Subject: [PATCH 20/29] rebasing --- docs/user_guide/09_svs_vamana.ipynb | 825 ++++++++++++++++++++++++++++ docs/user_guide/index.md | 1 + 2 files changed, 826 insertions(+) create mode 100644 docs/user_guide/09_svs_vamana.ipynb diff --git a/docs/user_guide/09_svs_vamana.ipynb b/docs/user_guide/09_svs_vamana.ipynb new file mode 100644 index 00000000..16a933e8 --- /dev/null +++ b/docs/user_guide/09_svs_vamana.ipynb @@ -0,0 +1,825 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SVS-VAMANA Vector Search\n", + "\n", + "In this notebook, we will explore SVS-VAMANA, a high-performance vector search algorithm that provides fast approximate nearest neighbor search with vector compression capabilities.\n", + "\n", + "SVS-VAMANA offers:\n", + "- **Fast approximate nearest neighbor search** using graph-based algorithms\n", + "- **Vector compression** (LVQ, LeanVec) with up to 87.5% memory savings\n", + "- **Dimensionality reduction** (optional, with LeanVec)\n", + "- **Automatic performance optimization** through CompressionAdvisor\n", + "\n", + "## Table of Contents\n", + "\n", + "1. [Prerequisites](#Prerequisites)\n", + "2. [Quick Start with CompressionAdvisor](#Quick-Start-with-CompressionAdvisor)\n", + "3. [Creating an SVS-VAMANA Index](#Creating-an-SVS-VAMANA-Index)\n", + "4. [Loading Sample Data](#Loading-Sample-Data)\n", + "5. [Performing Vector Searches](#Performing-Vector-Searches)\n", + "6. [Understanding Compression Types](#Understanding-Compression-Types)\n", + "7. [Hybrid Queries with SVS-VAMANA](#Hybrid-Queries-with-SVS-VAMANA)\n", + "8. [Performance Monitoring](#Performance-Monitoring)\n", + "9. [Manual Configuration (Advanced)](#Manual-Configuration-(Advanced))\n", + "10. [Best Practices and Tips](#Best-Practices-and-Tips)\n", + "11. [Cleanup](#Cleanup)\n", + "\n", + "---\n", + "\n", + "## Prerequisites\n", + "\n", + "Before running this notebook, ensure you have:\n", + "1. Installed `redisvl` and have that environment active for this notebook\n", + "2. A running Redis Stack instance with:\n", + " - Redis >= 8.2.0\n", + " - RediSearch >= 2.8.10\n", + "\n", + "For example, you can run Redis Stack locally with Docker:\n", + "\n", + "```bash\n", + "docker run -d -p 6379:6379 -p 8001:8001 redis/redis-stack:latest\n", + "```\n", + "\n", + "**Note:** SVS-VAMANA only supports FLOAT16 and FLOAT32 datatypes." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:08.569435Z", + "iopub.status.busy": "2025-10-15T21:32:08.569310Z", + "iopub.status.idle": "2025-10-15T21:32:08.696705Z", + "shell.execute_reply": "2025-10-15T21:32:08.696395Z" + } + }, + "outputs": [], + "source": [ + "# Import necessary modules\n", + "import numpy as np\n", + "from redisvl.index import SearchIndex\n", + "from redisvl.query import VectorQuery\n", + "from redisvl.utils import CompressionAdvisor\n", + "from redisvl.redis.utils import array_to_buffer\n", + "\n", + "# Set random seed for reproducible results\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:08.697903Z", + "iopub.status.busy": "2025-10-15T21:32:08.697811Z", + "iopub.status.idle": "2025-10-15T21:32:08.699342Z", + "shell.execute_reply": "2025-10-15T21:32:08.699114Z" + } + }, + "outputs": [], + "source": [ + "# Redis connection\n", + "REDIS_URL = \"redis://localhost:6379\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Start with CompressionAdvisor\n", + "\n", + "The easiest way to get started with SVS-VAMANA is using the `CompressionAdvisor` utility, which automatically recommends optimal configuration based on your vector dimensions and performance priorities." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:08.700313Z", + "iopub.status.busy": "2025-10-15T21:32:08.700251Z", + "iopub.status.idle": "2025-10-15T21:32:08.702206Z", + "shell.execute_reply": "2025-10-15T21:32:08.702012Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Recommended Configuration:\n", + " algorithm: svs-vamana\n", + " datatype: float16\n", + " graph_max_degree: 64\n", + " construction_window_size: 300\n", + " compression: LeanVec4x8\n", + " reduce: 512\n", + " search_window_size: 30\n", + "\n", + "Estimated Memory Savings: 81.2%\n" + ] + } + ], + "source": [ + "# Get recommended configuration for common embedding dimensions\n", + "dims = 1024 # Common embedding dimensions (works reliably with SVS-VAMANA)\n", + "\n", + "config = CompressionAdvisor.recommend(\n", + " dims=dims,\n", + " priority=\"balanced\" # Options: \"memory\", \"speed\", \"balanced\"\n", + ")\n", + "\n", + "print(\"Recommended Configuration:\")\n", + "for key, value in config.items():\n", + " print(f\" {key}: {value}\")\n", + "\n", + "# Estimate memory savings\n", + "savings = CompressionAdvisor.estimate_memory_savings(\n", + " config[\"compression\"],\n", + " dims,\n", + " config.get(\"reduce\")\n", + ")\n", + "print(f\"\\nEstimated Memory Savings: {savings}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating an SVS-VAMANA Index\n", + "\n", + "Let's create an index using the recommended configuration. We'll use a simple schema with text content and vector embeddings." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:08.717380Z", + "iopub.status.busy": "2025-10-15T21:32:08.717285Z", + "iopub.status.idle": "2025-10-15T21:32:08.723852Z", + "shell.execute_reply": "2025-10-15T21:32:08.723644Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Created SVS-VAMANA index: svs_demo\n", + " Algorithm: svs-vamana\n", + " Compression: LeanVec4x8\n", + " Dimensions: 1024\n", + " Reduced to: 512 dimensions\n" + ] + } + ], + "source": [ + "# Create index schema with recommended SVS-VAMANA configuration\n", + "schema = {\n", + " \"index\": {\n", + " \"name\": \"svs_demo\",\n", + " \"prefix\": \"doc\",\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"content\", \"type\": \"text\"},\n", + " {\"name\": \"category\", \"type\": \"tag\"},\n", + " {\n", + " \"name\": \"embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": dims,\n", + " **config, # Use the recommended configuration\n", + " \"distance_metric\": \"cosine\"\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "# Create the index\n", + "index = SearchIndex.from_dict(schema, redis_url=REDIS_URL)\n", + "index.create(overwrite=True)\n", + "\n", + "print(f\"āœ… Created SVS-VAMANA index: {index.name}\")\n", + "print(f\" Algorithm: {config['algorithm']}\")\n", + "print(f\" Compression: {config['compression']}\")\n", + "print(f\" Dimensions: {dims}\")\n", + "if 'reduce' in config:\n", + " print(f\" Reduced to: {config['reduce']} dimensions\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Sample Data\n", + "\n", + "Let's create some sample documents with embeddings to demonstrate SVS-VAMANA search capabilities." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:08.724966Z", + "iopub.status.busy": "2025-10-15T21:32:08.724905Z", + "iopub.status.idle": "2025-10-15T21:32:10.740249Z", + "shell.execute_reply": "2025-10-15T21:32:10.739174Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating vectors with 512 dimensions (reduced from 1024 if applicable)\n", + "āœ… Loaded 10 documents into the index\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Index now contains 0 documents\n" + ] + } + ], + "source": [ + "# Generate sample data\n", + "sample_documents = [\n", + " {\"content\": \"Machine learning algorithms for data analysis\", \"category\": \"technology\"},\n", + " {\"content\": \"Natural language processing and text understanding\", \"category\": \"technology\"},\n", + " {\"content\": \"Computer vision and image recognition systems\", \"category\": \"technology\"},\n", + " {\"content\": \"Delicious pasta recipes from Italy\", \"category\": \"food\"},\n", + " {\"content\": \"Traditional French cooking techniques\", \"category\": \"food\"},\n", + " {\"content\": \"Healthy meal planning and nutrition\", \"category\": \"food\"},\n", + " {\"content\": \"Travel guide to European destinations\", \"category\": \"travel\"},\n", + " {\"content\": \"Adventure hiking in mountain regions\", \"category\": \"travel\"},\n", + " {\"content\": \"Cultural experiences in Asian cities\", \"category\": \"travel\"},\n", + " {\"content\": \"Financial planning for retirement\", \"category\": \"finance\"},\n", + "]\n", + "\n", + "# Generate random embeddings for demonstration\n", + "# In practice, you would use a real embedding model\n", + "data_to_load = []\n", + "\n", + "# Use reduced dimensions if LeanVec compression is applied\n", + "vector_dims = config.get(\"reduce\", dims)\n", + "print(f\"Creating vectors with {vector_dims} dimensions (reduced from {dims} if applicable)\")\n", + "\n", + "for i, doc in enumerate(sample_documents):\n", + " # Create a random vector with some category-based clustering\n", + " base_vector = np.random.random(vector_dims).astype(np.float32)\n", + " \n", + " # Add some category-based similarity (optional, for demo purposes)\n", + " category_offset = hash(doc[\"category\"]) % 100 / 1000.0\n", + " base_vector[0] += category_offset\n", + " \n", + " # Convert to the datatype specified in config\n", + " if config[\"datatype\"] == \"float16\":\n", + " base_vector = base_vector.astype(np.float16)\n", + " \n", + " data_to_load.append({\n", + " \"content\": doc[\"content\"],\n", + " \"category\": doc[\"category\"],\n", + " \"embedding\": array_to_buffer(base_vector, dtype=config[\"datatype\"])\n", + " })\n", + "\n", + "# Load data into the index\n", + "index.load(data_to_load)\n", + "print(f\"āœ… Loaded {len(data_to_load)} documents into the index\")\n", + "\n", + "# Wait a moment for indexing to complete\n", + "import time\n", + "time.sleep(2)\n", + "\n", + "# Verify the data was loaded\n", + "info = index.info()\n", + "print(f\" Index now contains {info.get('num_docs', 0)} documents\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Performing Vector Searches\n", + "\n", + "Now let's perform some vector similarity searches using our SVS-VAMANA index." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.744612Z", + "iopub.status.busy": "2025-10-15T21:32:10.744310Z", + "iopub.status.idle": "2025-10-15T21:32:10.751707Z", + "shell.execute_reply": "2025-10-15T21:32:10.751275Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Vector Search Results:\n", + "==================================================\n" + ] + } + ], + "source": [ + "# Create a query vector (in practice, this would be an embedding of your query text)\n", + "# Important: Query vector must match the index datatype and dimensions\n", + "vector_dims = config.get(\"reduce\", dims)\n", + "if config[\"datatype\"] == \"float16\":\n", + " query_vector = np.random.random(vector_dims).astype(np.float16)\n", + "else:\n", + " query_vector = np.random.random(vector_dims).astype(np.float32)\n", + "\n", + "# Perform a vector similarity search\n", + "query = VectorQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " num_results=5\n", + ")\n", + "\n", + "results = index.query(query)\n", + "\n", + "print(\"šŸ” Vector Search Results:\")\n", + "print(\"=\" * 50)\n", + "for i, result in enumerate(results, 1):\n", + " distance = result.get('vector_distance', 'N/A')\n", + " print(f\"{i}. [{result['category']}] {result['content']}\")\n", + " print(f\" Distance: {distance:.4f}\" if isinstance(distance, (int, float)) else f\" Distance: {distance}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Understanding Compression Types\n", + "\n", + "SVS-VAMANA supports different compression algorithms that trade off between memory usage and search quality. Let's explore the available options." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.753759Z", + "iopub.status.busy": "2025-10-15T21:32:10.753565Z", + "iopub.status.idle": "2025-10-15T21:32:10.757377Z", + "shell.execute_reply": "2025-10-15T21:32:10.757018Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Compression Recommendations for Different Priorities:\n", + "============================================================\n", + "\n", + "MEMORY Priority:\n", + " Compression: LeanVec4x8\n", + " Datatype: float16\n", + " Dimensionality reduction: 1024 ā†’ 512\n", + " Search window size: 20\n", + " Memory savings: 81.2%\n", + "\n", + "SPEED Priority:\n", + " Compression: LeanVec4x8\n", + " Datatype: float16\n", + " Dimensionality reduction: 1024 ā†’ 256\n", + " Search window size: 40\n", + " Memory savings: 90.6%\n", + "\n", + "BALANCED Priority:\n", + " Compression: LeanVec4x8\n", + " Datatype: float16\n", + " Dimensionality reduction: 1024 ā†’ 512\n", + " Search window size: 30\n", + " Memory savings: 81.2%\n" + ] + } + ], + "source": [ + "# Compare different compression priorities\n", + "print(\"Compression Recommendations for Different Priorities:\")\n", + "print(\"=\" * 60)\n", + "\n", + "priorities = [\"memory\", \"speed\", \"balanced\"]\n", + "for priority in priorities:\n", + " config = CompressionAdvisor.recommend(dims=dims, priority=priority)\n", + " savings = CompressionAdvisor.estimate_memory_savings(\n", + " config[\"compression\"],\n", + " dims,\n", + " config.get(\"reduce\")\n", + " )\n", + " \n", + " print(f\"\\n{priority.upper()} Priority:\")\n", + " print(f\" Compression: {config['compression']}\")\n", + " print(f\" Datatype: {config['datatype']}\")\n", + " if \"reduce\" in config:\n", + " print(f\" Dimensionality reduction: {dims} ā†’ {config['reduce']}\")\n", + " print(f\" Search window size: {config['search_window_size']}\")\n", + " print(f\" Memory savings: {savings}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compression Types Explained\n", + "\n", + "SVS-VAMANA offers several compression algorithms:\n", + "\n", + "### LVQ (Learned Vector Quantization)\n", + "- **LVQ4**: 4 bits per dimension (87.5% memory savings)\n", + "- **LVQ4x4**: 8 bits per dimension (75% memory savings)\n", + "- **LVQ4x8**: 12 bits per dimension (62.5% memory savings)\n", + "- **LVQ8**: 8 bits per dimension (75% memory savings)\n", + "\n", + "### LeanVec (Compression + Dimensionality Reduction)\n", + "- **LeanVec4x8**: 12 bits per dimension + dimensionality reduction\n", + "- **LeanVec8x8**: 16 bits per dimension + dimensionality reduction\n", + "\n", + "The CompressionAdvisor automatically chooses the best compression type based on your vector dimensions and priority." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.759026Z", + "iopub.status.busy": "2025-10-15T21:32:10.758895Z", + "iopub.status.idle": "2025-10-15T21:32:10.762424Z", + "shell.execute_reply": "2025-10-15T21:32:10.762083Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Memory Savings by Vector Dimension:\n", + "==================================================\n", + "Dims Compression Savings Strategy\n", + "--------------------------------------------------\n", + "384 LVQ4x4 75.0% LVQ\n", + "768 LVQ4x4 75.0% LVQ\n", + "1024 LeanVec4x8 81.2% LeanVec\n", + "1536 LeanVec4x8 81.2% LeanVec\n", + "3072 LeanVec4x8 81.2% LeanVec\n" + ] + } + ], + "source": [ + "# Demonstrate compression savings for different vector dimensions\n", + "test_dimensions = [384, 768, 1024, 1536, 3072]\n", + "\n", + "print(\"Memory Savings by Vector Dimension:\")\n", + "print(\"=\" * 50)\n", + "print(f\"{'Dims':<6} {'Compression':<12} {'Savings':<8} {'Strategy'}\")\n", + "print(\"-\" * 50)\n", + "\n", + "for dims in test_dimensions:\n", + " config = CompressionAdvisor.recommend(dims=dims, priority=\"balanced\")\n", + " savings = CompressionAdvisor.estimate_memory_savings(\n", + " config[\"compression\"],\n", + " dims,\n", + " config.get(\"reduce\")\n", + " )\n", + " \n", + " strategy = \"LeanVec\" if dims >= 1024 else \"LVQ\"\n", + " print(f\"{dims:<6} {config['compression']:<12} {savings:>6.1f}% {strategy}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hybrid Queries with SVS-VAMANA\n", + "\n", + "SVS-VAMANA can be combined with other Redis search capabilities for powerful hybrid queries that filter by metadata while performing vector similarity search." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.763978Z", + "iopub.status.busy": "2025-10-15T21:32:10.763840Z", + "iopub.status.idle": "2025-10-15T21:32:10.768306Z", + "shell.execute_reply": "2025-10-15T21:32:10.768005Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ” Hybrid Search Results (Technology category only):\n", + "=======================================================\n" + ] + } + ], + "source": [ + "# Perform a hybrid search: vector similarity + category filter\n", + "hybrid_query = VectorQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " num_results=3\n", + ")\n", + "\n", + "# Add a filter to only search within \"technology\" category\n", + "hybrid_query.set_filter(\"@category:{technology}\")\n", + "\n", + "filtered_results = index.query(hybrid_query)\n", + "\n", + "print(\"šŸ” Hybrid Search Results (Technology category only):\")\n", + "print(\"=\" * 55)\n", + "for i, result in enumerate(filtered_results, 1):\n", + " distance = result.get('vector_distance', 'N/A')\n", + " print(f\"{i}. [{result['category']}] {result['content']}\")\n", + " print(f\" Distance: {distance:.4f}\" if isinstance(distance, (int, float)) else f\" Distance: {distance}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Performance Monitoring\n", + "\n", + "Let's examine the index statistics to understand the performance characteristics of our SVS-VAMANA index." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.770248Z", + "iopub.status.busy": "2025-10-15T21:32:10.770114Z", + "iopub.status.idle": "2025-10-15T21:32:10.774772Z", + "shell.execute_reply": "2025-10-15T21:32:10.774498Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ“Š Index Statistics:\n", + "==============================\n", + "Documents: 0\n", + "Vector index size: 0.00 MB\n", + "Total indexing time: 1.36 seconds\n", + "Memory efficiency calculation requires documents and vector index size > 0\n" + ] + } + ], + "source": [ + "# Get detailed index information\n", + "info = index.info()\n", + "\n", + "print(\"šŸ“Š Index Statistics:\")\n", + "print(\"=\" * 30)\n", + "print(f\"Documents: {info.get('num_docs', 0)}\")\n", + "\n", + "# Handle vector_index_sz_mb which might be a string\n", + "vector_size = info.get('vector_index_sz_mb', 0)\n", + "if isinstance(vector_size, str):\n", + " try:\n", + " vector_size = float(vector_size)\n", + " except ValueError:\n", + " vector_size = 0.0\n", + "print(f\"Vector index size: {vector_size:.2f} MB\")\n", + "\n", + "# Handle total_indexing_time which might also be a string\n", + "indexing_time = info.get('total_indexing_time', 0)\n", + "if isinstance(indexing_time, str):\n", + " try:\n", + " indexing_time = float(indexing_time)\n", + " except ValueError:\n", + " indexing_time = 0.0\n", + "print(f\"Total indexing time: {indexing_time:.2f} seconds\")\n", + "\n", + "# Calculate memory efficiency\n", + "if info.get('num_docs', 0) > 0 and vector_size > 0:\n", + " mb_per_doc = vector_size / info.get('num_docs', 1)\n", + " print(f\"Memory per document: {mb_per_doc:.4f} MB\")\n", + " \n", + " # Estimate for larger datasets\n", + " for scale in [1000, 10000, 100000]:\n", + " estimated_mb = mb_per_doc * scale\n", + " print(f\"Estimated size for {scale:,} docs: {estimated_mb:.1f} MB\")\n", + "else:\n", + " print(\"Memory efficiency calculation requires documents and vector index size > 0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Manual Configuration (Advanced)\n", + "\n", + "For advanced users who want full control over SVS-VAMANA parameters, you can manually configure the algorithm instead of using CompressionAdvisor." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.776132Z", + "iopub.status.busy": "2025-10-15T21:32:10.776038Z", + "iopub.status.idle": "2025-10-15T21:32:10.779151Z", + "shell.execute_reply": "2025-10-15T21:32:10.778875Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Manual SVS-VAMANA Configuration:\n", + "========================================\n", + " algorithm: svs-vamana\n", + " datatype: float32\n", + " distance_metric: cosine\n", + " graph_max_degree: 64\n", + " construction_window_size: 300\n", + " search_window_size: 40\n", + " compression: LVQ4x4\n", + " training_threshold: 10000\n", + "\n", + "Estimated memory savings: 75.0%\n" + ] + } + ], + "source": [ + "# Example of manual SVS-VAMANA configuration\n", + "manual_schema = {\n", + " \"index\": {\n", + " \"name\": \"svs_manual\",\n", + " \"prefix\": \"manual\",\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"content\", \"type\": \"text\"},\n", + " {\n", + " \"name\": \"embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 768,\n", + " \"algorithm\": \"svs-vamana\",\n", + " \"datatype\": \"float32\",\n", + " \"distance_metric\": \"cosine\",\n", + " \n", + " # Graph construction parameters\n", + " \"graph_max_degree\": 64, # Higher = better recall, more memory\n", + " \"construction_window_size\": 300, # Higher = better quality, slower build\n", + " \n", + " # Search parameters\n", + " \"search_window_size\": 40, # Higher = better recall, slower search\n", + " \n", + " # Compression settings\n", + " \"compression\": \"LVQ4x4\", # Choose compression type\n", + " \"training_threshold\": 10000, # Min vectors before compression training\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "print(\"Manual SVS-VAMANA Configuration:\")\n", + "print(\"=\" * 40)\n", + "vector_attrs = manual_schema[\"fields\"][1][\"attrs\"]\n", + "for key, value in vector_attrs.items():\n", + " if key != \"dims\": # Skip dims as it's obvious\n", + " print(f\" {key}: {value}\")\n", + "\n", + "# Calculate memory savings for this configuration\n", + "manual_savings = CompressionAdvisor.estimate_memory_savings(\n", + " \"LVQ4x4\", 768, None\n", + ")\n", + "print(f\"\\nEstimated memory savings: {manual_savings}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Best Practices and Tips\n", + "\n", + "### When to Use SVS-VAMANA\n", + "- **Large datasets** (>10K vectors) where memory efficiency matters\n", + "- **High-dimensional vectors** (>512 dimensions) that benefit from compression\n", + "- **Applications** that can tolerate slight recall trade-offs for speed and memory savings\n", + "\n", + "### Parameter Tuning Guidelines\n", + "- **Start with CompressionAdvisor** recommendations\n", + "- **Increase search_window_size** if you need higher recall\n", + "- **Use LeanVec** for high-dimensional vectors (ā‰„1024 dims)\n", + "- **Use LVQ** for lower-dimensional vectors (<1024 dims)\n", + "\n", + "### Performance Considerations\n", + "- **Index build time** increases with higher construction_window_size\n", + "- **Search latency** increases with higher search_window_size\n", + "- **Memory usage** decreases with more aggressive compression\n", + "- **Recall quality** may decrease with more aggressive compression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup\n", + "\n", + "Clean up the indices created in this demo." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T21:32:10.780645Z", + "iopub.status.busy": "2025-10-15T21:32:10.780541Z", + "iopub.status.idle": "2025-10-15T21:32:10.783383Z", + "shell.execute_reply": "2025-10-15T21:32:10.783148Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cleaned up svs_demo index\n", + "\n", + "šŸŽ‰ SVS-VAMANA demo completed!\n", + "\n", + "Next steps:\n", + "- Try SVS-VAMANA with your own embeddings\n", + "- Experiment with different compression settings\n", + "- Compare performance with FLAT and HNSW algorithms\n", + "- Use hybrid queries for complex search scenarios\n" + ] + } + ], + "source": [ + "# Clean up demo indices\n", + "try:\n", + " index.delete()\n", + " print(\"Cleaned up svs_demo index\")\n", + "except:\n", + " print(\"- svs_demo index was already deleted or doesn't exist\")\n", + "\n", + "# Note: The manual index wasn't created in this demo, so no need to delete it\n", + "print(\"\\nšŸŽ‰ SVS-VAMANA demo completed!\")\n", + "print(\"\\nNext steps:\")\n", + "print(\"- Try SVS-VAMANA with your own embeddings\")\n", + "print(\"- Experiment with different compression settings\")\n", + "print(\"- Compare performance with FLAT and HNSW algorithms\")\n", + "print(\"- Use hybrid queries for complex search scenarios\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 53c6a8ae..faf0e128 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -21,6 +21,7 @@ User guides provide helpful resources for using RedisVL and its different compon 06_rerankers 07_message_history 08_semantic_router +09_svs_vamana 11_advanced_queries ``` # TODO: Nitin Add the new user guide for SVS-VAMANA \ No newline at end of file From 77b1260f1fe5b0c2d2105091f1bb09ce5b25d722 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Wed, 15 Oct 2025 17:49:15 -0400 Subject: [PATCH 21/29] Updates --- docs/user_guide/09_svs_vamana.ipynb | 157 ++++++---------------------- 1 file changed, 32 insertions(+), 125 deletions(-) diff --git a/docs/user_guide/09_svs_vamana.ipynb b/docs/user_guide/09_svs_vamana.ipynb index 16a933e8..56ec4d88 100644 --- a/docs/user_guide/09_svs_vamana.ipynb +++ b/docs/user_guide/09_svs_vamana.ipynb @@ -7,15 +7,28 @@ "source": [ "# SVS-VAMANA Vector Search\n", "\n", - "In this notebook, we will explore SVS-VAMANA, a high-performance vector search algorithm that provides fast approximate nearest neighbor search with vector compression capabilities.\n", + "In this notebook, we will explore SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm), a graph-based vector search algorithm that is optimized to work with compression methods to reduce memory usage. It combines the Vamana graph algorithm with advanced compression techniques (LVQ and LeanVec) and is optimized for Intel hardware.\n", "\n", - "SVS-VAMANA offers:\n", + "**How it works**\n", + "\n", + "Vamana builds a single-layer proximity graph and prunes edges during construction based on tunable parameters, similar to HNSW but with a simpler structure. The compression methods apply per-vector normalization and scalar quantization, learning parameters directly from the data to enable fast, on-the-fly distance computations with SIMD-optimized layout Vector quantization and compression.\n", + "\n", + "\n", + "**SVS-VAMANA offers:**\n", "- **Fast approximate nearest neighbor search** using graph-based algorithms\n", "- **Vector compression** (LVQ, LeanVec) with up to 87.5% memory savings\n", "- **Dimensionality reduction** (optional, with LeanVec)\n", "- **Automatic performance optimization** through CompressionAdvisor\n", "\n", - "## Table of Contents\n", + "**Use SVS-VAMANA when:**\n", + "- Large datasets where memory is expensive\n", + "- Cloud deployments with memory-based pricing\n", + "- When 90-95% recall is acceptable\n", + "- High-dimensional vectors (>1024 dims) with LeanVec compression\n", + "\n", + "\n", + "\n", + "**Table of Contents**\n", "\n", "1. [Prerequisites](#Prerequisites)\n", "2. [Quick Start with CompressionAdvisor](#Quick-Start-with-CompressionAdvisor)\n", @@ -51,14 +64,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:08.569435Z", - "iopub.status.busy": "2025-10-15T21:32:08.569310Z", - "iopub.status.idle": "2025-10-15T21:32:08.696705Z", - "shell.execute_reply": "2025-10-15T21:32:08.696395Z" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Import necessary modules\n", @@ -75,14 +81,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:08.697903Z", - "iopub.status.busy": "2025-10-15T21:32:08.697811Z", - "iopub.status.idle": "2025-10-15T21:32:08.699342Z", - "shell.execute_reply": "2025-10-15T21:32:08.699114Z" - } - }, + "metadata": {}, "outputs": [], "source": [ "# Redis connection\n", @@ -101,14 +100,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:08.700313Z", - "iopub.status.busy": "2025-10-15T21:32:08.700251Z", - "iopub.status.idle": "2025-10-15T21:32:08.702206Z", - "shell.execute_reply": "2025-10-15T21:32:08.702012Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -161,14 +153,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:08.717380Z", - "iopub.status.busy": "2025-10-15T21:32:08.717285Z", - "iopub.status.idle": "2025-10-15T21:32:08.723852Z", - "shell.execute_reply": "2025-10-15T21:32:08.723644Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -228,27 +213,14 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:08.724966Z", - "iopub.status.busy": "2025-10-15T21:32:08.724905Z", - "iopub.status.idle": "2025-10-15T21:32:10.740249Z", - "shell.execute_reply": "2025-10-15T21:32:10.739174Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Creating vectors with 512 dimensions (reduced from 1024 if applicable)\n", - "āœ… Loaded 10 documents into the index\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "āœ… Loaded 10 documents into the index\n", " Index now contains 0 documents\n" ] } @@ -319,14 +291,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.744612Z", - "iopub.status.busy": "2025-10-15T21:32:10.744310Z", - "iopub.status.idle": "2025-10-15T21:32:10.751707Z", - "shell.execute_reply": "2025-10-15T21:32:10.751275Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -377,14 +342,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.753759Z", - "iopub.status.busy": "2025-10-15T21:32:10.753565Z", - "iopub.status.idle": "2025-10-15T21:32:10.757377Z", - "shell.execute_reply": "2025-10-15T21:32:10.757018Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -463,14 +421,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.759026Z", - "iopub.status.busy": "2025-10-15T21:32:10.758895Z", - "iopub.status.idle": "2025-10-15T21:32:10.762424Z", - "shell.execute_reply": "2025-10-15T21:32:10.762083Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -521,14 +472,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.763978Z", - "iopub.status.busy": "2025-10-15T21:32:10.763840Z", - "iopub.status.idle": "2025-10-15T21:32:10.768306Z", - "shell.execute_reply": "2025-10-15T21:32:10.768005Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -574,14 +518,7 @@ { "cell_type": "code", "execution_count": 10, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.770248Z", - "iopub.status.busy": "2025-10-15T21:32:10.770114Z", - "iopub.status.idle": "2025-10-15T21:32:10.774772Z", - "shell.execute_reply": "2025-10-15T21:32:10.774498Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -591,7 +528,7 @@ "==============================\n", "Documents: 0\n", "Vector index size: 0.00 MB\n", - "Total indexing time: 1.36 seconds\n", + "Total indexing time: 1.58 seconds\n", "Memory efficiency calculation requires documents and vector index size > 0\n" ] } @@ -647,14 +584,7 @@ { "cell_type": "code", "execution_count": 11, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.776132Z", - "iopub.status.busy": "2025-10-15T21:32:10.776038Z", - "iopub.status.idle": "2025-10-15T21:32:10.779151Z", - "shell.execute_reply": "2025-10-15T21:32:10.778875Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -758,28 +688,13 @@ { "cell_type": "code", "execution_count": 12, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-15T21:32:10.780645Z", - "iopub.status.busy": "2025-10-15T21:32:10.780541Z", - "iopub.status.idle": "2025-10-15T21:32:10.783383Z", - "shell.execute_reply": "2025-10-15T21:32:10.783148Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Cleaned up svs_demo index\n", - "\n", - "šŸŽ‰ SVS-VAMANA demo completed!\n", - "\n", - "Next steps:\n", - "- Try SVS-VAMANA with your own embeddings\n", - "- Experiment with different compression settings\n", - "- Compare performance with FLAT and HNSW algorithms\n", - "- Use hybrid queries for complex search scenarios\n" + "Cleaned up svs_demo index\n" ] } ], @@ -789,15 +704,7 @@ " index.delete()\n", " print(\"Cleaned up svs_demo index\")\n", "except:\n", - " print(\"- svs_demo index was already deleted or doesn't exist\")\n", - "\n", - "# Note: The manual index wasn't created in this demo, so no need to delete it\n", - "print(\"\\nšŸŽ‰ SVS-VAMANA demo completed!\")\n", - "print(\"\\nNext steps:\")\n", - "print(\"- Try SVS-VAMANA with your own embeddings\")\n", - "print(\"- Experiment with different compression settings\")\n", - "print(\"- Compare performance with FLAT and HNSW algorithms\")\n", - "print(\"- Use hybrid queries for complex search scenarios\")" + " print(\"- svs_demo index was already deleted or doesn't exist\")" ] } ], From e5f364e6ee0e2a92c48aff6f3904afaa6ca7cd55 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Thu, 16 Oct 2025 13:32:50 -0400 Subject: [PATCH 22/29] Update makefile for tests --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 58001ba6..c9bb7e6e 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,14 @@ test-all: ## Run all tests including API tests test-notebooks: ## Run notebook tests @echo "šŸ““ Running notebook tests" - uv run python -m pytest --nbval-lax ./docs/user_guide -vvv $(ARGS) + @echo "šŸ” Checking Redis version..." + @if uv run python -c "import redis; from redisvl.redis.connection import supports_svs; client = redis.Redis.from_url('redis://localhost:6379'); exit(0 if supports_svs(client) else 1)" 2>/dev/null; then \ + echo "āœ… Redis 8.2.0+ detected - running all notebooks"; \ + uv run python -m pytest --nbval-lax ./docs/user_guide -vvv $(ARGS); \ + else \ + echo "āš ļø Redis < 8.2.0 detected - skipping SVS notebook"; \ + uv run python -m pytest --nbval-lax ./docs/user_guide -vvv --ignore=./docs/user_guide/09_svs_vamana.ipynb $(ARGS); \ + fi check: lint test ## Run all checks (lint + test) From 46deb0bf32e04e7a866c893cfe3d3a40a8376ea5 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 19:48:34 -0400 Subject: [PATCH 23/29] Convert to pydantic and update docs # Conflicts: # docs/user_guide/index.md --- docs/api/schema.rst | 4 +- docs/user_guide/index.md | 4 +- redisvl/schema/fields.py | 4 +- redisvl/utils/compression.py | 79 ++++++------- tests/unit/test_compression_advisor.py | 158 ++++++++++++------------- 5 files changed, 122 insertions(+), 127 deletions(-) diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 113af2b0..720cef3f 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -262,7 +262,7 @@ FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requir :color: info **Use FLAT when:** - - Small datasets (<10K vectors) where exact results are required + - Small datasets (<100K vectors) where exact results are required - Search accuracy is critical and approximate results are not acceptable - Baseline comparisons when evaluating approximate algorithms - Simple use cases where setup simplicity is more important than performance @@ -317,7 +317,7 @@ Algorithm Selection Guide - Memory Usage - Trade-offs * - **FLAT** - - Small datasets (<10K vectors) + - Small datasets (<100K vectors) - 100% recall, O(n) search - Minimal overhead - Exact but slow for large data diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index faf0e128..221663e6 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -22,6 +22,6 @@ User guides provide helpful resources for using RedisVL and its different compon 07_message_history 08_semantic_router 09_svs_vamana +10_embeddings_cache 11_advanced_queries -``` -# TODO: Nitin Add the new user guide for SVS-VAMANA \ No newline at end of file +``` \ No newline at end of file diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index 27444130..3a3f2206 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -9,7 +9,7 @@ - NumericField: Numeric values for range queries and sorting - GeoField: Geographic coordinates for location-based search - VectorField: Vector embeddings for semantic similarity search - - FlatVectorField: Brute-force exact search (100% recall) + - FlatVectorField: Exact search (100% recall) - HNSWVectorField: Approximate nearest neighbor search (fast, high recall) - SVSVectorField: Compressed vector search with memory savings @@ -489,7 +489,7 @@ def as_redis_field(self) -> RedisField: class FlatVectorField(BaseField): - """Vector field with FLAT (brute-force) indexing for exact nearest neighbor search.""" + """Vector field with FLAT (exact search) indexing for exact nearest neighbor search.""" type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR attrs: FlatVectorFieldAttributes diff --git a/redisvl/utils/compression.py b/redisvl/utils/compression.py index fcee3a22..44a0cf35 100644 --- a/redisvl/utils/compression.py +++ b/redisvl/utils/compression.py @@ -1,10 +1,12 @@ """SVS-VAMANA compression configuration utilities.""" -from typing import Literal, Optional, TypedDict, cast +from typing import Literal, Optional +from pydantic import BaseModel, Field -class SVSConfig(TypedDict, total=False): - """SVS-VAMANA configuration dictionary. + +class SVSConfig(BaseModel): + """SVS-VAMANA configuration model. Attributes: algorithm: Always "svs-vamana" @@ -16,13 +18,15 @@ class SVSConfig(TypedDict, total=False): search_window_size: Query-time candidates """ - algorithm: Literal["svs-vamana"] - datatype: str - compression: str - reduce: int # only for LeanVec - graph_max_degree: int - construction_window_size: int - search_window_size: int + algorithm: Literal["svs-vamana"] = "svs-vamana" + datatype: Optional[str] = None + compression: Optional[str] = None + reduce: Optional[int] = Field( + default=None, description="Reduced dimensionality (only for LeanVec)" + ) + graph_max_degree: Optional[int] = None + construction_window_size: Optional[int] = None + search_window_size: Optional[int] = None class CompressionAdvisor: @@ -35,9 +39,9 @@ class CompressionAdvisor: Examples: >>> # Get recommendations for high-dimensional vectors >>> config = CompressionAdvisor.recommend(dims=1536, priority="balanced") - >>> config["compression"] + >>> config.compression 'LeanVec4x8' - >>> config["reduce"] + >>> config.reduce 768 >>> # Estimate memory savings @@ -95,14 +99,14 @@ def recommend( Examples: >>> # High-dimensional embeddings (e.g., OpenAI ada-002) >>> config = CompressionAdvisor.recommend(dims=1536, priority="memory") - >>> config["compression"] + >>> config.compression 'LeanVec4x8' - >>> config["reduce"] + >>> config.reduce 768 >>> # Lower-dimensional embeddings >>> config = CompressionAdvisor.recommend(dims=384, priority="speed") - >>> config["compression"] + >>> config.compression 'LVQ4x8' """ if dims <= 0: @@ -118,34 +122,25 @@ def recommend( } if priority == "memory": - return cast( - SVSConfig, - { - **base, - "compression": "LeanVec4x8", - "reduce": dims // 2, - "search_window_size": 20, - }, + return SVSConfig( + **base, + compression="LeanVec4x8", + reduce=dims // 2, + search_window_size=20, ) elif priority == "speed": - return cast( - SVSConfig, - { - **base, - "compression": "LeanVec4x8", - "reduce": max(256, dims // 4), - "search_window_size": 40, - }, + return SVSConfig( + **base, + compression="LeanVec4x8", + reduce=max(256, dims // 4), + search_window_size=40, ) else: # balanced - return cast( - SVSConfig, - { - **base, - "compression": "LeanVec4x8", - "reduce": dims // 2, - "search_window_size": 30, - }, + return SVSConfig( + **base, + compression="LeanVec4x8", + reduce=dims // 2, + search_window_size=30, ) # Lower-dimensional vectors - use LVQ @@ -159,11 +154,11 @@ def recommend( } if priority == "memory": - return cast(SVSConfig, {**base, "compression": "LVQ4"}) + return SVSConfig(**base, compression="LVQ4") elif priority == "speed": - return cast(SVSConfig, {**base, "compression": "LVQ4x8"}) + return SVSConfig(**base, compression="LVQ4x8") else: # balanced - return cast(SVSConfig, {**base, "compression": "LVQ4x4"}) + return SVSConfig(**base, compression="LVQ4x4") @staticmethod def estimate_memory_savings( diff --git a/tests/unit/test_compression_advisor.py b/tests/unit/test_compression_advisor.py index 6226d543..2a9cbbe0 100644 --- a/tests/unit/test_compression_advisor.py +++ b/tests/unit/test_compression_advisor.py @@ -12,116 +12,116 @@ def test_recommend_high_dim_memory_priority(self): """Test memory-optimized config for high-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=1536, priority="memory") - assert config["algorithm"] == "svs-vamana" - assert config["datatype"] == "float16" - assert config["compression"] == "LeanVec4x8" - assert config["reduce"] == 768 # dims // 2 - assert config["graph_max_degree"] == 64 - assert config["construction_window_size"] == 300 - assert config["search_window_size"] == 20 + assert config.algorithm == "svs-vamana" + assert config.datatype == "float16" + assert config.compression == "LeanVec4x8" + assert config.reduce == 768 # dims // 2 + assert config.graph_max_degree == 64 + assert config.construction_window_size == 300 + assert config.search_window_size == 20 def test_recommend_high_dim_speed_priority(self): """Test speed-optimized config for high-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=1536, priority="speed") - assert config["algorithm"] == "svs-vamana" - assert config["datatype"] == "float16" - assert config["compression"] == "LeanVec4x8" - assert config["reduce"] == 384 # dims // 4 - assert config["graph_max_degree"] == 64 - assert config["construction_window_size"] == 300 - assert config["search_window_size"] == 40 + assert config.algorithm == "svs-vamana" + assert config.datatype == "float16" + assert config.compression == "LeanVec4x8" + assert config.reduce == 384 # dims // 4 + assert config.graph_max_degree == 64 + assert config.construction_window_size == 300 + assert config.search_window_size == 40 def test_recommend_high_dim_balanced_priority(self): """Test balanced config for high-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=1536, priority="balanced") - assert config["algorithm"] == "svs-vamana" - assert config["datatype"] == "float16" - assert config["compression"] == "LeanVec4x8" - assert config["reduce"] == 768 # dims // 2 - assert config["graph_max_degree"] == 64 - assert config["construction_window_size"] == 300 - assert config["search_window_size"] == 30 + assert config.algorithm == "svs-vamana" + assert config.datatype == "float16" + assert config.compression == "LeanVec4x8" + assert config.reduce == 768 # dims // 2 + assert config.graph_max_degree == 64 + assert config.construction_window_size == 300 + assert config.search_window_size == 30 def test_recommend_high_dim_default_priority(self): """Test default priority (balanced) for high-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=2048) - assert config["compression"] == "LeanVec4x8" - assert config["reduce"] == 1024 - assert config["search_window_size"] == 30 + assert config.compression == "LeanVec4x8" + assert config.reduce == 1024 + assert config.search_window_size == 30 def test_recommend_low_dim_memory_priority(self): """Test memory-optimized config for low-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=384, priority="memory") - assert config["algorithm"] == "svs-vamana" - assert config["datatype"] == "float32" - assert config["compression"] == "LVQ4" - assert "reduce" not in config # LVQ doesn't use reduce - assert config["graph_max_degree"] == 40 - assert config["construction_window_size"] == 250 - assert config["search_window_size"] == 20 + assert config.algorithm == "svs-vamana" + assert config.datatype == "float32" + assert config.compression == "LVQ4" + assert config.reduce is None # LVQ doesn't use reduce + assert config.graph_max_degree == 40 + assert config.construction_window_size == 250 + assert config.search_window_size == 20 def test_recommend_low_dim_speed_priority(self): """Test speed-optimized config for low-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=384, priority="speed") - assert config["algorithm"] == "svs-vamana" - assert config["datatype"] == "float32" - assert config["compression"] == "LVQ4x8" - assert "reduce" not in config - assert config["graph_max_degree"] == 40 - assert config["construction_window_size"] == 250 - assert config["search_window_size"] == 20 + assert config.algorithm == "svs-vamana" + assert config.datatype == "float32" + assert config.compression == "LVQ4x8" + assert config.reduce is None + assert config.graph_max_degree == 40 + assert config.construction_window_size == 250 + assert config.search_window_size == 20 def test_recommend_low_dim_balanced_priority(self): """Test balanced config for low-dimensional vectors.""" config = CompressionAdvisor.recommend(dims=768, priority="balanced") - assert config["algorithm"] == "svs-vamana" - assert config["datatype"] == "float32" - assert config["compression"] == "LVQ4x4" - assert "reduce" not in config - assert config["graph_max_degree"] == 40 - assert config["construction_window_size"] == 250 - assert config["search_window_size"] == 20 + assert config.algorithm == "svs-vamana" + assert config.datatype == "float32" + assert config.compression == "LVQ4x4" + assert config.reduce is None + assert config.graph_max_degree == 40 + assert config.construction_window_size == 250 + assert config.search_window_size == 20 def test_recommend_threshold_boundary_low(self): """Test recommendation at threshold boundary (1023 dims).""" config = CompressionAdvisor.recommend(dims=1023) # Should use LVQ (below threshold) - assert config["compression"] in ["LVQ4", "LVQ4x4", "LVQ4x8"] - assert config["datatype"] == "float32" - assert "reduce" not in config + assert config.compression in ["LVQ4", "LVQ4x4", "LVQ4x8"] + assert config.datatype == "float32" + assert config.reduce is None def test_recommend_threshold_boundary_high(self): """Test recommendation at threshold boundary (1024 dims).""" config = CompressionAdvisor.recommend(dims=1024) # Should use LeanVec (at threshold) - assert config["compression"] == "LeanVec4x8" - assert config["datatype"] == "float16" - assert "reduce" in config + assert config.compression == "LeanVec4x8" + assert config.datatype == "float16" + assert config.reduce is not None def test_recommend_custom_datatype(self): """Test custom datatype override.""" config = CompressionAdvisor.recommend(dims=1536, datatype="float32") - assert config["datatype"] == "float32" + assert config.datatype == "float32" def test_recommend_speed_reduce_minimum(self): """Test that speed priority respects minimum reduce value.""" config = CompressionAdvisor.recommend(dims=1024, priority="speed") # dims // 4 = 256, max(256, 256) = 256 - assert config["reduce"] == 256 + assert config.reduce == 256 config = CompressionAdvisor.recommend(dims=512, priority="speed") # Below threshold, should use LVQ - assert "reduce" not in config + assert config.reduce is None def test_recommend_invalid_dims_zero(self): """Test that zero dims raises ValueError.""" @@ -217,34 +217,34 @@ def test_estimate_rounding(self): assert isinstance(savings, float) -class TestSVSConfigTypedDict: - """Tests for SVSConfig TypedDict structure.""" +class TestSVSConfigModel: + """Tests for SVSConfig Pydantic model structure.""" def test_svs_config_structure(self): """Test that SVSConfig can be constructed with all fields.""" - config: SVSConfig = { - "algorithm": "svs-vamana", - "datatype": "float16", - "compression": "LeanVec4x8", - "reduce": 768, - "graph_max_degree": 64, - "construction_window_size": 300, - "search_window_size": 30, - } - - assert config["algorithm"] == "svs-vamana" - assert config["reduce"] == 768 + config = SVSConfig( + algorithm="svs-vamana", + datatype="float16", + compression="LeanVec4x8", + reduce=768, + graph_max_degree=64, + construction_window_size=300, + search_window_size=30, + ) + + assert config.algorithm == "svs-vamana" + assert config.reduce == 768 def test_svs_config_without_reduce(self): """Test that SVSConfig can be constructed without reduce field.""" - config: SVSConfig = { - "algorithm": "svs-vamana", - "datatype": "float32", - "compression": "LVQ4", - "graph_max_degree": 40, - "construction_window_size": 250, - "search_window_size": 20, - } - - assert "reduce" not in config - assert config["compression"] == "LVQ4" + config = SVSConfig( + algorithm="svs-vamana", + datatype="float32", + compression="LVQ4", + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + ) + + assert config.reduce is None + assert config.compression == "LVQ4" From a9ed4aebbe97a3bbd62c03f68ea006752f5ba0ba Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 10:54:43 -0400 Subject: [PATCH 24/29] Explicitly write all arguments to avoid mypy failed tests --- redisvl/utils/compression.py | 57 ++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/redisvl/utils/compression.py b/redisvl/utils/compression.py index 44a0cf35..c2a32a45 100644 --- a/redisvl/utils/compression.py +++ b/redisvl/utils/compression.py @@ -114,30 +114,34 @@ def recommend( # High-dimensional vectors (>= 1024) - use LeanVec if dims >= CompressionAdvisor.HIGH_DIM_THRESHOLD: - base = { - "algorithm": "svs-vamana", - "datatype": datatype or "float16", - "graph_max_degree": 64, - "construction_window_size": 300, - } + base_datatype = datatype or "float16" if priority == "memory": return SVSConfig( - **base, + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=64, + construction_window_size=300, compression="LeanVec4x8", reduce=dims // 2, search_window_size=20, ) elif priority == "speed": return SVSConfig( - **base, + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=64, + construction_window_size=300, compression="LeanVec4x8", reduce=max(256, dims // 4), search_window_size=40, ) else: # balanced return SVSConfig( - **base, + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=64, + construction_window_size=300, compression="LeanVec4x8", reduce=dims // 2, search_window_size=30, @@ -145,20 +149,35 @@ def recommend( # Lower-dimensional vectors - use LVQ else: - base = { - "algorithm": "svs-vamana", - "datatype": datatype or "float32", - "graph_max_degree": 40, - "construction_window_size": 250, - "search_window_size": 20, - } + base_datatype = datatype or "float32" if priority == "memory": - return SVSConfig(**base, compression="LVQ4") + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + compression="LVQ4", + ) elif priority == "speed": - return SVSConfig(**base, compression="LVQ4x8") + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + compression="LVQ4x8", + ) else: # balanced - return SVSConfig(**base, compression="LVQ4x4") + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + compression="LVQ4x4", + ) @staticmethod def estimate_memory_savings( From 53229c034808753c29a74c3c4725b51e7d197c66 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 10:55:44 -0400 Subject: [PATCH 25/29] lock file --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 548a33aa..a569cf0a 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9, <3.14" resolution-markers = [ "python_full_version >= '3.13'", @@ -3680,7 +3680,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2,<3" }, { name = "python-ulid", specifier = ">=3.0.0" }, { name = "pyyaml", specifier = ">=5.4,<7.0" }, - { name = "redis", specifier = ">=6.4.0,<7.0" }, + { name = "redis", specifier = ">=5.0,<7.0" }, { name = "sentence-transformers", marker = "extra == 'sentence-transformers'", specifier = ">=3.4.0,<4" }, { name = "tenacity", specifier = ">=8.2.2" }, { name = "urllib3", marker = "extra == 'bedrock'", specifier = "<2.2.0" }, From c7428cfd0e622fad4133cecb71abe6d885e46e66 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 11:03:20 -0400 Subject: [PATCH 26/29] Bump version - SVS is a new feature --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3fc4202f..ec4e08b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "redisvl" -version = "0.10.0" +version = "0.11.0" description = "Python client library and CLI for using Redis as a vector database" authors = [{ name = "Redis Inc.", email = "applied.ai@redis.com" }] requires-python = ">=3.9,<3.14" diff --git a/uv.lock b/uv.lock index a569cf0a..cad34f92 100644 --- a/uv.lock +++ b/uv.lock @@ -3593,7 +3593,7 @@ wheels = [ [[package]] name = "redisvl" -version = "0.10.0" +version = "0.11.0" source = { editable = "." } dependencies = [ { name = "jsonpath-ng" }, From 895119813dab1ef2b77b3c158e34ee8fcaa3ef46 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 13:40:55 -0400 Subject: [PATCH 27/29] Add compression advisor --- docs/api/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api/index.md b/docs/api/index.md index f7c1c661..eb849513 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -20,6 +20,7 @@ query filter vectorizer reranker +utils cache message_history router From 7316bf2309c6a11dfee76472aab40fed922be004 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 14:40:34 -0400 Subject: [PATCH 28/29] move compression advisor in docs --- docs/api/schema.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 720cef3f..bc8d3ded 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -297,6 +297,27 @@ FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requir **Note**: FLAT is recommended for small datasets or when exact results are mandatory. For larger datasets, consider HNSW or SVS-VAMANA for better performance. +SVS-VAMANA Configuration Utilities +================================== + +For SVS-VAMANA indices, RedisVL provides utilities to help configure compression settings and estimate memory savings. + +CompressionAdvisor +------------------ + +.. currentmodule:: redisvl.utils.compression + +.. autoclass:: CompressionAdvisor + :members: + :show-inheritance: + +SVSConfig +--------- + +.. autoclass:: SVSConfig + :members: + :show-inheritance: + .. _vector-algorithm-comparison: Vector Algorithm Comparison From 623b6086fecddb9f170fee93b2b7bcf6b43c1f92 Mon Sep 17 00:00:00 2001 From: Nitin Kanukolanu Date: Tue, 21 Oct 2025 14:46:27 -0400 Subject: [PATCH 29/29] Update docs --- docs/api/schema.rst | 81 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/docs/api/schema.rst b/docs/api/schema.rst index bc8d3ded..f9e165f9 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -60,36 +60,80 @@ Fields in the schema can be defined in YAML format or as a Python dictionary, sp } } -Supported Field Types and Attributes -==================================== +Basic Field Types +================= -Each field type supports specific attributes that customize its behavior. Below are the field types and their available attributes: +RedisVL supports several basic field types for indexing different kinds of data. Each field type has specific attributes that customize its indexing and search behavior. -**Text Field Attributes**: +Text Fields +----------- + +Text fields support full-text search with stemming, phonetic matching, and other text analysis features. + +.. currentmodule:: redisvl.schema.fields + +.. autoclass:: TextField + :members: + :show-inheritance: .. autoclass:: TextFieldAttributes :members: :undoc-members: -**Tag Field Attributes**: +Tag Fields +---------- + +Tag fields are optimized for exact-match filtering and faceted search on categorical data. + +.. autoclass:: TagField + :members: + :show-inheritance: .. autoclass:: TagFieldAttributes :members: :undoc-members: -**Numeric Field Attributes**: +Numeric Fields +-------------- + +Numeric fields support range queries and sorting on numeric data. + +.. autoclass:: NumericField + :members: + :show-inheritance: .. autoclass:: NumericFieldAttributes :members: :undoc-members: -**Geo Field Attributes**: +Geo Fields +---------- + +Geo fields enable location-based search with geographic coordinates. + +.. autoclass:: GeoField + :members: + :show-inheritance: .. autoclass:: GeoFieldAttributes :members: :undoc-members: -**Common Vector Field Attributes**: +Vector Field Types +================== + +Vector fields enable semantic similarity search using various algorithms. All vector fields share common attributes but have algorithm-specific configurations. + +Common Vector Attributes +------------------------ + +All vector field types share these base attributes: + +.. autoclass:: BaseVectorFieldAttributes + :members: + :undoc-members: + +**Key Attributes:** - `dims`: Dimensionality of the vector (e.g., 768, 1536). - `algorithm`: Indexing algorithm for vector search: @@ -106,7 +150,8 @@ Each field type supports specific attributes that customize its behavior. Below - `initial_cap`: Initial capacity hint for memory allocation (optional). - `index_missing`: When True, allows searching for documents missing this field (optional). -**HNSW Vector Field Specific Attributes**: +HNSW Vector Fields +------------------ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with excellent recall. **Best for general-purpose vector search (10K-1M+ vectors).** @@ -127,7 +172,9 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with - **Recall quality**: Excellent recall rates (95-99%), often better than other approximate methods - **Build time**: Moderate construction time, faster than SVS-VAMANA for smaller datasets -.. currentmodule:: redisvl.schema.fields +.. autoclass:: HNSWVectorField + :members: + :show-inheritance: .. autoclass:: HNSWVectorFieldAttributes :members: @@ -167,7 +214,8 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with ef_construction: 400 ef_runtime: 50 -**SVS-VAMANA Vector Field Specific Attributes**: +SVS-VAMANA Vector Fields +------------------------ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. **Best for large datasets (>100K vectors) on Intel hardware with memory constraints.** @@ -204,6 +252,10 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap - LeanVec4x8 + reduce to 384: 580 MB (~81% savings) +.. autoclass:: SVSVectorField + :members: + :show-inheritance: + .. autoclass:: SVSVectorFieldAttributes :members: :undoc-members: @@ -254,7 +306,8 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap - **Platform considerations**: Intel's proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. - **Performance tip**: Start with default parameters and tune `search_window_size` first for your speed vs accuracy requirements. -**FLAT Vector Field Specific Attributes**: +FLAT Vector Fields +------------------ FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requiring 100% accuracy.** @@ -277,6 +330,10 @@ FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requir - **vs HNSW**: Much slower search but exact results, faster index building - **vs SVS-VAMANA**: Slower search and higher memory usage, but exact results +.. autoclass:: FlatVectorField + :members: + :show-inheritance: + .. autoclass:: FlatVectorFieldAttributes :members: :undoc-members: