diff --git a/Makefile b/Makefile index 58001ba6..c9bb7e6e 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,14 @@ test-all: ## Run all tests including API tests test-notebooks: ## Run notebook tests @echo "📓 Running notebook tests" - uv run python -m pytest --nbval-lax ./docs/user_guide -vvv $(ARGS) + @echo "🔍 Checking Redis version..." + @if uv run python -c "import redis; from redisvl.redis.connection import supports_svs; client = redis.Redis.from_url('redis://localhost:6379'); exit(0 if supports_svs(client) else 1)" 2>/dev/null; then \ + echo "✅ Redis 8.2.0+ detected - running all notebooks"; \ + uv run python -m pytest --nbval-lax ./docs/user_guide -vvv $(ARGS); \ + else \ + echo "⚠️ Redis < 8.2.0 detected - skipping SVS notebook"; \ + uv run python -m pytest --nbval-lax ./docs/user_guide -vvv --ignore=./docs/user_guide/09_svs_vamana.ipynb $(ARGS); \ + fi check: lint test ## Run all checks (lint + test) diff --git a/docs/api/index.md b/docs/api/index.md index f7c1c661..eb849513 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -20,6 +20,7 @@ query filter vectorizer reranker +utils cache message_history router diff --git a/docs/api/schema.rst b/docs/api/schema.rst index 7f6ae174..f9e165f9 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -60,53 +60,410 @@ Fields in the schema can be defined in YAML format or as a Python dictionary, sp } } -Supported Field Types and Attributes -==================================== +Basic Field Types +================= -Each field type supports specific attributes that customize its behavior. Below are the field types and their available attributes: +RedisVL supports several basic field types for indexing different kinds of data. Each field type has specific attributes that customize its indexing and search behavior. -**Text Field Attributes**: +Text Fields +----------- -- `weight`: Importance of the field in result calculation. -- `no_stem`: Disables stemming during indexing. -- `withsuffixtrie`: Optimizes queries by maintaining a suffix trie. -- `phonetic_matcher`: Enables phonetic matching. -- `sortable`: Allows sorting on this field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). -- `unf`: Un-normalized form. When True, preserves original case for sorting (requires `sortable=True`). +Text fields support full-text search with stemming, phonetic matching, and other text analysis features. -**Tag Field Attributes**: +.. currentmodule:: redisvl.schema.fields -- `separator`: Character for splitting text into individual tags. -- `case_sensitive`: Case sensitivity in tag matching. -- `withsuffixtrie`: Suffix trie optimization for queries. -- `sortable`: Enables sorting based on the tag field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). +.. autoclass:: TextField + :members: + :show-inheritance: + +.. autoclass:: TextFieldAttributes + :members: + :undoc-members: + +Tag Fields +---------- + +Tag fields are optimized for exact-match filtering and faceted search on categorical data. + +.. autoclass:: TagField + :members: + :show-inheritance: + +.. autoclass:: TagFieldAttributes + :members: + :undoc-members: + +Numeric Fields +-------------- + +Numeric fields support range queries and sorting on numeric data. + +.. autoclass:: NumericField + :members: + :show-inheritance: + +.. autoclass:: NumericFieldAttributes + :members: + :undoc-members: + +Geo Fields +---------- + +Geo fields enable location-based search with geographic coordinates. + +.. autoclass:: GeoField + :members: + :show-inheritance: + +.. autoclass:: GeoFieldAttributes + :members: + :undoc-members: + +Vector Field Types +================== + +Vector fields enable semantic similarity search using various algorithms. All vector fields share common attributes but have algorithm-specific configurations. + +Common Vector Attributes +------------------------ + +All vector field types share these base attributes: + +.. autoclass:: BaseVectorFieldAttributes + :members: + :undoc-members: + +**Key Attributes:** + +- `dims`: Dimensionality of the vector (e.g., 768, 1536). +- `algorithm`: Indexing algorithm for vector search: + + - `flat`: Brute-force exact search. 100% recall, slower for large datasets. Best for <10K vectors. + - `hnsw`: Graph-based approximate search. Fast with high recall (95-99%). Best for general use. + - `svs-vamana`: SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. + + .. note:: + For detailed algorithm comparison and selection guidance, see :ref:`vector-algorithm-comparison`. + +- `datatype`: Float precision (`bfloat16`, `float16`, `float32`, `float64`). Note: SVS-VAMANA only supports `float16` and `float32`. +- `distance_metric`: Similarity metric (`COSINE`, `L2`, `IP`). +- `initial_cap`: Initial capacity hint for memory allocation (optional). +- `index_missing`: When True, allows searching for documents missing this field (optional). + +HNSW Vector Fields +------------------ + +HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with excellent recall. **Best for general-purpose vector search (10K-1M+ vectors).** + +.. dropdown:: When to use HNSW & Performance Details + :color: info + + **Use HNSW when:** + + - Medium to large datasets (10K-1M+ vectors) requiring high recall rates + - Search accuracy is more important than memory usage + - Need general-purpose vector search with balanced performance + - Cross-platform deployments where hardware-specific optimizations aren't available + + **Performance characteristics:** + + - **Search speed**: Very fast approximate search with tunable accuracy + - **Memory usage**: Higher than compressed SVS-VAMANA but reasonable for most applications + - **Recall quality**: Excellent recall rates (95-99%), often better than other approximate methods + - **Build time**: Moderate construction time, faster than SVS-VAMANA for smaller datasets + +.. autoclass:: HNSWVectorField + :members: + :show-inheritance: + +.. autoclass:: HNSWVectorFieldAttributes + :members: + :undoc-members: + +**HNSW Examples:** + +**Balanced configuration (recommended starting point):** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + distance_metric: cosine + datatype: float32 + # Balanced settings for good recall and performance + m: 16 + ef_construction: 200 + ef_runtime: 10 -**Numeric Field Attributes**: +**High-recall configuration:** -- `sortable`: Enables sorting on the numeric field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). -- `unf`: Un-normalized form. When True, maintains original numeric representation for sorting (requires `sortable=True`). +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + distance_metric: cosine + datatype: float32 + # Tuned for maximum accuracy + m: 32 + ef_construction: 400 + ef_runtime: 50 + +SVS-VAMANA Vector Fields +------------------------ + +SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast approximate nearest neighbor search with optional compression support. This algorithm is optimized for Intel hardware and offers reduced memory usage through vector compression. **Best for large datasets (>100K vectors) on Intel hardware with memory constraints.** + +.. dropdown:: When to use SVS-VAMANA & Detailed Guide + :color: info + + **Requirements:** + - Redis >= 8.2.0 with RediSearch >= 2.8.10 + - datatype must be 'float16' or 'float32' (float64/bfloat16 not supported) -**Geo Field Attributes**: + **Use SVS-VAMANA when:** + - Large datasets where memory is expensive + - Cloud deployments with memory-based pricing + - When 90-95% recall is acceptable + - High-dimensional vectors (>1024 dims) with LeanVec compression -- `sortable`: Enables sorting based on the geo field. -- `no_index`: When True, field is not indexed but can be returned in results (requires `sortable=True`). + **Performance vs other algorithms:** + - **vs FLAT**: Much faster search, significantly lower memory usage with compression, but approximate results -**Common Vector Field Attributes**: + - **vs HNSW**: Better memory efficiency with compression, similar or better recall, Intel-optimized -- `dims`: Dimensionality of the vector. -- `algorithm`: Indexing algorithm (`flat` or `hnsw`). -- `datatype`: Float datatype of the vector (`bfloat16`, `float16`, `float32`, `float64`). -- `distance_metric`: Metric for measuring query relevance (`COSINE`, `L2`, `IP`). + **Compression selection guide:** -**HNSW Vector Field Specific Attributes**: + - **No compression**: Best performance, standard memory usage -- `m`: Max outgoing edges per node in each layer. -- `ef_construction`: Max edge candidates during build time. -- `ef_runtime`: Max top candidates during search. -- `epsilon`: Range search boundary factor. + - **LVQ4/LVQ8**: Good balance of compression (2x-4x) and performance + + - **LeanVec4x8/LeanVec8x8**: Maximum compression (up to 8x) with dimensionality reduction + + **Memory Savings Examples (1M vectors, 768 dims):** + - No compression (float32): 3.1 GB + + - LVQ4x4 compression: 1.6 GB (~48% savings) + + - LeanVec4x8 + reduce to 384: 580 MB (~81% savings) + +.. autoclass:: SVSVectorField + :members: + :show-inheritance: -Note: - See fully documented Redis-supported fields and options here: https://redis.io/commands/ft.create/ \ No newline at end of file +.. autoclass:: SVSVectorFieldAttributes + :members: + :undoc-members: + +**SVS-VAMANA Examples:** + +**Basic configuration (no compression):** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: svs-vamana + dims: 768 + distance_metric: cosine + datatype: float32 + # Standard settings for balanced performance + graph_max_degree: 40 + construction_window_size: 250 + search_window_size: 20 + +**High-performance configuration with compression:** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: svs-vamana + dims: 768 + distance_metric: cosine + datatype: float32 + # Tuned for better recall + graph_max_degree: 64 + construction_window_size: 500 + search_window_size: 40 + # Maximum compression with dimensionality reduction + compression: LeanVec4x8 + reduce: 384 # 50% dimensionality reduction + training_threshold: 1000 + +**Important Notes:** + +- **Requirements**: SVS-VAMANA requires Redis >= 8.2 with RediSearch >= 2.8.10. +- **Datatype limitations**: SVS-VAMANA only supports `float16` and `float32` datatypes (not `bfloat16` or `float64`). +- **Compression compatibility**: The `reduce` parameter is only valid with LeanVec compression types (`LeanVec4x8` or `LeanVec8x8`). +- **Platform considerations**: Intel's proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. +- **Performance tip**: Start with default parameters and tune `search_window_size` first for your speed vs accuracy requirements. + +FLAT Vector Fields +------------------ + +FLAT - Brute-force exact search. **Best for small datasets (<10K vectors) requiring 100% accuracy.** + +.. dropdown:: When to use FLAT & Performance Details + :color: info + + **Use FLAT when:** + - Small datasets (<100K vectors) where exact results are required + - Search accuracy is critical and approximate results are not acceptable + - Baseline comparisons when evaluating approximate algorithms + - Simple use cases where setup simplicity is more important than performance + + **Performance characteristics:** + - **Search accuracy**: 100% exact results (no approximation) + - **Search speed**: Linear time O(n) - slower as dataset grows + - **Memory usage**: Minimal overhead, stores vectors as-is + - **Build time**: Fastest index construction (no preprocessing) + + **Trade-offs vs other algorithms:** + - **vs HNSW**: Much slower search but exact results, faster index building + - **vs SVS-VAMANA**: Slower search and higher memory usage, but exact results + +.. autoclass:: FlatVectorField + :members: + :show-inheritance: + +.. autoclass:: FlatVectorFieldAttributes + :members: + :undoc-members: + +**FLAT Example:** + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: flat + dims: 768 + distance_metric: cosine + datatype: float32 + # Optional: tune for batch processing + block_size: 1024 + +**Note**: FLAT is recommended for small datasets or when exact results are mandatory. For larger datasets, consider HNSW or SVS-VAMANA for better performance. + +SVS-VAMANA Configuration Utilities +================================== + +For SVS-VAMANA indices, RedisVL provides utilities to help configure compression settings and estimate memory savings. + +CompressionAdvisor +------------------ + +.. currentmodule:: redisvl.utils.compression + +.. autoclass:: CompressionAdvisor + :members: + :show-inheritance: + +SVSConfig +--------- + +.. autoclass:: SVSConfig + :members: + :show-inheritance: + +.. _vector-algorithm-comparison: + +Vector Algorithm Comparison +=========================== + +This section provides detailed guidance for choosing between vector search algorithms. + +Algorithm Selection Guide +------------------------- + +.. list-table:: Vector Algorithm Comparison + :header-rows: 1 + :widths: 15 20 20 20 25 + + * - Algorithm + - Best For + - Performance + - Memory Usage + - Trade-offs + * - **FLAT** + - Small datasets (<100K vectors) + - 100% recall, O(n) search + - Minimal overhead + - Exact but slow for large data + * - **HNSW** + - General purpose (10K-1M+ vectors) + - 95-99% recall, O(log n) search + - Moderate (graph overhead) + - Fast approximate search + * - **SVS-VAMANA** + - Large datasets with memory constraints + - 90-95% recall, O(log n) search + - Low (with compression) + - Intel-optimized, requires newer Redis + +When to Use Each Algorithm +-------------------------- + +**Choose FLAT when:** + - Dataset size < 10,000 vectors + - Exact results are mandatory + - Simple setup is preferred + - Query latency is not critical + +**Choose HNSW when:** + - Dataset size 10K - 1M+ vectors + - Need balanced speed and accuracy + - Cross-platform compatibility required + - Most common choice for production + +**Choose SVS-VAMANA when:** + - Dataset size > 100K vectors + - Memory usage is a primary concern + - Running on Intel hardware + - Can accept 90-95% recall for memory savings + +Performance Characteristics +--------------------------- + +**Search Speed:** + - FLAT: Linear time O(n) - gets slower as data grows + - HNSW: Logarithmic time O(log n) - scales well + - SVS-VAMANA: Logarithmic time O(log n) - scales well + +**Memory Usage (1M vectors, 768 dims, float32):** + - FLAT: ~3.1 GB (baseline) + - HNSW: ~3.7 GB (20% overhead for graph) + - SVS-VAMANA: 1.6-3.1 GB (depends on compression) + +**Recall Quality:** + - FLAT: 100% (exact search) + - HNSW: 95-99% (tunable via ef_runtime) + - SVS-VAMANA: 90-95% (depends on compression) + +Migration Considerations +------------------------ + +**From FLAT to HNSW:** + - Straightforward migration + - Expect slight recall reduction but major speed improvement + - Tune ef_runtime to balance speed vs accuracy + +**From HNSW to SVS-VAMANA:** + - Requires Redis >= 8.2 with RediSearch >= 2.8.10 + - Change datatype to float16 or float32 if using others + - Consider compression options for memory savings + +**From SVS-VAMANA to others:** + - May need to change datatype back if using float64/bfloat16 + - HNSW provides similar performance with broader compatibility + +For complete Redis field documentation, see: https://redis.io/commands/ft.create/ \ No newline at end of file diff --git a/docs/user_guide/01_getting_started.ipynb b/docs/user_guide/01_getting_started.ipynb index 7b8b60df..9f5034c9 100644 --- a/docs/user_guide/01_getting_started.ipynb +++ b/docs/user_guide/01_getting_started.ipynb @@ -761,7 +761,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -775,10 +775,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" - }, - "orig_nbformat": 4 + "version": "3.12.6" + } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/docs/user_guide/09_svs_vamana.ipynb b/docs/user_guide/09_svs_vamana.ipynb new file mode 100644 index 00000000..56ec4d88 --- /dev/null +++ b/docs/user_guide/09_svs_vamana.ipynb @@ -0,0 +1,732 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SVS-VAMANA Vector Search\n", + "\n", + "In this notebook, we will explore SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm), a graph-based vector search algorithm that is optimized to work with compression methods to reduce memory usage. It combines the Vamana graph algorithm with advanced compression techniques (LVQ and LeanVec) and is optimized for Intel hardware.\n", + "\n", + "**How it works**\n", + "\n", + "Vamana builds a single-layer proximity graph and prunes edges during construction based on tunable parameters, similar to HNSW but with a simpler structure. The compression methods apply per-vector normalization and scalar quantization, learning parameters directly from the data to enable fast, on-the-fly distance computations with SIMD-optimized layout Vector quantization and compression.\n", + "\n", + "\n", + "**SVS-VAMANA offers:**\n", + "- **Fast approximate nearest neighbor search** using graph-based algorithms\n", + "- **Vector compression** (LVQ, LeanVec) with up to 87.5% memory savings\n", + "- **Dimensionality reduction** (optional, with LeanVec)\n", + "- **Automatic performance optimization** through CompressionAdvisor\n", + "\n", + "**Use SVS-VAMANA when:**\n", + "- Large datasets where memory is expensive\n", + "- Cloud deployments with memory-based pricing\n", + "- When 90-95% recall is acceptable\n", + "- High-dimensional vectors (>1024 dims) with LeanVec compression\n", + "\n", + "\n", + "\n", + "**Table of Contents**\n", + "\n", + "1. [Prerequisites](#Prerequisites)\n", + "2. [Quick Start with CompressionAdvisor](#Quick-Start-with-CompressionAdvisor)\n", + "3. [Creating an SVS-VAMANA Index](#Creating-an-SVS-VAMANA-Index)\n", + "4. [Loading Sample Data](#Loading-Sample-Data)\n", + "5. [Performing Vector Searches](#Performing-Vector-Searches)\n", + "6. [Understanding Compression Types](#Understanding-Compression-Types)\n", + "7. [Hybrid Queries with SVS-VAMANA](#Hybrid-Queries-with-SVS-VAMANA)\n", + "8. [Performance Monitoring](#Performance-Monitoring)\n", + "9. [Manual Configuration (Advanced)](#Manual-Configuration-(Advanced))\n", + "10. [Best Practices and Tips](#Best-Practices-and-Tips)\n", + "11. [Cleanup](#Cleanup)\n", + "\n", + "---\n", + "\n", + "## Prerequisites\n", + "\n", + "Before running this notebook, ensure you have:\n", + "1. Installed `redisvl` and have that environment active for this notebook\n", + "2. A running Redis Stack instance with:\n", + " - Redis >= 8.2.0\n", + " - RediSearch >= 2.8.10\n", + "\n", + "For example, you can run Redis Stack locally with Docker:\n", + "\n", + "```bash\n", + "docker run -d -p 6379:6379 -p 8001:8001 redis/redis-stack:latest\n", + "```\n", + "\n", + "**Note:** SVS-VAMANA only supports FLOAT16 and FLOAT32 datatypes." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary modules\n", + "import numpy as np\n", + "from redisvl.index import SearchIndex\n", + "from redisvl.query import VectorQuery\n", + "from redisvl.utils import CompressionAdvisor\n", + "from redisvl.redis.utils import array_to_buffer\n", + "\n", + "# Set random seed for reproducible results\n", + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Redis connection\n", + "REDIS_URL = \"redis://localhost:6379\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Start with CompressionAdvisor\n", + "\n", + "The easiest way to get started with SVS-VAMANA is using the `CompressionAdvisor` utility, which automatically recommends optimal configuration based on your vector dimensions and performance priorities." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Recommended Configuration:\n", + " algorithm: svs-vamana\n", + " datatype: float16\n", + " graph_max_degree: 64\n", + " construction_window_size: 300\n", + " compression: LeanVec4x8\n", + " reduce: 512\n", + " search_window_size: 30\n", + "\n", + "Estimated Memory Savings: 81.2%\n" + ] + } + ], + "source": [ + "# Get recommended configuration for common embedding dimensions\n", + "dims = 1024 # Common embedding dimensions (works reliably with SVS-VAMANA)\n", + "\n", + "config = CompressionAdvisor.recommend(\n", + " dims=dims,\n", + " priority=\"balanced\" # Options: \"memory\", \"speed\", \"balanced\"\n", + ")\n", + "\n", + "print(\"Recommended Configuration:\")\n", + "for key, value in config.items():\n", + " print(f\" {key}: {value}\")\n", + "\n", + "# Estimate memory savings\n", + "savings = CompressionAdvisor.estimate_memory_savings(\n", + " config[\"compression\"],\n", + " dims,\n", + " config.get(\"reduce\")\n", + ")\n", + "print(f\"\\nEstimated Memory Savings: {savings}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating an SVS-VAMANA Index\n", + "\n", + "Let's create an index using the recommended configuration. We'll use a simple schema with text content and vector embeddings." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Created SVS-VAMANA index: svs_demo\n", + " Algorithm: svs-vamana\n", + " Compression: LeanVec4x8\n", + " Dimensions: 1024\n", + " Reduced to: 512 dimensions\n" + ] + } + ], + "source": [ + "# Create index schema with recommended SVS-VAMANA configuration\n", + "schema = {\n", + " \"index\": {\n", + " \"name\": \"svs_demo\",\n", + " \"prefix\": \"doc\",\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"content\", \"type\": \"text\"},\n", + " {\"name\": \"category\", \"type\": \"tag\"},\n", + " {\n", + " \"name\": \"embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": dims,\n", + " **config, # Use the recommended configuration\n", + " \"distance_metric\": \"cosine\"\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "# Create the index\n", + "index = SearchIndex.from_dict(schema, redis_url=REDIS_URL)\n", + "index.create(overwrite=True)\n", + "\n", + "print(f\"✅ Created SVS-VAMANA index: {index.name}\")\n", + "print(f\" Algorithm: {config['algorithm']}\")\n", + "print(f\" Compression: {config['compression']}\")\n", + "print(f\" Dimensions: {dims}\")\n", + "if 'reduce' in config:\n", + " print(f\" Reduced to: {config['reduce']} dimensions\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Sample Data\n", + "\n", + "Let's create some sample documents with embeddings to demonstrate SVS-VAMANA search capabilities." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating vectors with 512 dimensions (reduced from 1024 if applicable)\n", + "✅ Loaded 10 documents into the index\n", + " Index now contains 0 documents\n" + ] + } + ], + "source": [ + "# Generate sample data\n", + "sample_documents = [\n", + " {\"content\": \"Machine learning algorithms for data analysis\", \"category\": \"technology\"},\n", + " {\"content\": \"Natural language processing and text understanding\", \"category\": \"technology\"},\n", + " {\"content\": \"Computer vision and image recognition systems\", \"category\": \"technology\"},\n", + " {\"content\": \"Delicious pasta recipes from Italy\", \"category\": \"food\"},\n", + " {\"content\": \"Traditional French cooking techniques\", \"category\": \"food\"},\n", + " {\"content\": \"Healthy meal planning and nutrition\", \"category\": \"food\"},\n", + " {\"content\": \"Travel guide to European destinations\", \"category\": \"travel\"},\n", + " {\"content\": \"Adventure hiking in mountain regions\", \"category\": \"travel\"},\n", + " {\"content\": \"Cultural experiences in Asian cities\", \"category\": \"travel\"},\n", + " {\"content\": \"Financial planning for retirement\", \"category\": \"finance\"},\n", + "]\n", + "\n", + "# Generate random embeddings for demonstration\n", + "# In practice, you would use a real embedding model\n", + "data_to_load = []\n", + "\n", + "# Use reduced dimensions if LeanVec compression is applied\n", + "vector_dims = config.get(\"reduce\", dims)\n", + "print(f\"Creating vectors with {vector_dims} dimensions (reduced from {dims} if applicable)\")\n", + "\n", + "for i, doc in enumerate(sample_documents):\n", + " # Create a random vector with some category-based clustering\n", + " base_vector = np.random.random(vector_dims).astype(np.float32)\n", + " \n", + " # Add some category-based similarity (optional, for demo purposes)\n", + " category_offset = hash(doc[\"category\"]) % 100 / 1000.0\n", + " base_vector[0] += category_offset\n", + " \n", + " # Convert to the datatype specified in config\n", + " if config[\"datatype\"] == \"float16\":\n", + " base_vector = base_vector.astype(np.float16)\n", + " \n", + " data_to_load.append({\n", + " \"content\": doc[\"content\"],\n", + " \"category\": doc[\"category\"],\n", + " \"embedding\": array_to_buffer(base_vector, dtype=config[\"datatype\"])\n", + " })\n", + "\n", + "# Load data into the index\n", + "index.load(data_to_load)\n", + "print(f\"✅ Loaded {len(data_to_load)} documents into the index\")\n", + "\n", + "# Wait a moment for indexing to complete\n", + "import time\n", + "time.sleep(2)\n", + "\n", + "# Verify the data was loaded\n", + "info = index.info()\n", + "print(f\" Index now contains {info.get('num_docs', 0)} documents\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Performing Vector Searches\n", + "\n", + "Now let's perform some vector similarity searches using our SVS-VAMANA index." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🔍 Vector Search Results:\n", + "==================================================\n" + ] + } + ], + "source": [ + "# Create a query vector (in practice, this would be an embedding of your query text)\n", + "# Important: Query vector must match the index datatype and dimensions\n", + "vector_dims = config.get(\"reduce\", dims)\n", + "if config[\"datatype\"] == \"float16\":\n", + " query_vector = np.random.random(vector_dims).astype(np.float16)\n", + "else:\n", + " query_vector = np.random.random(vector_dims).astype(np.float32)\n", + "\n", + "# Perform a vector similarity search\n", + "query = VectorQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " num_results=5\n", + ")\n", + "\n", + "results = index.query(query)\n", + "\n", + "print(\"🔍 Vector Search Results:\")\n", + "print(\"=\" * 50)\n", + "for i, result in enumerate(results, 1):\n", + " distance = result.get('vector_distance', 'N/A')\n", + " print(f\"{i}. [{result['category']}] {result['content']}\")\n", + " print(f\" Distance: {distance:.4f}\" if isinstance(distance, (int, float)) else f\" Distance: {distance}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Understanding Compression Types\n", + "\n", + "SVS-VAMANA supports different compression algorithms that trade off between memory usage and search quality. Let's explore the available options." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Compression Recommendations for Different Priorities:\n", + "============================================================\n", + "\n", + "MEMORY Priority:\n", + " Compression: LeanVec4x8\n", + " Datatype: float16\n", + " Dimensionality reduction: 1024 → 512\n", + " Search window size: 20\n", + " Memory savings: 81.2%\n", + "\n", + "SPEED Priority:\n", + " Compression: LeanVec4x8\n", + " Datatype: float16\n", + " Dimensionality reduction: 1024 → 256\n", + " Search window size: 40\n", + " Memory savings: 90.6%\n", + "\n", + "BALANCED Priority:\n", + " Compression: LeanVec4x8\n", + " Datatype: float16\n", + " Dimensionality reduction: 1024 → 512\n", + " Search window size: 30\n", + " Memory savings: 81.2%\n" + ] + } + ], + "source": [ + "# Compare different compression priorities\n", + "print(\"Compression Recommendations for Different Priorities:\")\n", + "print(\"=\" * 60)\n", + "\n", + "priorities = [\"memory\", \"speed\", \"balanced\"]\n", + "for priority in priorities:\n", + " config = CompressionAdvisor.recommend(dims=dims, priority=priority)\n", + " savings = CompressionAdvisor.estimate_memory_savings(\n", + " config[\"compression\"],\n", + " dims,\n", + " config.get(\"reduce\")\n", + " )\n", + " \n", + " print(f\"\\n{priority.upper()} Priority:\")\n", + " print(f\" Compression: {config['compression']}\")\n", + " print(f\" Datatype: {config['datatype']}\")\n", + " if \"reduce\" in config:\n", + " print(f\" Dimensionality reduction: {dims} → {config['reduce']}\")\n", + " print(f\" Search window size: {config['search_window_size']}\")\n", + " print(f\" Memory savings: {savings}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compression Types Explained\n", + "\n", + "SVS-VAMANA offers several compression algorithms:\n", + "\n", + "### LVQ (Learned Vector Quantization)\n", + "- **LVQ4**: 4 bits per dimension (87.5% memory savings)\n", + "- **LVQ4x4**: 8 bits per dimension (75% memory savings)\n", + "- **LVQ4x8**: 12 bits per dimension (62.5% memory savings)\n", + "- **LVQ8**: 8 bits per dimension (75% memory savings)\n", + "\n", + "### LeanVec (Compression + Dimensionality Reduction)\n", + "- **LeanVec4x8**: 12 bits per dimension + dimensionality reduction\n", + "- **LeanVec8x8**: 16 bits per dimension + dimensionality reduction\n", + "\n", + "The CompressionAdvisor automatically chooses the best compression type based on your vector dimensions and priority." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Memory Savings by Vector Dimension:\n", + "==================================================\n", + "Dims Compression Savings Strategy\n", + "--------------------------------------------------\n", + "384 LVQ4x4 75.0% LVQ\n", + "768 LVQ4x4 75.0% LVQ\n", + "1024 LeanVec4x8 81.2% LeanVec\n", + "1536 LeanVec4x8 81.2% LeanVec\n", + "3072 LeanVec4x8 81.2% LeanVec\n" + ] + } + ], + "source": [ + "# Demonstrate compression savings for different vector dimensions\n", + "test_dimensions = [384, 768, 1024, 1536, 3072]\n", + "\n", + "print(\"Memory Savings by Vector Dimension:\")\n", + "print(\"=\" * 50)\n", + "print(f\"{'Dims':<6} {'Compression':<12} {'Savings':<8} {'Strategy'}\")\n", + "print(\"-\" * 50)\n", + "\n", + "for dims in test_dimensions:\n", + " config = CompressionAdvisor.recommend(dims=dims, priority=\"balanced\")\n", + " savings = CompressionAdvisor.estimate_memory_savings(\n", + " config[\"compression\"],\n", + " dims,\n", + " config.get(\"reduce\")\n", + " )\n", + " \n", + " strategy = \"LeanVec\" if dims >= 1024 else \"LVQ\"\n", + " print(f\"{dims:<6} {config['compression']:<12} {savings:>6.1f}% {strategy}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hybrid Queries with SVS-VAMANA\n", + "\n", + "SVS-VAMANA can be combined with other Redis search capabilities for powerful hybrid queries that filter by metadata while performing vector similarity search." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🔍 Hybrid Search Results (Technology category only):\n", + "=======================================================\n" + ] + } + ], + "source": [ + "# Perform a hybrid search: vector similarity + category filter\n", + "hybrid_query = VectorQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " num_results=3\n", + ")\n", + "\n", + "# Add a filter to only search within \"technology\" category\n", + "hybrid_query.set_filter(\"@category:{technology}\")\n", + "\n", + "filtered_results = index.query(hybrid_query)\n", + "\n", + "print(\"🔍 Hybrid Search Results (Technology category only):\")\n", + "print(\"=\" * 55)\n", + "for i, result in enumerate(filtered_results, 1):\n", + " distance = result.get('vector_distance', 'N/A')\n", + " print(f\"{i}. [{result['category']}] {result['content']}\")\n", + " print(f\" Distance: {distance:.4f}\" if isinstance(distance, (int, float)) else f\" Distance: {distance}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Performance Monitoring\n", + "\n", + "Let's examine the index statistics to understand the performance characteristics of our SVS-VAMANA index." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "📊 Index Statistics:\n", + "==============================\n", + "Documents: 0\n", + "Vector index size: 0.00 MB\n", + "Total indexing time: 1.58 seconds\n", + "Memory efficiency calculation requires documents and vector index size > 0\n" + ] + } + ], + "source": [ + "# Get detailed index information\n", + "info = index.info()\n", + "\n", + "print(\"📊 Index Statistics:\")\n", + "print(\"=\" * 30)\n", + "print(f\"Documents: {info.get('num_docs', 0)}\")\n", + "\n", + "# Handle vector_index_sz_mb which might be a string\n", + "vector_size = info.get('vector_index_sz_mb', 0)\n", + "if isinstance(vector_size, str):\n", + " try:\n", + " vector_size = float(vector_size)\n", + " except ValueError:\n", + " vector_size = 0.0\n", + "print(f\"Vector index size: {vector_size:.2f} MB\")\n", + "\n", + "# Handle total_indexing_time which might also be a string\n", + "indexing_time = info.get('total_indexing_time', 0)\n", + "if isinstance(indexing_time, str):\n", + " try:\n", + " indexing_time = float(indexing_time)\n", + " except ValueError:\n", + " indexing_time = 0.0\n", + "print(f\"Total indexing time: {indexing_time:.2f} seconds\")\n", + "\n", + "# Calculate memory efficiency\n", + "if info.get('num_docs', 0) > 0 and vector_size > 0:\n", + " mb_per_doc = vector_size / info.get('num_docs', 1)\n", + " print(f\"Memory per document: {mb_per_doc:.4f} MB\")\n", + " \n", + " # Estimate for larger datasets\n", + " for scale in [1000, 10000, 100000]:\n", + " estimated_mb = mb_per_doc * scale\n", + " print(f\"Estimated size for {scale:,} docs: {estimated_mb:.1f} MB\")\n", + "else:\n", + " print(\"Memory efficiency calculation requires documents and vector index size > 0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Manual Configuration (Advanced)\n", + "\n", + "For advanced users who want full control over SVS-VAMANA parameters, you can manually configure the algorithm instead of using CompressionAdvisor." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Manual SVS-VAMANA Configuration:\n", + "========================================\n", + " algorithm: svs-vamana\n", + " datatype: float32\n", + " distance_metric: cosine\n", + " graph_max_degree: 64\n", + " construction_window_size: 300\n", + " search_window_size: 40\n", + " compression: LVQ4x4\n", + " training_threshold: 10000\n", + "\n", + "Estimated memory savings: 75.0%\n" + ] + } + ], + "source": [ + "# Example of manual SVS-VAMANA configuration\n", + "manual_schema = {\n", + " \"index\": {\n", + " \"name\": \"svs_manual\",\n", + " \"prefix\": \"manual\",\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"content\", \"type\": \"text\"},\n", + " {\n", + " \"name\": \"embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 768,\n", + " \"algorithm\": \"svs-vamana\",\n", + " \"datatype\": \"float32\",\n", + " \"distance_metric\": \"cosine\",\n", + " \n", + " # Graph construction parameters\n", + " \"graph_max_degree\": 64, # Higher = better recall, more memory\n", + " \"construction_window_size\": 300, # Higher = better quality, slower build\n", + " \n", + " # Search parameters\n", + " \"search_window_size\": 40, # Higher = better recall, slower search\n", + " \n", + " # Compression settings\n", + " \"compression\": \"LVQ4x4\", # Choose compression type\n", + " \"training_threshold\": 10000, # Min vectors before compression training\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "print(\"Manual SVS-VAMANA Configuration:\")\n", + "print(\"=\" * 40)\n", + "vector_attrs = manual_schema[\"fields\"][1][\"attrs\"]\n", + "for key, value in vector_attrs.items():\n", + " if key != \"dims\": # Skip dims as it's obvious\n", + " print(f\" {key}: {value}\")\n", + "\n", + "# Calculate memory savings for this configuration\n", + "manual_savings = CompressionAdvisor.estimate_memory_savings(\n", + " \"LVQ4x4\", 768, None\n", + ")\n", + "print(f\"\\nEstimated memory savings: {manual_savings}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Best Practices and Tips\n", + "\n", + "### When to Use SVS-VAMANA\n", + "- **Large datasets** (>10K vectors) where memory efficiency matters\n", + "- **High-dimensional vectors** (>512 dimensions) that benefit from compression\n", + "- **Applications** that can tolerate slight recall trade-offs for speed and memory savings\n", + "\n", + "### Parameter Tuning Guidelines\n", + "- **Start with CompressionAdvisor** recommendations\n", + "- **Increase search_window_size** if you need higher recall\n", + "- **Use LeanVec** for high-dimensional vectors (≥1024 dims)\n", + "- **Use LVQ** for lower-dimensional vectors (<1024 dims)\n", + "\n", + "### Performance Considerations\n", + "- **Index build time** increases with higher construction_window_size\n", + "- **Search latency** increases with higher search_window_size\n", + "- **Memory usage** decreases with more aggressive compression\n", + "- **Recall quality** may decrease with more aggressive compression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup\n", + "\n", + "Clean up the indices created in this demo." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cleaned up svs_demo index\n" + ] + } + ], + "source": [ + "# Clean up demo indices\n", + "try:\n", + " index.delete()\n", + " print(\"Cleaned up svs_demo index\")\n", + "except:\n", + " print(\"- svs_demo index was already deleted or doesn't exist\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index ca900d63..221663e6 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -21,5 +21,7 @@ User guides provide helpful resources for using RedisVL and its different compon 06_rerankers 07_message_history 08_semantic_router +09_svs_vamana +10_embeddings_cache 11_advanced_queries -``` +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3fc4202f..ec4e08b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "redisvl" -version = "0.10.0" +version = "0.11.0" description = "Python client library and CLI for using Redis as a vector database" authors = [{ name = "Redis Inc.", email = "applied.ai@redis.com" }] requires-python = ">=3.9,<3.14" diff --git a/redisvl/exceptions.py b/redisvl/exceptions.py index 2d8b5bf4..79748c67 100644 --- a/redisvl/exceptions.py +++ b/redisvl/exceptions.py @@ -31,3 +31,25 @@ class QueryValidationError(RedisVLError): """Error when validating a query.""" pass + + +class RedisModuleVersionError(RedisVLError): + """Error when Redis or module versions are incompatible with requested features.""" + + @classmethod + def for_svs_vamana(cls, min_redis_version: str): + """Create error for unsupported SVS-VAMANA. + + Args: + min_redis_version: Minimum required Redis version + + Returns: + RedisModuleVersionError with formatted message + """ + message = ( + f"SVS-VAMANA requires Redis >= {min_redis_version} with RediSearch >= 2.8.10. " + f"Options: 1) Upgrade Redis Stack, " + f"2) Use algorithm='hnsw' or 'flat', " + f"3) Remove compression parameters" + ) + return cls(message) diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 4d7ee9b5..e9916518 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -67,6 +67,7 @@ from redisvl.exceptions import ( QueryValidationError, + RedisModuleVersionError, RedisSearchError, RedisVLError, SchemaValidationError, @@ -83,10 +84,14 @@ from redisvl.redis.connection import ( RedisConnectionFactory, convert_index_info_to_schema, + supports_svs, + supports_svs_async, ) +from redisvl.redis.constants import SVS_MIN_REDIS_VERSION from redisvl.schema import IndexSchema, StorageType from redisvl.schema.fields import ( VECTOR_NORM_MAP, + SVSVectorField, VectorDistanceMetric, VectorIndexAlgorithm, ) @@ -228,6 +233,12 @@ def _storage(self) -> BaseStorage: index_schema=self.schema ) + def _uses_svs_vamana(self) -> bool: + """Check if schema contains any SVS-VAMANA vector fields.""" + return any( + isinstance(field, SVSVectorField) for field in self.schema.fields.values() + ) + def _validate_query(self, query: BaseQuery) -> None: """Validate a query.""" if isinstance(query, VectorQuery): @@ -535,6 +546,15 @@ def set_client(self, redis_client: SyncRedisClient, **kwargs): self.__redis_client = redis_client return self + def _check_svs_support(self) -> None: + """Validate SVS-VAMANA support. + + Raises: + RedisModuleVersionError: If SVS-VAMANA requirements are not met. + """ + if not supports_svs(self._redis_client): + raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) + def create(self, overwrite: bool = False, drop: bool = False) -> None: """Create an index in Redis with the current schema and properties. @@ -566,6 +586,10 @@ def create(self, overwrite: bool = False, drop: bool = False) -> None: if not isinstance(overwrite, bool): raise TypeError("overwrite must be of type bool") + # Check if schema uses SVS-VAMANA and validate Redis capabilities + if self._uses_svs_vamana(): + self._check_svs_support() + if self.exists(): if not overwrite: logger.info("Index already exists, not overwriting.") @@ -1302,6 +1326,16 @@ async def _info(name: str, redis_client: AsyncRedisClient) -> Dict[str, Any]: f"Error while fetching {name} index info: {str(e)}" ) from e + async def _check_svs_support_async(self) -> None: + """Validate SVS-VAMANA support. + + Raises: + RedisModuleVersionError: If SVS-VAMANA requirements are not met. + """ + client = await self._get_client() + if not await supports_svs_async(client): + raise RedisModuleVersionError.for_svs_vamana(SVS_MIN_REDIS_VERSION) + async def create(self, overwrite: bool = False, drop: bool = False) -> None: """Asynchronously create an index in Redis with the current schema and properties. @@ -1335,6 +1369,10 @@ async def create(self, overwrite: bool = False, drop: bool = False) -> None: if not isinstance(overwrite, bool): raise TypeError("overwrite must be of type bool") + # Check if schema uses SVS-VAMANA and validate Redis capabilities + if self._uses_svs_vamana(): + await self._check_svs_support_async() + if await self.exists(): if not overwrite: logger.info("Index already exists, not overwriting.") diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 0dce70c2..2f2cf994 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -1,4 +1,5 @@ import os +from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, overload from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from warnings import warn @@ -15,7 +16,11 @@ from redis.sentinel import Sentinel from redisvl import __version__ -from redisvl.redis.constants import REDIS_URL_ENV_VAR +from redisvl.redis.constants import ( + REDIS_URL_ENV_VAR, + SVS_MIN_REDIS_VERSION, + SVS_MIN_SEARCH_VERSION, +) from redisvl.redis.utils import convert_bytes, is_cluster_url from redisvl.types import AsyncRedisClient, RedisClient, SyncRedisClient from redisvl.utils.utils import deprecated_function @@ -67,16 +72,16 @@ def _strip_cluster_from_url_and_kwargs( return cleaned_url, cleaned_kwargs -def compare_versions(version1: str, version2: str): +def is_version_gte(version1: str, version2: str) -> bool: """ - Compare two Redis version strings numerically. + Check if version1 >= version2. Parameters: - version1 (str): The first version string (e.g., "7.2.4"). - version2 (str): The second version string (e.g., "6.2.1"). + version1 (str): The first version string (e.g., "7.2.4"). + version2 (str): The second version string (e.g., "6.2.1"). Returns: - int: -1 if version1 < version2, 0 if version1 == version2, 1 if version1 > version2. + bool: True if version1 >= version2, False otherwise. """ v1_parts = list(map(int, version1.split("."))) v2_parts = list(map(int, version2.split("."))) @@ -101,6 +106,64 @@ def unpack_redis_modules(module_list: List[Dict[str, Any]]) -> Dict[str, Any]: return {module["name"]: module["ver"] for module in module_list} +def supports_svs(client: SyncRedisClient) -> bool: + """Check if Redis server supports SVS-VAMANA. + + Args: + client: Sync Redis client instance + + Returns: + True if SVS-VAMANA is supported, False otherwise + """ + info = client.info("server") # type: ignore[union-attr] + redis_version = info.get("redis_version", "0.0.0") # type: ignore[union-attr] + + modules = RedisConnectionFactory.get_modules(client) + search_ver = modules.get("search", 0) + searchlight_ver = modules.get("searchlight", 0) + + # Check if SVS-VAMANA requirements are met + redis_ok = is_version_gte(redis_version, SVS_MIN_REDIS_VERSION) + + # Check either search or searchlight module (only one is typically installed) + # RediSearch is the open-source module, SearchLight is the enterprise version + modules_ok = ( + search_ver >= SVS_MIN_SEARCH_VERSION + or searchlight_ver >= SVS_MIN_SEARCH_VERSION + ) + + return redis_ok and modules_ok + + +async def supports_svs_async(client: AsyncRedisClient) -> bool: + """Async version of _supports_svs. + + Args: + client: Async Redis client instance + + Returns: + True if SVS-VAMANA is supported, False otherwise + """ + info = await client.info("server") # type: ignore[union-attr] + redis_version = info.get("redis_version", "0.0.0") # type: ignore[union-attr] + + modules = await RedisConnectionFactory.get_modules_async(client) + search_ver = modules.get("search", 0) + searchlight_ver = modules.get("searchlight", 0) + + # Check if SVS-VAMANA requirements are met + redis_ok = is_version_gte(redis_version, SVS_MIN_REDIS_VERSION) + + # Check either search or searchlight module (only one is typically installed) + # RediSearch is the open-source module, SearchLight is the enterprise version + modules_ok = ( + search_ver >= SVS_MIN_SEARCH_VERSION + or searchlight_ver >= SVS_MIN_SEARCH_VERSION + ) + + return redis_ok and modules_ok + + def get_address_from_env() -> str: """Get Redis URL from environment variable.""" redis_url = os.getenv(REDIS_URL_ENV_VAR) @@ -227,6 +290,57 @@ def parse_vector_attrs(attrs): # Default to float32 if missing normalized["datatype"] = "float32" + # Handle SVS-VAMANA specific parameters + # Compression - Redis uses different internal names, so we need to map them + if "compression" in vector_attrs: + compression_value = vector_attrs["compression"] + # Map Redis internal names to our enum values + compression_mapping = { + "GlobalSQ8": "LVQ4x4", # Default mapping + "GlobalSQ4": "LVQ4", + # Add more mappings as we discover them + } + # Try to map, otherwise use the value as-is + normalized["compression"] = compression_mapping.get( + compression_value, compression_value + ) + + # Dimensionality reduction (reduce parameter) + if "reduce" in vector_attrs: + try: + normalized["reduce"] = int(vector_attrs["reduce"]) + except (ValueError, TypeError): + pass + + # Graph parameters + if "graph_max_degree" in vector_attrs: + try: + normalized["graph_max_degree"] = int(vector_attrs["graph_max_degree"]) + except (ValueError, TypeError): + pass + + if "construction_window_size" in vector_attrs: + try: + normalized["construction_window_size"] = int( + vector_attrs["construction_window_size"] + ) + except (ValueError, TypeError): + pass + + if "search_window_size" in vector_attrs: + try: + normalized["search_window_size"] = int( + vector_attrs["search_window_size"] + ) + except (ValueError, TypeError): + pass + + if "epsilon" in vector_attrs: + try: + normalized["epsilon"] = float(vector_attrs["epsilon"]) + except (ValueError, TypeError): + pass + # Validate that we have required dims if "dims" not in normalized: # Could not parse dims - this field is not properly supported @@ -407,6 +521,7 @@ async def _get_aredis_connection( """ url = url or get_address_from_env() + client: AsyncRedisClient if url.startswith("redis+sentinel"): client = RedisConnectionFactory._redis_sentinel_client( url, AsyncRedis, **kwargs diff --git a/redisvl/redis/constants.py b/redisvl/redis/constants.py index 434a2ff5..fa6861d1 100644 --- a/redisvl/redis/constants.py +++ b/redisvl/redis/constants.py @@ -4,6 +4,17 @@ {"name": "searchlight", "ver": 20600}, ] +# SVS-VAMANA requires Redis 8.2+ with RediSearch 2.8.10+ +SVS_REQUIRED_MODULES = [ + {"name": "search", "ver": 20810}, # RediSearch 2.8.10+ + {"name": "searchlight", "ver": 20810}, +] + +# Minimum Redis version for SVS-VAMANA +SVS_MIN_REDIS_VERSION = "8.2.0" +# Minimum search module version for SVS-VAMANA (2.8.10) +SVS_MIN_SEARCH_VERSION = 20810 + # default tag separator REDIS_TAG_SEPARATOR = "," diff --git a/redisvl/schema/__init__.py b/redisvl/schema/__init__.py index c835ccd5..a652e01a 100644 --- a/redisvl/schema/__init__.py +++ b/redisvl/schema/__init__.py @@ -1,10 +1,12 @@ from redisvl.schema.fields import ( BaseField, + CompressionType, FieldTypes, FlatVectorField, GeoField, HNSWVectorField, NumericField, + SVSVectorField, TagField, TextField, VectorDataType, @@ -24,6 +26,7 @@ "VectorDistanceMetric", "VectorDataType", "VectorIndexAlgorithm", + "CompressionType", "BaseField", "TextField", "TagField", @@ -31,5 +34,6 @@ "GeoField", "FlatVectorField", "HNSWVectorField", + "SVSVectorField", "validate_object", ] diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index 7533e8c4..3a3f2206 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -1,14 +1,43 @@ """ -RedisVL Fields, FieldAttributes, and Enums - -Reference Redis search source documentation as needed: https://redis.io/commands/ft.create/ -Reference Redis vector search documentation as needed: https://redis.io/docs/interact/search-and-query/advanced-concepts/vectors/ +RedisVL Schema Fields and Attributes + +This module defines field types and their attributes for creating Redis search indices. + +Field Types: + - TextField: Full-text search with stemming, phonetic matching + - TagField: Exact-match categorical data (tags, categories, IDs) + - NumericField: Numeric values for range queries and sorting + - GeoField: Geographic coordinates for location-based search + - VectorField: Vector embeddings for semantic similarity search + - FlatVectorField: Exact search (100% recall) + - HNSWVectorField: Approximate nearest neighbor search (fast, high recall) + - SVSVectorField: Compressed vector search with memory savings + +Common Vector Field Attributes (all algorithms): + - dims: Number of dimensions in the vector (e.g., 768, 1536) + - algorithm: Indexing algorithm ('flat', 'hnsw', or 'svs-vamana') + - datatype: Float precision ('float16', 'float32', 'float64', 'bfloat16') + Note: SVS-VAMANA only supports 'float16' and 'float32' + - distance_metric: Similarity metric ('COSINE', 'L2', 'IP') + - initial_cap: Initial capacity hint for memory allocation (optional) + - index_missing: Allow searching for documents without this field (optional) + +Algorithm-Specific Parameters: + - FLAT: block_size (memory management for dynamic indices) + - HNSW: m, ef_construction, ef_runtime, epsilon (graph tuning) + - SVS-VAMANA: graph_max_degree, construction_window_size, search_window_size, + compression, reduce, training_threshold (VAMANA graph algorithm + with Intel hardware optimization and vector compression) + +References: + - Redis FT.CREATE: https://redis.io/commands/ft.create/ + - Vector Search: https://redis.io/docs/interact/search-and-query/advanced-concepts/vectors/ """ from enum import Enum from typing import Any, Dict, Literal, Optional, Tuple, Type, Union -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from redis.commands.search.field import Field as RedisField from redis.commands.search.field import GeoField as RedisGeoField from redis.commands.search.field import NumericField as RedisNumericField @@ -16,8 +45,11 @@ from redis.commands.search.field import TextField as RedisTextField from redis.commands.search.field import VectorField as RedisVectorField +from redisvl.utils.log import get_logger from redisvl.utils.utils import norm_cosine_distance, norm_l2_distance +logger = get_logger(__name__) + VECTOR_NORM_MAP = { "COSINE": norm_cosine_distance, "L2": norm_l2_distance, @@ -51,6 +83,18 @@ class VectorDataType(str, Enum): class VectorIndexAlgorithm(str, Enum): FLAT = "FLAT" HNSW = "HNSW" + SVS_VAMANA = "SVS-VAMANA" + + +class CompressionType(str, Enum): + """Vector compression types for SVS-VAMANA algorithm""" + + LVQ4 = "LVQ4" + LVQ4x4 = "LVQ4x4" + LVQ4x8 = "LVQ4x8" + LVQ8 = "LVQ8" + LeanVec4x8 = "LeanVec4x8" + LeanVec8x8 = "LeanVec8x8" ### Field Attributes ### @@ -111,12 +155,12 @@ class GeoFieldAttributes(BaseFieldAttributes): class BaseVectorFieldAttributes(BaseModel): - """Base vector field attributes shared by both FLAT and HNSW fields""" + """Base vector field attributes shared by FLAT, HNSW, and SVS-VAMANA fields""" dims: int """Dimensionality of the vector embeddings field""" algorithm: VectorIndexAlgorithm - """The indexing algorithm for the field: HNSW or FLAT""" + """The indexing algorithm for the field: FLAT, HNSW, or SVS-VAMANA""" datatype: VectorDataType = Field(default=VectorDataType.FLOAT32) """The float datatype for the vector embeddings""" distance_metric: VectorDistanceMetric = Field(default=VectorDistanceMetric.COSINE) @@ -148,27 +192,120 @@ def field_data(self) -> Dict[str, Any]: class FlatVectorFieldAttributes(BaseVectorFieldAttributes): - """FLAT vector field attributes""" + """FLAT vector field attributes for exact nearest neighbor search.""" algorithm: Literal[VectorIndexAlgorithm.FLAT] = VectorIndexAlgorithm.FLAT - """The indexing algorithm for the vector field""" + """The indexing algorithm (fixed as 'flat')""" + block_size: Optional[int] = None - """Block size to hold amount of vectors in a contiguous array. This is useful when the index is dynamic with respect to addition and deletion""" + """Block size for processing (optional) - improves batch operation throughput""" class HNSWVectorFieldAttributes(BaseVectorFieldAttributes): - """HNSW vector field attributes""" + """HNSW vector field attributes for approximate nearest neighbor search.""" algorithm: Literal[VectorIndexAlgorithm.HNSW] = VectorIndexAlgorithm.HNSW - """The indexing algorithm for the vector field""" + """The indexing algorithm (fixed as 'hnsw')""" m: int = Field(default=16) - """Number of max outgoing edges for each graph node in each layer""" + """Max outgoing edges per node in each layer (default: 16, range: 8-64)""" + ef_construction: int = Field(default=200) - """Number of max allowed potential outgoing edges candidates for each node in the graph during build time""" + """Max edge candidates during build time (default: 200, range: 100-800)""" + ef_runtime: int = Field(default=10) - """Number of maximum top candidates to hold during KNN search""" + """Max top candidates during search (default: 10) - primary tuning parameter""" + + epsilon: float = Field(default=0.01) + """Range search boundary factor (default: 0.01)""" + + +class SVSVectorFieldAttributes(BaseVectorFieldAttributes): + """SVS-VAMANA vector field attributes with compression support.""" + + algorithm: Literal[VectorIndexAlgorithm.SVS_VAMANA] = ( + VectorIndexAlgorithm.SVS_VAMANA + ) + """The indexing algorithm for the vector field""" + + # Graph Construction Parameters + graph_max_degree: int = Field(default=40) + """Max edges per node (default: 40) - affects recall vs memory""" + + construction_window_size: int = Field(default=250) + """Build-time candidates (default: 250) - affects quality vs build time""" + + search_window_size: int = Field(default=20) + """Search candidates (default: 20) - primary tuning parameter""" + epsilon: float = Field(default=0.01) - """Relative factor that sets the boundaries in which a range query may search for candidates""" + """Range query boundary factor (default: 0.01)""" + + # Compression Parameters + compression: Optional[CompressionType] = None + """Vector compression: LVQ4, LVQ8, LeanVec4x8, LeanVec8x8""" + + reduce: Optional[int] = None + """Dimensionality reduction for LeanVec types (must be < dims)""" + + training_threshold: Optional[int] = None + """Min vectors before compression training (default: 10,240)""" + + @model_validator(mode="after") + def validate_svs_params(self): + """Validate SVS-VAMANA specific constraints""" + # Datatype validation: SVS only supports FLOAT16 and FLOAT32 + if self.datatype not in (VectorDataType.FLOAT16, VectorDataType.FLOAT32): + raise ValueError( + f"SVS-VAMANA only supports FLOAT16 and FLOAT32 datatypes. " + f"Got: {self.datatype}. " + f"Unsupported types: BFLOAT16, FLOAT64, INT8, UINT8." + ) + + # Reduce validation: must be less than dims and only valid with LeanVec + if self.reduce is not None: + if self.reduce >= self.dims: + raise ValueError( + f"reduce ({self.reduce}) must be less than dims ({self.dims})" + ) + + # Validate that reduce is only used with LeanVec compression + if self.compression is None: + raise ValueError( + "reduce parameter requires compression to be set. " + "Use LeanVec4x8 or LeanVec8x8 compression with reduce." + ) + + if not self.compression.value.startswith("LeanVec"): + raise ValueError( + f"reduce parameter is only supported with LeanVec compression types. " + f"Got compression={self.compression.value}. " + f"Either use LeanVec4x8/LeanVec8x8 or remove the reduce parameter." + ) + + # LeanVec without reduce is not recommended + if ( + self.compression + and self.compression.value.startswith("LeanVec") + and not self.reduce + ): + logger.warning( + f"LeanVec compression selected without 'reduce'. " + f"Consider setting reduce={self.dims//2} for better performance" + ) + + if self.graph_max_degree and self.graph_max_degree < 32: + logger.warning( + f"graph_max_degree={self.graph_max_degree} is low. " + f"Consider values between 32-64 for better recall." + ) + + if self.search_window_size and self.search_window_size > 100: + logger.warning( + f"search_window_size={self.search_window_size} is high. " + f"This may impact query latency. Consider values between 20-50." + ) + + return self ### Field Classes ### @@ -352,7 +489,7 @@ def as_redis_field(self) -> RedisField: class FlatVectorField(BaseField): - "Vector field with a FLAT index (brute force nearest neighbors search)" + """Vector field with FLAT (exact search) indexing for exact nearest neighbor search.""" type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR attrs: FlatVectorFieldAttributes @@ -367,7 +504,7 @@ def as_redis_field(self) -> RedisField: class HNSWVectorField(BaseField): - """Vector field with an HNSW index (approximate nearest neighbors search)""" + """Vector field with HNSW (Hierarchical Navigable Small World) indexing for approximate nearest neighbor search.""" type: Literal["vector"] = "vector" attrs: HNSWVectorFieldAttributes @@ -387,6 +524,33 @@ def as_redis_field(self) -> RedisField: return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name) +class SVSVectorField(BaseField): + """Vector field with SVS-VAMANA indexing and compression for memory-efficient approximate nearest neighbor search.""" + + type: Literal[FieldTypes.VECTOR] = FieldTypes.VECTOR + attrs: SVSVectorFieldAttributes + + def as_redis_field(self) -> RedisField: + name, as_name = self._handle_names() + field_data = self.attrs.field_data + field_data.update( + { + "GRAPH_MAX_DEGREE": self.attrs.graph_max_degree, + "CONSTRUCTION_WINDOW_SIZE": self.attrs.construction_window_size, + "SEARCH_WINDOW_SIZE": self.attrs.search_window_size, + "EPSILON": self.attrs.epsilon, + } + ) + # Add compression parameters if specified + if self.attrs.compression is not None: + field_data["COMPRESSION"] = self.attrs.compression + if self.attrs.reduce is not None: + field_data["REDUCE"] = self.attrs.reduce + if self.attrs.training_threshold is not None: + field_data["TRAINING_THRESHOLD"] = self.attrs.training_threshold + return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name) + + FIELD_TYPE_MAP = { "tag": TagField, "text": TextField, @@ -397,6 +561,7 @@ def as_redis_field(self) -> RedisField: VECTOR_FIELD_TYPE_MAP = { "flat": FlatVectorField, "hnsw": HNSWVectorField, + "svs-vamana": SVSVectorField, } diff --git a/redisvl/utils/__init__.py b/redisvl/utils/__init__.py index e69de29b..639e6e2a 100644 --- a/redisvl/utils/__init__.py +++ b/redisvl/utils/__init__.py @@ -0,0 +1,3 @@ +from redisvl.utils.compression import CompressionAdvisor + +__all__ = ["CompressionAdvisor"] diff --git a/redisvl/utils/compression.py b/redisvl/utils/compression.py new file mode 100644 index 00000000..c2a32a45 --- /dev/null +++ b/redisvl/utils/compression.py @@ -0,0 +1,230 @@ +"""SVS-VAMANA compression configuration utilities.""" + +from typing import Literal, Optional + +from pydantic import BaseModel, Field + + +class SVSConfig(BaseModel): + """SVS-VAMANA configuration model. + + Attributes: + algorithm: Always "svs-vamana" + datatype: Vector datatype (float16, float32) + compression: Compression type (LVQ4, LeanVec4x8, etc.) + reduce: Reduced dimensionality (only for LeanVec) + graph_max_degree: Max edges per node + construction_window_size: Build-time candidates + search_window_size: Query-time candidates + """ + + algorithm: Literal["svs-vamana"] = "svs-vamana" + datatype: Optional[str] = None + compression: Optional[str] = None + reduce: Optional[int] = Field( + default=None, description="Reduced dimensionality (only for LeanVec)" + ) + graph_max_degree: Optional[int] = None + construction_window_size: Optional[int] = None + search_window_size: Optional[int] = None + + +class CompressionAdvisor: + """Helper to recommend compression settings based on vector characteristics. + + This class provides utilities to: + - Recommend optimal SVS-VAMANA configurations based on vector dimensions and priorities + - Estimate memory savings from compression and dimensionality reduction + + Examples: + >>> # Get recommendations for high-dimensional vectors + >>> config = CompressionAdvisor.recommend(dims=1536, priority="balanced") + >>> config.compression + 'LeanVec4x8' + >>> config.reduce + 768 + + >>> # Estimate memory savings + >>> savings = CompressionAdvisor.estimate_memory_savings( + ... compression="LeanVec4x8", + ... dims=1536, + ... reduce=768 + ... ) + >>> savings + 81.2 + """ + + # Dimension thresholds + HIGH_DIM_THRESHOLD = 1024 + + # Compression bit rates (bits per dimension) + COMPRESSION_BITS = { + "LVQ4": 4, + "LVQ4x4": 8, + "LVQ4x8": 12, + "LVQ8": 8, + "LeanVec4x8": 12, + "LeanVec8x8": 16, + } + + @staticmethod + def recommend( + dims: int, + priority: Literal["speed", "memory", "balanced"] = "balanced", + datatype: Optional[str] = None, + ) -> SVSConfig: + """Recommend compression settings based on dimensions and priorities. + + Args: + dims: Vector dimensionality (must be > 0) + priority: Optimization priority: + - "memory": Maximize memory savings + - "speed": Optimize for query speed + - "balanced": Balance between memory and speed + datatype: Override datatype (default: float16 for high-dim, float32 for low-dim) + + Returns: + dict: Complete SVS-VAMANA configuration including: + - algorithm: "svs-vamana" + - datatype: Recommended datatype + - compression: Compression type + - reduce: Dimensionality reduction (for LeanVec only) + - graph_max_degree: Graph connectivity + - construction_window_size: Build-time candidates + - search_window_size: Query-time candidates + + Raises: + ValueError: If dims <= 0 + + Examples: + >>> # High-dimensional embeddings (e.g., OpenAI ada-002) + >>> config = CompressionAdvisor.recommend(dims=1536, priority="memory") + >>> config.compression + 'LeanVec4x8' + >>> config.reduce + 768 + + >>> # Lower-dimensional embeddings + >>> config = CompressionAdvisor.recommend(dims=384, priority="speed") + >>> config.compression + 'LVQ4x8' + """ + if dims <= 0: + raise ValueError(f"dims must be positive, got {dims}") + + # High-dimensional vectors (>= 1024) - use LeanVec + if dims >= CompressionAdvisor.HIGH_DIM_THRESHOLD: + base_datatype = datatype or "float16" + + if priority == "memory": + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=64, + construction_window_size=300, + compression="LeanVec4x8", + reduce=dims // 2, + search_window_size=20, + ) + elif priority == "speed": + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=64, + construction_window_size=300, + compression="LeanVec4x8", + reduce=max(256, dims // 4), + search_window_size=40, + ) + else: # balanced + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=64, + construction_window_size=300, + compression="LeanVec4x8", + reduce=dims // 2, + search_window_size=30, + ) + + # Lower-dimensional vectors - use LVQ + else: + base_datatype = datatype or "float32" + + if priority == "memory": + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + compression="LVQ4", + ) + elif priority == "speed": + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + compression="LVQ4x8", + ) + else: # balanced + return SVSConfig( + algorithm="svs-vamana", + datatype=base_datatype, + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + compression="LVQ4x4", + ) + + @staticmethod + def estimate_memory_savings( + compression: str, dims: int, reduce: Optional[int] = None + ) -> float: + """Estimate memory savings percentage from compression. + + Calculates the percentage of memory saved compared to uncompressed float32 vectors. + + Args: + compression: Compression type (e.g., "LVQ4", "LeanVec4x8") + dims: Original vector dimensionality + reduce: Reduced dimensionality (for LeanVec compression) + + Returns: + float: Memory savings percentage (0-100) + + Examples: + >>> # LeanVec with dimensionality reduction + >>> CompressionAdvisor.estimate_memory_savings( + ... compression="LeanVec4x8", + ... dims=1536, + ... reduce=768 + ... ) + 81.2 + + >>> # LVQ without dimensionality reduction + >>> CompressionAdvisor.estimate_memory_savings( + ... compression="LVQ4", + ... dims=384 + ... ) + 87.5 + """ + # Base bits per dimension (float32) + base_bits = 32 + + # Compressed bits per dimension + compression_bits = CompressionAdvisor.COMPRESSION_BITS.get( + compression, base_bits + ) + + # Account for dimensionality reduction + effective_dims = reduce if reduce else dims + + # Calculate savings + original_size = dims * base_bits + compressed_size = effective_dims * compression_bits + savings = (1 - compressed_size / original_size) * 100 + + return round(savings, 1) diff --git a/tests/conftest.py b/tests/conftest.py index 692ce77d..9ff9795b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ from testcontainers.compose import DockerCompose from redisvl.index.index import AsyncSearchIndex, SearchIndex -from redisvl.redis.connection import RedisConnectionFactory, compare_versions +from redisvl.redis.connection import RedisConnectionFactory, is_version_gte from redisvl.redis.utils import array_to_buffer from redisvl.utils.vectorize import HFTextVectorizer @@ -699,7 +699,7 @@ def skip_if_redis_version_below(client, min_version: str, message: str = None): message: Custom skip message """ redis_version = get_redis_version(client) - if not compare_versions(redis_version, min_version): + if not is_version_gte(redis_version, min_version): skip_msg = message or f"Redis version {redis_version} < {min_version} required" pytest.skip(skip_msg) @@ -716,7 +716,7 @@ async def skip_if_redis_version_below_async( message: Custom skip message """ redis_version = await get_redis_version_async(client) - if not compare_versions(redis_version, min_version): + if not is_version_gte(redis_version, min_version): skip_msg = message or f"Redis version {redis_version} < {min_version} required" pytest.skip(skip_msg) diff --git a/tests/integration/test_semantic_router.py b/tests/integration/test_semantic_router.py index d082ceaa..47e7c451 100644 --- a/tests/integration/test_semantic_router.py +++ b/tests/integration/test_semantic_router.py @@ -11,7 +11,7 @@ Route, RoutingConfig, ) -from redisvl.redis.connection import compare_versions +from redisvl.redis.connection import is_version_gte from tests.conftest import skip_if_no_redisearch, skip_if_redis_version_below @@ -159,7 +159,7 @@ def test_add_route(semantic_router): assert "political speech" in route.references redis_version = semantic_router._index.client.info()["redis_version"] - if compare_versions(redis_version, "7.0.0"): + if is_version_gte(redis_version, "7.0.0"): match = semantic_router("political speech") print(match, flush=True) assert match is not None diff --git a/tests/integration/test_svs_integration.py b/tests/integration/test_svs_integration.py new file mode 100644 index 00000000..2cccebc5 --- /dev/null +++ b/tests/integration/test_svs_integration.py @@ -0,0 +1,495 @@ +""" +Integration tests for SVS-VAMANA vector indexing. + +These tests require: +- Redis >= 8.2.0 +- RediSearch >= 2.8.10 +- Environment variable: REDISVL_TEST_SVS=1 + +Run with: + REDISVL_TEST_SVS=1 pytest tests/integration/test_svs_integration.py + +To run with Redis 8.2+ in Docker: + docker run -d -p 6379:6379 redis/redis-stack-server:8.2.2-v0 + REDISVL_TEST_SVS=1 pytest tests/integration/test_svs_integration.py + +Or set the Redis image for the test suite: + REDIS_IMAGE=redis/redis-stack-server:8.2.2-v0 REDISVL_TEST_SVS=1 pytest tests/integration/test_svs_integration.py +""" + +import os + +import numpy as np +import pytest + +from redisvl.exceptions import RedisModuleVersionError +from redisvl.index import SearchIndex +from redisvl.query import VectorQuery +from redisvl.redis.connection import supports_svs +from redisvl.redis.utils import array_to_buffer +from redisvl.schema import IndexSchema +from redisvl.utils import CompressionAdvisor + +# Skip all tests in this module if REDISVL_TEST_SVS is not set +pytestmark = pytest.mark.skipif( + not os.getenv("REDISVL_TEST_SVS"), + reason="SVS tests require REDISVL_TEST_SVS=1 and Redis 8.2+", +) + + +@pytest.fixture +def svs_schema_lvq(worker_id): + """Create SVS-VAMANA schema with LVQ compression.""" + return IndexSchema.from_dict( + { + "index": { + "name": f"svs_lvq_{worker_id}", + "prefix": f"svs_lvq_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "content", "type": "text"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 384, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LVQ4x4", + "graph_max_degree": 40, + "construction_window_size": 250, + "search_window_size": 20, + }, + }, + ], + } + ) + + +@pytest.fixture +def svs_schema_leanvec(worker_id): + """Create SVS-VAMANA schema with LeanVec compression and dimensionality reduction.""" + return IndexSchema.from_dict( + { + "index": { + "name": f"svs_leanvec_{worker_id}", + "prefix": f"svs_leanvec_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "content", "type": "text"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 768, + "graph_max_degree": 64, + "construction_window_size": 300, + "search_window_size": 30, + }, + }, + ], + } + ) + + +@pytest.fixture +def svs_index_lvq(svs_schema_lvq, client): + """Create SVS-VAMANA index with LVQ compression.""" + index = SearchIndex(schema=svs_schema_lvq, redis_client=client) + index.create(overwrite=True) + yield index + index.delete(drop=True) + + +@pytest.fixture +def svs_index_leanvec(svs_schema_leanvec, client): + """Create SVS-VAMANA index with LeanVec compression.""" + index = SearchIndex(schema=svs_schema_leanvec, redis_client=client) + index.create(overwrite=True) + yield index + index.delete(drop=True) + + +def generate_test_vectors(dims, count=100, dtype="float32"): + """Generate random test vectors.""" + vectors = [] + for i in range(count): + vector = np.random.random(dims).astype(dtype) + vectors.append( + { + "id": f"doc_{i}", + "content": f"This is test document {i}", + "embedding": array_to_buffer(vector, dtype=dtype), + } + ) + return vectors + + +class TestSVSCapabilityDetection: + """Test SVS-VAMANA capability detection.""" + + def test_check_svs_capabilities(self, client): + """Test that SVS-VAMANA is supported on the test Redis instance.""" + # These tests require Redis 8.2+ with RediSearch 2.8.10+ + assert supports_svs(client) is True, ( + "SVS-VAMANA not supported. " + "Requires Redis >= 8.2.0 with RediSearch >= 2.8.10" + ) + + +class TestSVSIndexCreation: + """Test creating SVS-VAMANA indices with various configurations.""" + + def test_create_svs_index_lvq(self, svs_index_lvq): + """Test creating SVS-VAMANA index with LVQ compression.""" + assert svs_index_lvq.exists() + + # Verify index info + info = svs_index_lvq.info() + assert info["num_docs"] == 0 + + def test_create_svs_index_leanvec(self, svs_index_leanvec): + """Test creating SVS-VAMANA index with LeanVec compression.""" + assert svs_index_leanvec.exists() + + # Verify index info + info = svs_index_leanvec.info() + assert info["num_docs"] == 0 + + def test_create_svs_with_compression_advisor(self, client, worker_id): + """Test creating SVS-VAMANA index using CompressionAdvisor.""" + dims = 768 + config = CompressionAdvisor.recommend(dims=dims, priority="balanced") + + schema = IndexSchema.from_dict( + { + "index": { + "name": f"svs_advisor_{worker_id}", + "prefix": f"svs_advisor_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + **config, + "distance_metric": "cosine", + }, + }, + ], + } + ) + + index = SearchIndex(schema=schema, redis_client=client) + index.create(overwrite=True) + + try: + assert index.exists() + info = index.info() + assert info["num_docs"] == 0 + finally: + index.delete(drop=True) + + +class TestSVSDataIngestion: + """Test loading data into SVS-VAMANA indices.""" + + def test_load_data_lvq(self, svs_index_lvq): + """Test loading data into SVS-VAMANA index with LVQ compression.""" + vectors = generate_test_vectors(dims=384, count=50, dtype="float32") + svs_index_lvq.load(vectors) + + # Verify data was loaded + info = svs_index_lvq.info() + assert info["num_docs"] == 50 + + def test_load_data_leanvec(self, svs_index_leanvec): + """Test loading data into SVS-VAMANA index with LeanVec compression.""" + vectors = generate_test_vectors(dims=1536, count=50, dtype="float32") + svs_index_leanvec.load(vectors) + + # Verify data was loaded + info = svs_index_leanvec.info() + assert info["num_docs"] == 50 + + def test_load_large_batch(self, svs_index_lvq): + """Test loading larger batch of data.""" + vectors = generate_test_vectors(dims=384, count=200, dtype="float32") + svs_index_lvq.load(vectors) + + # Verify data was loaded + info = svs_index_lvq.info() + assert info["num_docs"] == 200 + + +class TestSVSQuerying: + """Test querying SVS-VAMANA indices.""" + + def test_vector_query_lvq(self, svs_index_lvq): + """Test vector similarity search on SVS-VAMANA index with LVQ.""" + # Load test data + vectors = generate_test_vectors(dims=384, count=100, dtype="float32") + svs_index_lvq.load(vectors) + + # Create query vector + query_vector = np.random.random(384).astype(np.float32) + + # Execute query + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id", "content"], + num_results=10, + ) + + results = svs_index_lvq.query(query) + + # Verify results + assert len(results) <= 10 + assert all("id" in result for result in results) + assert all("content" in result for result in results) + + def test_vector_query_leanvec(self, svs_index_leanvec): + """Test vector similarity search on SVS-VAMANA index with LeanVec.""" + # Load test data + vectors = generate_test_vectors(dims=1536, count=100, dtype="float32") + svs_index_leanvec.load(vectors) + + # Create query vector + query_vector = np.random.random(1536).astype(np.float32) + + # Execute query + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id", "content"], + num_results=5, + ) + + results = svs_index_leanvec.query(query) + + # Verify results + assert len(results) <= 5 + assert all("id" in result for result in results) + + def test_query_with_filters(self, svs_index_lvq): + """Test vector query with filters on SVS-VAMANA index.""" + # Load test data with specific IDs + vectors = [] + for i in range(50): + vector = np.random.random(384).astype(np.float32) + vectors.append( + { + "id": f"category_a_{i}" if i < 25 else f"category_b_{i}", + "content": f"Document {i}", + "embedding": array_to_buffer(vector, dtype="float32"), + } + ) + svs_index_lvq.load(vectors) + + # Query with filter + query_vector = np.random.random(384).astype(np.float32) + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id", "content"], + num_results=10, + filter_expression="@id:{category_a*}", + ) + + results = svs_index_lvq.query(query) + + # Verify all results match filter + assert len(results) <= 10 + assert all(result["id"].startswith("category_a") for result in results) + + +class TestSVSFromExisting: + """Test loading existing SVS-VAMANA indices.""" + + def test_from_existing_lvq(self, svs_index_lvq, client): + """Test loading existing SVS-VAMANA index with LVQ compression.""" + # Load some data + vectors = generate_test_vectors(dims=384, count=20, dtype="float32") + svs_index_lvq.load(vectors) + + # Load the index from existing + loaded_index = SearchIndex.from_existing( + svs_index_lvq.name, redis_client=client + ) + + # Verify the loaded index + assert loaded_index.exists() + assert loaded_index.name == svs_index_lvq.name + + # Verify schema was loaded correctly + embedding_field = loaded_index.schema.fields["embedding"] + assert embedding_field.attrs.algorithm.value == "SVS-VAMANA" + assert embedding_field.attrs.compression.value == "LVQ4x4" + assert embedding_field.attrs.dims == 384 + + # Verify data is accessible + info = loaded_index.info() + assert info["num_docs"] == 20 + + def test_from_existing_leanvec(self, svs_index_leanvec, client): + """Test loading existing SVS-VAMANA index with LeanVec compression.""" + # Load some data + vectors = generate_test_vectors(dims=1536, count=20, dtype="float32") + svs_index_leanvec.load(vectors) + + # Load the index from existing + loaded_index = SearchIndex.from_existing( + svs_index_leanvec.name, redis_client=client + ) + + # Verify the loaded index + assert loaded_index.exists() + assert loaded_index.name == svs_index_leanvec.name + + # Verify schema was loaded correctly + embedding_field = loaded_index.schema.fields["embedding"] + assert embedding_field.attrs.algorithm.value == "SVS-VAMANA" + assert embedding_field.attrs.compression.value == "LeanVec4x8" + assert embedding_field.attrs.dims == 1536 + assert embedding_field.attrs.reduce == 768 + + # Verify data is accessible + info = loaded_index.info() + assert info["num_docs"] == 20 + + +class TestSVSCompressionTypes: + """Test different compression types for SVS-VAMANA.""" + + @pytest.mark.parametrize( + "compression,dims,dtype", + [ + ("LVQ4", 384, "float32"), + ("LVQ4x4", 384, "float32"), + ("LVQ4x8", 384, "float32"), + ("LVQ8", 384, "float32"), + ], + ) + def test_lvq_compression_types(self, client, worker_id, compression, dims, dtype): + """Test various LVQ compression types.""" + schema = IndexSchema.from_dict( + { + "index": { + "name": f"svs_{compression.lower()}_{worker_id}", + "prefix": f"svs_{compression.lower()}_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + "algorithm": "svs-vamana", + "datatype": dtype, + "distance_metric": "cosine", + "compression": compression, + }, + }, + ], + } + ) + + index = SearchIndex(schema=schema, redis_client=client) + index.create(overwrite=True) + + try: + # Load data + vectors = generate_test_vectors(dims=dims, count=50, dtype=dtype) + index.load(vectors) + + # Verify + assert index.exists() + info = index.info() + assert info["num_docs"] == 50 + + # Query + query_vector = np.random.random(dims).astype(dtype) + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id"], + num_results=5, + ) + results = index.query(query) + assert len(results) <= 5 + finally: + index.delete(drop=True) + + @pytest.mark.parametrize( + "compression,dims,reduce,dtype", + [ + ("LeanVec4x8", 1024, 512, "float16"), + ("LeanVec4x8", 1536, 768, "float16"), + ("LeanVec8x8", 1536, 768, "float16"), + ], + ) + def test_leanvec_compression_types( + self, client, worker_id, compression, dims, reduce, dtype + ): + """Test various LeanVec compression types with dimensionality reduction.""" + schema = IndexSchema.from_dict( + { + "index": { + "name": f"svs_{compression.lower()}_{reduce}_{worker_id}", + "prefix": f"svs_{compression.lower()}_{reduce}_{worker_id}", + }, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": dims, + "algorithm": "svs-vamana", + "datatype": dtype, + "distance_metric": "cosine", + "compression": compression, + "reduce": reduce, + }, + }, + ], + } + ) + + index = SearchIndex(schema=schema, redis_client=client) + index.create(overwrite=True) + + try: + # Load data + vectors = generate_test_vectors(dims=dims, count=50, dtype="float32") + index.load(vectors) + + # Verify + assert index.exists() + info = index.info() + assert info["num_docs"] == 50 + + # Query + query_vector = np.random.random(dims).astype(np.float32) + query = VectorQuery( + vector=query_vector, + vector_field_name="embedding", + return_fields=["id"], + num_results=5, + ) + results = index.query(query) + assert len(results) <= 5 + finally: + index.delete(drop=True) diff --git a/tests/unit/test_compression_advisor.py b/tests/unit/test_compression_advisor.py new file mode 100644 index 00000000..2a9cbbe0 --- /dev/null +++ b/tests/unit/test_compression_advisor.py @@ -0,0 +1,250 @@ +"""Unit tests for CompressionAdvisor utility.""" + +import pytest + +from redisvl.utils.compression import CompressionAdvisor, SVSConfig + + +class TestCompressionAdvisorRecommend: + """Tests for CompressionAdvisor.recommend() method.""" + + def test_recommend_high_dim_memory_priority(self): + """Test memory-optimized config for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=1536, priority="memory") + + assert config.algorithm == "svs-vamana" + assert config.datatype == "float16" + assert config.compression == "LeanVec4x8" + assert config.reduce == 768 # dims // 2 + assert config.graph_max_degree == 64 + assert config.construction_window_size == 300 + assert config.search_window_size == 20 + + def test_recommend_high_dim_speed_priority(self): + """Test speed-optimized config for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=1536, priority="speed") + + assert config.algorithm == "svs-vamana" + assert config.datatype == "float16" + assert config.compression == "LeanVec4x8" + assert config.reduce == 384 # dims // 4 + assert config.graph_max_degree == 64 + assert config.construction_window_size == 300 + assert config.search_window_size == 40 + + def test_recommend_high_dim_balanced_priority(self): + """Test balanced config for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=1536, priority="balanced") + + assert config.algorithm == "svs-vamana" + assert config.datatype == "float16" + assert config.compression == "LeanVec4x8" + assert config.reduce == 768 # dims // 2 + assert config.graph_max_degree == 64 + assert config.construction_window_size == 300 + assert config.search_window_size == 30 + + def test_recommend_high_dim_default_priority(self): + """Test default priority (balanced) for high-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=2048) + + assert config.compression == "LeanVec4x8" + assert config.reduce == 1024 + assert config.search_window_size == 30 + + def test_recommend_low_dim_memory_priority(self): + """Test memory-optimized config for low-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=384, priority="memory") + + assert config.algorithm == "svs-vamana" + assert config.datatype == "float32" + assert config.compression == "LVQ4" + assert config.reduce is None # LVQ doesn't use reduce + assert config.graph_max_degree == 40 + assert config.construction_window_size == 250 + assert config.search_window_size == 20 + + def test_recommend_low_dim_speed_priority(self): + """Test speed-optimized config for low-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=384, priority="speed") + + assert config.algorithm == "svs-vamana" + assert config.datatype == "float32" + assert config.compression == "LVQ4x8" + assert config.reduce is None + assert config.graph_max_degree == 40 + assert config.construction_window_size == 250 + assert config.search_window_size == 20 + + def test_recommend_low_dim_balanced_priority(self): + """Test balanced config for low-dimensional vectors.""" + config = CompressionAdvisor.recommend(dims=768, priority="balanced") + + assert config.algorithm == "svs-vamana" + assert config.datatype == "float32" + assert config.compression == "LVQ4x4" + assert config.reduce is None + assert config.graph_max_degree == 40 + assert config.construction_window_size == 250 + assert config.search_window_size == 20 + + def test_recommend_threshold_boundary_low(self): + """Test recommendation at threshold boundary (1023 dims).""" + config = CompressionAdvisor.recommend(dims=1023) + + # Should use LVQ (below threshold) + assert config.compression in ["LVQ4", "LVQ4x4", "LVQ4x8"] + assert config.datatype == "float32" + assert config.reduce is None + + def test_recommend_threshold_boundary_high(self): + """Test recommendation at threshold boundary (1024 dims).""" + config = CompressionAdvisor.recommend(dims=1024) + + # Should use LeanVec (at threshold) + assert config.compression == "LeanVec4x8" + assert config.datatype == "float16" + assert config.reduce is not None + + def test_recommend_custom_datatype(self): + """Test custom datatype override.""" + config = CompressionAdvisor.recommend(dims=1536, datatype="float32") + + assert config.datatype == "float32" + + def test_recommend_speed_reduce_minimum(self): + """Test that speed priority respects minimum reduce value.""" + config = CompressionAdvisor.recommend(dims=1024, priority="speed") + + # dims // 4 = 256, max(256, 256) = 256 + assert config.reduce == 256 + + config = CompressionAdvisor.recommend(dims=512, priority="speed") + # Below threshold, should use LVQ + assert config.reduce is None + + def test_recommend_invalid_dims_zero(self): + """Test that zero dims raises ValueError.""" + with pytest.raises(ValueError, match="dims must be positive"): + CompressionAdvisor.recommend(dims=0) + + def test_recommend_invalid_dims_negative(self): + """Test that negative dims raises ValueError.""" + with pytest.raises(ValueError, match="dims must be positive"): + CompressionAdvisor.recommend(dims=-100) + + +class TestCompressionAdvisorEstimateMemorySavings: + """Tests for CompressionAdvisor.estimate_memory_savings() method.""" + + def test_estimate_lvq4_no_reduce(self): + """Test memory savings for LVQ4 without dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LVQ4", dims=384 + ) + + # Original: 384 * 32 = 12,288 bits + # Compressed: 384 * 4 = 1,536 bits + # Savings: (1 - 1536/12288) * 100 = 87.5% + assert savings == 87.5 + + def test_estimate_lvq4x4_no_reduce(self): + """Test memory savings for LVQ4x4 without dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LVQ4x4", dims=768 + ) + + # Original: 768 * 32 = 24,576 bits + # Compressed: 768 * 8 = 6,144 bits + # Savings: (1 - 6144/24576) * 100 = 75.0% + assert savings == 75.0 + + def test_estimate_leanvec4x8_with_reduce(self): + """Test memory savings for LeanVec4x8 with dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LeanVec4x8", dims=1536, reduce=768 + ) + + # Original: 1536 * 32 = 49,152 bits + # Compressed: 768 * 12 = 9,216 bits + # Savings: (1 - 9216/49152) * 100 = 81.25% -> 81.2% (rounded) + assert savings == 81.2 + + def test_estimate_leanvec4x8_no_reduce(self): + """Test memory savings for LeanVec4x8 without dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LeanVec4x8", dims=1536 + ) + + # Original: 1536 * 32 = 49,152 bits + # Compressed: 1536 * 12 = 18,432 bits + # Savings: (1 - 18432/49152) * 100 = 62.5% + assert savings == 62.5 + + def test_estimate_leanvec8x8_with_reduce(self): + """Test memory savings for LeanVec8x8 with dimensionality reduction.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LeanVec8x8", dims=2048, reduce=1024 + ) + + # Original: 2048 * 32 = 65,536 bits + # Compressed: 1024 * 16 = 16,384 bits + # Savings: (1 - 16384/65536) * 100 = 75.0% + assert savings == 75.0 + + def test_estimate_unknown_compression(self): + """Test that unknown compression type defaults to no savings.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="UNKNOWN", dims=512 + ) + + # Should default to base_bits (32), so no savings + # Original: 512 * 32 = 16,384 bits + # Compressed: 512 * 32 = 16,384 bits + # Savings: 0% + assert savings == 0.0 + + def test_estimate_rounding(self): + """Test that savings are rounded to 1 decimal place.""" + savings = CompressionAdvisor.estimate_memory_savings( + compression="LVQ4", dims=333 + ) + + # Original: 333 * 32 = 10,656 bits + # Compressed: 333 * 4 = 1,332 bits + # Savings: (1 - 1332/10656) * 100 = 87.5% + assert savings == 87.5 + assert isinstance(savings, float) + + +class TestSVSConfigModel: + """Tests for SVSConfig Pydantic model structure.""" + + def test_svs_config_structure(self): + """Test that SVSConfig can be constructed with all fields.""" + config = SVSConfig( + algorithm="svs-vamana", + datatype="float16", + compression="LeanVec4x8", + reduce=768, + graph_max_degree=64, + construction_window_size=300, + search_window_size=30, + ) + + assert config.algorithm == "svs-vamana" + assert config.reduce == 768 + + def test_svs_config_without_reduce(self): + """Test that SVSConfig can be constructed without reduce field.""" + config = SVSConfig( + algorithm="svs-vamana", + datatype="float32", + compression="LVQ4", + graph_max_degree=40, + construction_window_size=250, + search_window_size=20, + ) + + assert config.reduce is None + assert config.compression == "LVQ4" diff --git a/tests/unit/test_error_handling.py b/tests/unit/test_error_handling.py index 8c31e8ac..478743f9 100644 --- a/tests/unit/test_error_handling.py +++ b/tests/unit/test_error_handling.py @@ -38,6 +38,7 @@ def test_redis_error_in_create_method(self, mock_validate): # Create a mock schema schema = Mock(spec=IndexSchema) schema.redis_fields = ["test_field"] + schema.fields = {} # Dict[str, BaseField] for compatibility schema.index = Mock() schema.index.name = "test_index" schema.index.prefix = "test:" @@ -68,6 +69,7 @@ def test_unexpected_error_in_create_method(self, mock_validate): # Create a mock schema schema = Mock(spec=IndexSchema) schema.redis_fields = ["test_field"] + schema.fields = {} # Dict[str, BaseField] for compatibility schema.index = Mock() schema.index.name = "test_index" schema.index.prefix = "test:" diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index bd8b41ac..eeeda604 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -1,3 +1,5 @@ +from unittest.mock import AsyncMock, Mock, patch + import pytest from redis.commands.search.field import GeoField as RedisGeoField from redis.commands.search.field import NumericField as RedisNumericField @@ -5,12 +7,16 @@ from redis.commands.search.field import TextField as RedisTextField from redis.commands.search.field import VectorField as RedisVectorField +from redisvl.exceptions import RedisModuleVersionError +from redisvl.index import AsyncSearchIndex, SearchIndex +from redisvl.schema import IndexSchema from redisvl.schema.fields import ( FieldFactory, FlatVectorField, GeoField, HNSWVectorField, NumericField, + SVSVectorField, TagField, TextField, ) @@ -72,6 +78,24 @@ def create_hnsw_vector_field(**kwargs): return HNSWVectorField(**defaults) +def create_svs_vector_field(**kwargs): + defaults = { + "name": "example_svsvectorfield", + "attrs": { + "dims": 128, + "algorithm": "SVS-VAMANA", + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 40, + "construction_window_size": 250, + "search_window_size": 20, + "epsilon": 0.01, + }, + } + defaults["attrs"].update(kwargs) + return SVSVectorField(**defaults) + + # Tests for field schema creation and validation @pytest.mark.parametrize( "schema_func,field_class", @@ -422,3 +446,534 @@ def test_field_factory_with_new_attributes(): ) assert isinstance(vector_field, FlatVectorField) assert vector_field.attrs.index_missing == True + + +def test_svs_vector_field_creation(): + """Test basic SVS-VAMANA vector field creation.""" + svs_field = create_svs_vector_field() + assert svs_field.name == "example_svsvectorfield" + assert svs_field.attrs.algorithm == "SVS-VAMANA" + assert svs_field.attrs.dims == 128 + assert svs_field.attrs.datatype.value == "FLOAT32" + assert svs_field.attrs.distance_metric.value == "COSINE" + assert svs_field.attrs.graph_max_degree == 40 + assert svs_field.attrs.construction_window_size == 250 + assert svs_field.attrs.search_window_size == 20 + assert svs_field.attrs.epsilon == 0.01 + + +def test_svs_vector_field_as_redis_field(): + """Test SVS-VAMANA field conversion to Redis field.""" + svs_field = create_svs_vector_field() + redis_field = svs_field.as_redis_field() + + assert isinstance(redis_field, RedisVectorField) + assert redis_field.name == "example_svsvectorfield" + + # Check that SVS-VAMANA specific parameters are in args + assert "GRAPH_MAX_DEGREE" in redis_field.args + assert "CONSTRUCTION_WINDOW_SIZE" in redis_field.args + assert "SEARCH_WINDOW_SIZE" in redis_field.args + assert "EPSILON" in redis_field.args + + +def test_svs_vector_field_default_params(): + """Test SVS-VAMANA field with default parameters.""" + svs_field = SVSVectorField( + name="test_vector", + attrs={ + "dims": 768, + "algorithm": "SVS-VAMANA", + "datatype": "float32", + "distance_metric": "cosine", + }, + ) + + # Check defaults are applied + assert svs_field.attrs.graph_max_degree == 40 + assert svs_field.attrs.construction_window_size == 250 + assert svs_field.attrs.search_window_size == 20 + assert svs_field.attrs.epsilon == 0.01 + assert svs_field.attrs.compression is None + assert svs_field.attrs.reduce is None + assert svs_field.attrs.training_threshold is None + + +def test_svs_vector_field_with_custom_graph_params(): + """Test SVS-VAMANA field with custom graph parameters.""" + svs_field = create_svs_vector_field( + graph_max_degree=64, + construction_window_size=500, + search_window_size=40, + epsilon=0.02, + ) + + redis_field = svs_field.as_redis_field() + + # Verify custom parameters are set + assert redis_field.args[redis_field.args.index("GRAPH_MAX_DEGREE") + 1] == 64 + assert ( + redis_field.args[redis_field.args.index("CONSTRUCTION_WINDOW_SIZE") + 1] == 500 + ) + assert redis_field.args[redis_field.args.index("SEARCH_WINDOW_SIZE") + 1] == 40 + assert redis_field.args[redis_field.args.index("EPSILON") + 1] == 0.02 + + +def test_svs_vector_field_with_lvq4_compression(): + """Test SVS-VAMANA field with LVQ4 compression.""" + svs_field = create_svs_vector_field(compression="LVQ4") + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LVQ4" + + +def test_svs_vector_field_with_lvq8_compression(): + """Test SVS-VAMANA field with LVQ8 compression.""" + svs_field = create_svs_vector_field(compression="LVQ8") + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LVQ8" + + +def test_svs_vector_field_with_leanvec_compression(): + """Test SVS-VAMANA field with LeanVec4x8 compression.""" + svs_field = create_svs_vector_field(compression="LeanVec4x8") + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec4x8" + + +def test_svs_vector_field_with_leanvec_and_reduce(): + """Test SVS-VAMANA field with LeanVec compression and reduce parameter.""" + svs_field = create_svs_vector_field(dims=768, compression="LeanVec4x8", reduce=384) + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec4x8" + assert "REDUCE" in redis_field.args + assert redis_field.args[redis_field.args.index("REDUCE") + 1] == 384 + + +def test_svs_vector_field_with_training_threshold(): + """Test SVS-VAMANA field with training_threshold parameter.""" + svs_field = create_svs_vector_field(compression="LVQ4", training_threshold=10000) + redis_field = svs_field.as_redis_field() + + assert "TRAINING_THRESHOLD" in redis_field.args + assert redis_field.args[redis_field.args.index("TRAINING_THRESHOLD") + 1] == 10000 + + +def test_svs_vector_field_reduce_with_lvq4_raises_error(): + """Test that reduce parameter with LVQ4 compression raises ValueError.""" + with pytest.raises( + ValueError, match="reduce parameter is only supported with LeanVec" + ): + create_svs_vector_field(dims=768, compression="LVQ4", reduce=384) + + +def test_svs_vector_field_reduce_with_lvq8_raises_error(): + """Test that reduce parameter with LVQ8 compression raises ValueError.""" + with pytest.raises( + ValueError, match="reduce parameter is only supported with LeanVec" + ): + create_svs_vector_field(dims=768, compression="LVQ8", reduce=384) + + +def test_svs_vector_field_reduce_without_compression_raises_error(): + """Test that reduce parameter without compression raises ValueError.""" + with pytest.raises(ValueError, match="reduce parameter requires compression"): + create_svs_vector_field(dims=768, reduce=384) + + +def test_svs_vector_field_reduce_greater_than_dims_raises_error(): + """Test that reduce >= dims raises ValueError.""" + with pytest.raises(ValueError, match="reduce.*must be less than dims"): + create_svs_vector_field(dims=768, compression="LeanVec4x8", reduce=768) + + +def test_svs_vector_field_reduce_equal_to_dims_raises_error(): + """Test that reduce == dims raises ValueError.""" + with pytest.raises(ValueError, match="reduce.*must be less than dims"): + create_svs_vector_field(dims=768, compression="LeanVec4x8", reduce=768) + + +def test_svs_vector_field_invalid_datatype_raises_error(): + """Test that invalid datatype (not float16/float32) raises ValueError.""" + with pytest.raises(Exception, match="SVS-VAMANA only supports FLOAT16 and FLOAT32"): + create_svs_vector_field(datatype="float64") + + +def test_svs_vector_field_float16_datatype(): + """Test SVS-VAMANA field with float16 datatype.""" + svs_field = create_svs_vector_field(datatype="float16") + redis_field = svs_field.as_redis_field() + + assert "TYPE" in redis_field.args + assert redis_field.args[redis_field.args.index("TYPE") + 1] == "FLOAT16" + + +def test_svs_vector_field_all_compression_types(): + """Test all valid compression types for SVS-VAMANA.""" + compression_types = ["LVQ4", "LVQ4x4", "LVQ4x8", "LVQ8", "LeanVec4x8", "LeanVec8x8"] + + for compression in compression_types: + svs_field = create_svs_vector_field(compression=compression) + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert ( + redis_field.args[redis_field.args.index("COMPRESSION") + 1] == compression + ) + + +def test_svs_vector_field_leanvec8x8_with_reduce(): + """Test SVS-VAMANA field with LeanVec8x8 compression and reduce.""" + svs_field = create_svs_vector_field(dims=1024, compression="LeanVec8x8", reduce=512) + redis_field = svs_field.as_redis_field() + + assert "COMPRESSION" in redis_field.args + assert redis_field.args[redis_field.args.index("COMPRESSION") + 1] == "LeanVec8x8" + assert "REDUCE" in redis_field.args + assert redis_field.args[redis_field.args.index("REDUCE") + 1] == 512 + + +def test_uses_svs_vamana_true(): + """Test _uses_svs_vamana returns True for SVS schema.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + {"name": "id", "type": "tag"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = Mock() + + with patch.object(SearchIndex, "_redis_client", mock_client): + index = SearchIndex(schema=schema) + assert index._uses_svs_vamana() is True + + +def test_check_svs_support_raises_error(): + """Test _check_svs_support raises error when not supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "7.2.4"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20612, "searchlight": 20612} + + index = SearchIndex(schema=schema) + + # Mock the _redis_client property + with patch.object( + type(index), + "_redis_client", + new_callable=lambda: property(lambda self: mock_client), + ): + with pytest.raises(RedisModuleVersionError) as exc_info: + index._check_svs_support() + + error_msg = str(exc_info.value) + assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg + + +def test_check_svs_support_passes(): + """Test _check_svs_support passes when supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = Mock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + index = SearchIndex(schema=schema) + + # Mock the _redis_client property + with patch.object( + type(index), + "_redis_client", + new_callable=lambda: property(lambda self: mock_client), + ): + # Should not raise + index._check_svs_support() + + +@pytest.mark.asyncio +async def test_check_svs_support_async_raises_error(): + """Test _check_svs_support_async raises error when not supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = AsyncMock() + mock_client.info.return_value = {"redis_version": "7.2.4"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules_async" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20612, "searchlight": 20612} + + index = AsyncSearchIndex(schema=schema) + + with patch.object(index, "_get_client", return_value=mock_client): + with pytest.raises(RedisModuleVersionError) as exc_info: + await index._check_svs_support_async() + + error_msg = str(exc_info.value) + assert "SVS-VAMANA requires Redis >= 8.2.0" in error_msg + + +@pytest.mark.asyncio +async def test_check_svs_support_async_passes(): + """Test _check_svs_support_async passes when supported.""" + schema_dict = { + "index": {"name": "test_index", "prefix": "doc"}, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 128, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + }, + }, + ], + } + schema = IndexSchema.from_dict(schema_dict) + + mock_client = AsyncMock() + mock_client.info.return_value = {"redis_version": "8.2.0"} + + with patch( + "redisvl.redis.connection.RedisConnectionFactory.get_modules_async" + ) as mock_get_modules: + mock_get_modules.return_value = {"search": 20810, "searchlight": 20810} + + index = AsyncSearchIndex(schema=schema) + + with patch.object(index, "_get_client", return_value=mock_client): + # Should not raise + await index._check_svs_support_async() + + +# Phase C: Warning Tests + + +def test_leanvec_without_reduce_warning(caplog): + """Test warning when LeanVec compression is used without reduce parameter.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + # No reduce parameter + }, + ) + + # Check warning was logged + assert len(caplog.records) == 1 + assert "LeanVec compression selected without 'reduce'" in caplog.records[0].message + assert "Consider setting reduce=768" in caplog.records[0].message + + +def test_leanvec_with_reduce_no_warning(caplog): + """Test no warning when LeanVec compression is used with reduce parameter.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 768, + }, + ) + + # No warnings should be logged + assert len(caplog.records) == 0 + + +def test_low_graph_max_degree_warning(caplog): + """Test warning when graph_max_degree is too low.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 16, # Too low + }, + ) + + # Check warning was logged + assert len(caplog.records) == 1 + assert "graph_max_degree=16 is low" in caplog.records[0].message + assert "Consider values between 32-64" in caplog.records[0].message + + +def test_normal_graph_max_degree_no_warning(caplog): + """Test no warning when graph_max_degree is in normal range.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 40, # Normal value + }, + ) + + # No warnings should be logged + assert len(caplog.records) == 0 + + +def test_high_search_window_size_warning(caplog): + """Test warning when search_window_size is too high.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "search_window_size": 150, # Too high + }, + ) + + # Check warning was logged + assert len(caplog.records) == 1 + assert "search_window_size=150 is high" in caplog.records[0].message + assert "This may impact query latency" in caplog.records[0].message + + +def test_normal_search_window_size_no_warning(caplog): + """Test no warning when search_window_size is in normal range.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 512, + "algorithm": "svs-vamana", + "datatype": "float32", + "distance_metric": "cosine", + "search_window_size": 30, # Normal value + }, + ) + + # No warnings should be logged + assert len(caplog.records) == 0 + + +def test_multiple_warnings(caplog): + """Test multiple warnings are logged when multiple issues exist.""" + import logging + + with caplog.at_level(logging.WARNING): + field = SVSVectorField( + name="embedding", + attrs={ + "dims": 1536, + "algorithm": "svs-vamana", + "datatype": "float16", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + # No reduce parameter - warning 1 + "graph_max_degree": 20, # Too low - warning 2 + "search_window_size": 120, # Too high - warning 3 + }, + ) + + # Check all three warnings were logged + assert len(caplog.records) == 3 + messages = [record.message for record in caplog.records] + assert any("LeanVec compression" in msg for msg in messages) + assert any("graph_max_degree=20" in msg for msg in messages) + assert any("search_window_size=120" in msg for msg in messages) diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index 68933938..fed3d13a 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -690,3 +690,293 @@ def test_explicit_none_fields_excluded(self, sample_hash_schema): assert "title" in validated assert "rating" not in validated assert "location" not in validated + + +class TestSVSVamanaValidation: + """Tests for SVS-VAMANA specific validation rules.""" + + def test_svs_vamana_with_valid_float32(self): + """Test SVS-VAMANA field with valid float32 datatype.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float32", + "distance_metric": "cosine", + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + assert len(schema.fields) == 1 + + def test_svs_vamana_with_valid_float16(self): + """Test SVS-VAMANA field with valid float16 datatype.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float16", + "distance_metric": "cosine", + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + + def test_svs_vamana_with_invalid_float64(self): + """Test SVS-VAMANA field rejects float64 datatype.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float64", + "distance_metric": "cosine", + }, + } + ], + } + # Should raise validation error + with pytest.raises( + Exception, match="SVS-VAMANA only supports FLOAT16 and FLOAT32" + ): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_with_leanvec_and_reduce(self): + """Test SVS-VAMANA with LeanVec compression and reduce parameter.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 384, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + + def test_svs_vamana_reduce_with_lvq4_raises_error(self): + """Test SVS-VAMANA rejects reduce parameter with LVQ4 compression.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LVQ4", + "reduce": 384, + }, + } + ], + } + # Should raise validation error + with pytest.raises( + Exception, match="reduce parameter is only supported with LeanVec" + ): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_reduce_without_compression_raises_error(self): + """Test SVS-VAMANA rejects reduce parameter without compression.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "reduce": 384, + }, + } + ], + } + # Should raise validation error + with pytest.raises(Exception, match="reduce parameter requires compression"): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_reduce_greater_than_dims_raises_error(self): + """Test SVS-VAMANA rejects reduce >= dims.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LeanVec4x8", + "reduce": 768, + }, + } + ], + } + # Should raise validation error + with pytest.raises(Exception, match="reduce.*must be less than dims"): + IndexSchema.from_dict(schema_dict) + + def test_svs_vamana_with_all_compression_types(self): + """Test SVS-VAMANA with all valid compression types.""" + compression_types = [ + "LVQ4", + "LVQ4x4", + "LVQ4x8", + "LVQ8", + "LeanVec4x8", + "LeanVec8x8", + ] + + for compression in compression_types: + schema_dict = { + "index": { + "name": f"test-svs-{compression}", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 128, + "datatype": "float32", + "distance_metric": "cosine", + "compression": compression, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + + def test_svs_vamana_with_graph_parameters(self): + """Test SVS-VAMANA with custom graph parameters.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "graph_max_degree": 64, + "construction_window_size": 500, + "search_window_size": 40, + "epsilon": 0.02, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + vector_field = schema.fields["embedding"] + assert vector_field.attrs.graph_max_degree == 64 + assert vector_field.attrs.construction_window_size == 500 + assert vector_field.attrs.search_window_size == 40 + assert vector_field.attrs.epsilon == 0.02 + + def test_svs_vamana_with_training_threshold(self): + """Test SVS-VAMANA with training_threshold parameter.""" + schema_dict = { + "index": { + "name": "test-svs-index", + "prefix": "test", + "storage_type": "hash", + }, + "fields": [ + { + "name": "embedding", + "type": "vector", + "attrs": { + "algorithm": "svs-vamana", + "dims": 768, + "datatype": "float32", + "distance_metric": "cosine", + "compression": "LVQ4", + "training_threshold": 10000, + }, + } + ], + } + # Should not raise any errors + schema = IndexSchema.from_dict(schema_dict) + assert schema is not None + vector_field = schema.fields["embedding"] + assert vector_field.attrs.training_threshold == 10000 diff --git a/uv.lock b/uv.lock index 09691c5e..cad34f92 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9, <3.14" resolution-markers = [ "python_full_version >= '3.13'", @@ -3581,19 +3581,19 @@ wheels = [ [[package]] name = "redis" -version = "6.2.0" +version = "6.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/9a/0551e01ba52b944f97480721656578c8a7c46b51b99d66814f85fe3a4f3e/redis-6.2.0.tar.gz", hash = "sha256:e821f129b75dde6cb99dd35e5c76e8c49512a5a0d8dfdc560b2fbd44b85ca977", size = 4639129, upload-time = "2025-05-28T05:01:18.91Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/67/e60968d3b0e077495a8fee89cf3f2373db98e528288a48f1ee44967f6e8c/redis-6.2.0-py3-none-any.whl", hash = "sha256:c8ddf316ee0aab65f04a11229e94a64b2618451dab7a67cb2f77eb799d872d5e", size = 278659, upload-time = "2025-05-28T05:01:16.955Z" }, + { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" }, ] [[package]] name = "redisvl" -version = "0.10.0" +version = "0.11.0" source = { editable = "." } dependencies = [ { name = "jsonpath-ng" },