Skip to content

Finalize spec for index schema and fields #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Feb 1, 2024
Merged

Finalize spec for index schema and fields #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9f046c3
wip
tylerhutcherson Jan 24, 2024
fb2067c
WIP
tylerhutcherson Jan 26, 2024
cfb87b2
handle old format temporarily
tylerhutcherson Jan 26, 2024
f7d9da1
refactor schema to 0.1.0
tylerhutcherson Jan 29, 2024
4bc7859
finish schema work and tests
tylerhutcherson Jan 30, 2024
a68a8f6
work on docs update
tylerhutcherson Jan 30, 2024
20c4075
doc tweaks
tylerhutcherson Jan 30, 2024
bee50a9
update notebook
tylerhutcherson Jan 30, 2024
33a9287
remove logger for now
tylerhutcherson Jan 30, 2024
212083b
fix formatting again
tylerhutcherson Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/_static/js/sidebar.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@ const toc = [
{ title: "Install", path: "/overview/installation.html" },
{ title: "CLI", path: "/user_guide/cli.html" },
]},
{ header: "User Guide", toc: [
{ header: "User Guides", toc: [
{ title: "Getting Started", path: "/user_guide/getting_started_01.html" },
{ title: "Query and Filter", path: "/user_guide/hybrid_queries_02.html" },
{ title: "JSON vs Hash Storage", path: "/user_guide/hash_vs_json_05.html" },
{ title: "Vectorizers", path: "/user_guide/vectorizers_04.html" },
{ title: "Semantic Caching", path: "/user_guide/llmcache_03.html" },
{ title: "JSON Storage", path: "/user_guide/hash_vs_json_05.html" }

]},
{ header: "API", toc: [
{ title: "Schema", path: "/api/indexschema.html"},
{ title: "Index", path: "/api/searchindex.html" },
{ title: "Query", path: "/api/query.html" },
{ title: "Filter", path: "/api/filter.html" },
Expand Down
30 changes: 23 additions & 7 deletions docs/api/indexschema.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

***********
IndexSchema
Schema
***********

IndexSchema
Expand All @@ -12,17 +12,14 @@ IndexSchema

.. autosummary::

IndexSchema.__init__
IndexSchema.name
IndexSchema.prefix
IndexSchema.key_separator
IndexSchema.storage_type
IndexSchema.index
IndexSchema.fields
IndexSchema.version
IndexSchema.field_names
IndexSchema.redis_fields
IndexSchema.add_field
IndexSchema.add_fields
IndexSchema.remove_field
IndexSchema.generate_fields
IndexSchema.from_yaml
IndexSchema.to_yaml
IndexSchema.from_dict
Expand All @@ -32,3 +29,22 @@ IndexSchema
:show-inheritance:
:inherited-members:
:members:


IndexInfo
=========

.. currentmodule:: redisvl.schema

.. autosummary::

IndexInfo.name
IndexInfo.prefix
IndexInfo.key_separator
IndexInfo.storage_type


.. autoclass:: IndexInfo
:show-inheritance:
:inherited-members:
:members:
3 changes: 2 additions & 1 deletion docs/api/searchindex.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ SearchIndex

.. autosummary::

SearchIndex.__init__
SearchIndex.from_yaml
SearchIndex.from_dict
SearchIndex.client
Expand All @@ -32,6 +31,8 @@ SearchIndex
SearchIndex.asearch
SearchIndex.query
SearchIndex.aquery
SearchIndex.query_batch
SearchIndex.aquery_batch
SearchIndex.delete
SearchIndex.adelete
SearchIndex.info
Expand Down
39 changes: 19 additions & 20 deletions docs/examples/openai_qna.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,15 +81,6 @@
"%pip install pandas wget tenacity tiktoken openai==0.28.1"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import redisvl"
]
},
{
"cell_type": "code",
"execution_count": 3,
Expand Down Expand Up @@ -653,7 +644,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -667,18 +658,22 @@
"source": [
"%%writefile wiki_schema.yaml\n",
"\n",
"version: '0.1.0'\n",
"\n",
"index:\n",
" name: wiki\n",
" prefix: oaiWiki\n",
" name: wikipedia\n",
" prefix: chunk\n",
"\n",
"fields:\n",
" text:\n",
" - name: content\n",
" - name: title\n",
" tag:\n",
" - name: id\n",
" vector:\n",
" - name: embedding\n",
" - name: content\n",
" type: text\n",
" - name: title\n",
" type: text\n",
" - name: id\n",
" type: tag\n",
" - name: embedding\n",
" type: vector\n",
" attrs:\n",
" dims: 1536\n",
" distance_metric: cosine\n",
" algorithm: flat"
Expand All @@ -690,10 +685,14 @@
"metadata": {},
"outputs": [],
"source": [
"import redis.asyncio as redis\n",
"\n",
"from redisvl.index import SearchIndex\n",
"\n",
"client = redis.Redis.from_url(\"redis://localhost:6379\")\n",
"\n",
"index = SearchIndex.from_yaml(\"wiki_schema.yaml\")\n",
"index.connect(\"redis://localhost:6379\", use_async=True)\n",
"index.set_client(client)\n",
"\n",
"await index.acreate()"
]
Expand Down
64 changes: 42 additions & 22 deletions docs/user_guide/cli.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:13:52\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m RedisVL version 0.0.5\n"
"\u001b[32m20:09:02\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m RedisVL version 0.0.7\n"
]
}
],
Expand All @@ -44,19 +44,22 @@
"first, we will create an index from a yaml schema that looks like the following\n",
"\n",
"```yaml\n",
"version: '0.1.0'\n",
"\n",
"index:\n",
" name: providers\n",
" prefix: rvl\n",
" name: vectorizers\n",
" prefix: doc\n",
" storage_type: hash\n",
"\n",
"fields:\n",
" text:\n",
" - name: sentence\n",
" vector:\n",
" - name: embedding\n",
" dims: 768\n",
" algorithm: flat\n",
" distance_metric: cosine\n",
" - name: sentence\n",
" type: text\n",
" - name: embedding\n",
" type: vector\n",
" attrs:\n",
" dims: 768\n",
" algorithm: flat\n",
" distance_metric: cosine\n",
"```"
]
},
Expand All @@ -69,7 +72,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:13:54\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index created successfully\n"
"\u001b[32m20:09:04\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index created successfully\n"
]
}
],
Expand All @@ -87,8 +90,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:13:56\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
"\u001b[32m11:13:56\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. providers\n"
"\u001b[32m20:09:05\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
"\u001b[32m20:09:05\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. vectorizers\n"
]
}
],
Expand All @@ -112,7 +115,7 @@
"╭──────────────┬────────────────┬────────────┬─────────────────┬────────────╮\n",
"│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n",
"├──────────────┼────────────────┼────────────┼─────────────────┼────────────┤\n",
"│ providers │ HASH │ ['rvl'] │ [] │ 0 │\n",
"│ vectorizers │ HASH │ ['doc'] │ [] │ 0 │\n",
"╰──────────────┴────────────────┴────────────┴─────────────────┴────────────╯\n",
"Index Fields:\n",
"╭───────────┬─────────────┬────────┬────────────────┬────────────────╮\n",
Expand All @@ -126,7 +129,7 @@
],
"source": [
"# inspect the index fields\n",
"!rvl index info -i providers"
"!rvl index info -i vectorizers"
]
},
{
Expand All @@ -138,13 +141,13 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:13:59\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index deleted successfully\n"
"\u001b[32m20:09:09\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index deleted successfully\n"
]
}
],
"source": [
"# delete an index without deleting the data within it\n",
"!rvl index delete -i providers"
"!rvl index delete -i vectorizers"
]
},
{
Expand All @@ -156,7 +159,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:14:00\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n"
"\u001b[32m20:09:11\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n"
]
}
],
Expand All @@ -183,7 +186,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:14:02\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index created successfully\n"
"\u001b[32m20:09:12\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index created successfully\n"
]
}
],
Expand All @@ -201,8 +204,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:14:03\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
"\u001b[32m11:14:03\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. providers\n"
"\u001b[32m20:09:14\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
"\u001b[32m20:09:14\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. vectorizers\n"
]
}
],
Expand Down Expand Up @@ -250,7 +253,24 @@
],
"source": [
"# see all the stats for the index\n",
"!rvl stats -i providers"
"!rvl stats -i vectorizers"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m20:09:33\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index deleted successfully\n"
]
}
],
"source": [
"!rvl index destroy -i vectorizers"
]
}
],
Expand Down
Loading