Skip to content

Commit

Permalink
sql storage engine fetch all
Browse files Browse the repository at this point in the history
  • Loading branch information
@yulaicui
yulaicui committed Jul 14, 2023
1 parent c173caf commit 7f75963
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 87 deletions.
1 change: 1 addition & 0 deletions evadb/storage/image_storage_engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def __init__(self, db: EvaDBDatabase):
def read(self, table: TableCatalogEntry) -> Iterator[Batch]:
for image_files in self._rdb_handler.read(self._get_metadata_table(table)):
for _, (row_id, file_name) in image_files.iterrows():
self._rdb_handler._sql_session.close()
system_file_name = self._xform_file_url_to_file_name(file_name)
image_file = Path(table.file_url) / system_file_name
# setting batch_mem_size = 1, we need fix it
Expand Down
2 changes: 1 addition & 1 deletion evadb/storage/sqlite_storage_engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def read(
"""
try:
table_to_read = self._try_loading_table_via_reflection(table.name)
result = self._sql_session.execute(table_to_read.select())
result = self._sql_session.execute(table_to_read.select()).fetchall()
data_batch = []
row_size = None
for row in result:
Expand Down
139 changes: 53 additions & 86 deletions tutorials/11-similarity-search-for-motif-mining.ipynb
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"id": "3a4df227",
"metadata": {},
Expand All @@ -10,7 +9,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "5d29b79c",
"metadata": {},
Expand All @@ -23,7 +21,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "e6df3057",
"metadata": {},
Expand All @@ -42,7 +39,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "50b1674e",
"metadata": {},
Expand All @@ -67,13 +63,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Note: you may need to restart the kernel to use updated packages.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
"grpcio-tools 1.56.0 requires protobuf<5.0dev,>=4.21.6, but you have protobuf 3.20.1 which is incompatible.\u001b[0m\u001b[31m\n",
"\u001b[0mNote: you may need to restart the kernel to use updated packages.\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
Expand All @@ -86,7 +78,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "cc9f4144",
"metadata": {},
Expand All @@ -111,50 +102,44 @@
"name": "stdout",
"output_type": "stream",
"text": [
"File ‘reddit-images.zip’ already there; not retrieving.\r\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Archive: reddit-images.zip\r\n",
"warning: stripped absolute path spec from /\r\n",
"mapname: conversion of failed\r\n",
" extracting: reddit-images/g348_d7jgzgf.jpg \r\n",
" extracting: reddit-images/g348_d7jphyc.jpg \r\n",
" extracting: reddit-images/g348_d7ju7dq.jpg \r\n",
" extracting: reddit-images/g348_d7jhhs3.jpg \r\n",
" extracting: reddit-images/g1074_d4n1lmn.jpg \r\n",
" extracting: reddit-images/g1074_d4mxztt.jpg \r\n",
" extracting: reddit-images/g1074_d4n60oy.jpg \r\n",
" extracting: reddit-images/g1074_d4n6fgs.jpg \r\n",
" extracting: reddit-images/g1190_cln9xzr.jpg \r\n",
" extracting: reddit-images/g1190_cln97xm.jpg \r\n",
" extracting: reddit-images/g1190_clna260.jpg \r\n",
" extracting: reddit-images/g1190_clna2x2.jpg \r\n",
" extracting: reddit-images/g1190_clna91w.jpg \r\n",
" extracting: reddit-images/g1190_clnad42.jpg \r\n",
" extracting: reddit-images/g1190_clnajd7.jpg \r\n",
" extracting: reddit-images/g1190_clnapoy.jpg \r\n",
" extracting: reddit-images/g1190_clnarjl.jpg \r\n",
" extracting: reddit-images/g1190_clnavnu.jpg \r\n",
" extracting: reddit-images/g1190_clnbalu.jpg \r\n",
" extracting: reddit-images/g1190_clnbf07.jpg \r\n",
" extracting: reddit-images/g1190_clnc4uy.jpg \r\n",
" extracting: reddit-images/g1190_clncot0.jpg \r\n",
" extracting: reddit-images/g1190_clndsnu.jpg \r\n",
" extracting: reddit-images/g1190_clnce4b.jpg \r\n",
" extracting: reddit-images/g1209_ct65pvl.jpg \r\n",
" extracting: reddit-images/g1209_ct66erw.jpg \r\n",
" extracting: reddit-images/g1209_ct67oqk.jpg \r\n",
" extracting: reddit-images/g1209_ct6a0g5.jpg \r\n",
" extracting: reddit-images/g1209_ct6bf1n.jpg \r\n",
" extracting: reddit-images/g1418_cj3o1h6.jpg \r\n",
" extracting: reddit-images/g1418_cj3om3h.jpg \r\n",
" extracting: reddit-images/g1418_cj3qysz.jpg \r\n",
" extracting: reddit-images/g1418_cj3r4gw.jpg \r\n",
" extracting: reddit-images/g1418_cj3z7jw.jpg \r\n"
"File ‘reddit-images.zip’ already there; not retrieving.\n",
"Archive: reddit-images.zip\n",
"warning: stripped absolute path spec from /\n",
"mapname: conversion of failed\n",
" extracting: reddit-images/g348_d7jgzgf.jpg \n",
" extracting: reddit-images/g348_d7jphyc.jpg \n",
" extracting: reddit-images/g348_d7ju7dq.jpg \n",
" extracting: reddit-images/g348_d7jhhs3.jpg \n",
" extracting: reddit-images/g1074_d4n1lmn.jpg \n",
" extracting: reddit-images/g1074_d4mxztt.jpg \n",
" extracting: reddit-images/g1074_d4n60oy.jpg \n",
" extracting: reddit-images/g1074_d4n6fgs.jpg \n",
" extracting: reddit-images/g1190_cln9xzr.jpg \n",
" extracting: reddit-images/g1190_cln97xm.jpg \n",
" extracting: reddit-images/g1190_clna260.jpg \n",
" extracting: reddit-images/g1190_clna2x2.jpg \n",
" extracting: reddit-images/g1190_clna91w.jpg \n",
" extracting: reddit-images/g1190_clnad42.jpg \n",
" extracting: reddit-images/g1190_clnajd7.jpg \n",
" extracting: reddit-images/g1190_clnapoy.jpg \n",
" extracting: reddit-images/g1190_clnarjl.jpg \n",
" extracting: reddit-images/g1190_clnavnu.jpg \n",
" extracting: reddit-images/g1190_clnbalu.jpg \n",
" extracting: reddit-images/g1190_clnbf07.jpg \n",
" extracting: reddit-images/g1190_clnc4uy.jpg \n",
" extracting: reddit-images/g1190_clncot0.jpg \n",
" extracting: reddit-images/g1190_clndsnu.jpg \n",
" extracting: reddit-images/g1190_clnce4b.jpg \n",
" extracting: reddit-images/g1209_ct65pvl.jpg \n",
" extracting: reddit-images/g1209_ct66erw.jpg \n",
" extracting: reddit-images/g1209_ct67oqk.jpg \n",
" extracting: reddit-images/g1209_ct6a0g5.jpg \n",
" extracting: reddit-images/g1209_ct6bf1n.jpg \n",
" extracting: reddit-images/g1418_cj3o1h6.jpg \n",
" extracting: reddit-images/g1418_cj3om3h.jpg \n",
" extracting: reddit-images/g1418_cj3qysz.jpg \n",
" extracting: reddit-images/g1418_cj3r4gw.jpg \n",
" extracting: reddit-images/g1418_cj3z7jw.jpg \n"
]
}
],
Expand All @@ -165,7 +150,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "0d8c9917",
"metadata": {},
Expand Down Expand Up @@ -237,7 +221,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "6743684c",
"metadata": {},
Expand Down Expand Up @@ -328,7 +311,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "cc5a1e73",
"metadata": {
Expand Down Expand Up @@ -361,7 +343,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"07-11-2023 03:28:21 WARNING[drop_object_executor:drop_object_executor.py:_handle_drop_index:0106] Index reddit_sift_image_index does not exist, therefore cannot be dropped.\n"
"07-13-2023 23:09:10 WARNING[drop_object_executor:drop_object_executor.py:_handle_drop_index:0106] Index reddit_sift_image_index does not exist, therefore cannot be dropped.\n"
]
},
{
Expand Down Expand Up @@ -467,7 +449,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "a19a5d51",
"metadata": {
Expand All @@ -484,7 +465,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "bce1b7fe",
"metadata": {},
Expand Down Expand Up @@ -554,7 +534,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "b356c435",
"metadata": {},
Expand Down Expand Up @@ -642,13 +621,7 @@
"output_type": "stream",
"text": [
" 0\n",
"0 Number of loaded IMAGE: 1\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 Number of loaded IMAGE: 1\n",
" yolo.bboxes\n",
"0 [[257.2467956542969, 256.8749084472656, 457.67...\n"
]
Expand Down Expand Up @@ -686,7 +659,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "40e563b2",
"metadata": {},
Expand All @@ -712,21 +684,9 @@
"output_type": "stream",
"text": [
" reddit_object_table.name\n",
"0 reddit-images/g1190_cln9xzr.jpg\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 reddit-images/g1190_cln9xzr.jpg\n",
" reddit_object_table.name\n",
"0 reddit-images/g1190_cln9xzr.jpg\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 reddit-images/g1190_cln9xzr.jpg\n",
" reddit_object_table.name\n",
"0 reddit-images/g348_d7jgzgf.jpg\n"
]
Expand All @@ -752,7 +712,6 @@
]
},
{
"attachments": {},
"cell_type": "markdown",
"id": "096d4970",
"metadata": {},
Expand Down Expand Up @@ -804,6 +763,14 @@
"\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f4adfa37",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down

0 comments on commit 7f75963

Please sign in to comment.