Feat : Sqlite Database Handler (#1023)

This diff adds extends the database interface to add support for SQLite.

Test Plan : UT

---------

Co-authored-by: Ankith Reddy Chitti <chitti@lawn-128-61-2-2.lawn.gatech.edu>
Co-authored-by: Ankith Reddy Chitti <chitti@Ankiths-Air.attlocal.net>

docs/_toc.yml

@@ -58,7 +58,8 @@ parts:
       - file: source/reference/databases/index
         title: Data Sources
         sections: 
-          - file: source/reference/databases/postgres 
+          - file: source/reference/databases/postgres
+          - file: source/reference/databases/sqlite
           - file: source/reference/databases/mysql
 
       - file: source/reference/udfs/index

docs/source/reference/databases/sqlite.rst

@@ -0,0 +1,27 @@
+SQLite
+==========
+
+The connection to SQLite is based on the `sqlite3 <https://docs.python.org/3/library/sqlite3.html>`_ library.
+
+Dependency
+----------
+
+* sqlite3
+
+
+Parameters
+----------
+
+Required:
+
+* `database` is the path to the database file to be opened. You can pass ":memory:" to create an SQLite database existing only in memory, and open a connection to it.
+
+
+Create Connection
+-----------------
+
+.. code-block:: text
+
+   CREATE DATABASE sqlite_data WITH ENGINE = 'sqlite', PARAMETERS = {
+        "database": "evadb.db"
+   };

evadb/third_party/databases/interface.py

@@ -33,6 +33,8 @@ def get_database_handler(engine: str, **kwargs):
 
     if engine == "postgres":
         return mod.PostgresHandler(engine, **kwargs)
+    elif engine == "sqlite":
+        return mod.SQLiteHandler(engine, **kwargs)
     elif engine == "mysql":
         return mod.MysqlHandler(engine, **kwargs)
     else:

evadb/third_party/databases/postgres/postgres_handler.py

@@ -24,6 +24,12 @@
 
 class PostgresHandler(DBHandler):
     def __init__(self, name: str, **kwargs):
+        """
+        Initialize the handler.
+        Args:
+            name (str): name of the DB handler instance
+            **kwargs: arbitrary keyword arguments for establishing the connection.
+        """
         super().__init__(name)
         self.host = kwargs.get("host")
         self.port = kwargs.get("port")
@@ -32,7 +38,12 @@ def __init__(self, name: str, **kwargs):
         self.database = kwargs.get("database")
         self.connection = None
 
-    def connect(self):
+    def connect(self) -> DBHandlerStatus:
+        """
+        Set up the connection required by the handler.
+        Returns:
+            DBHandlerStatus
+        """
         try:
             self.connection = psycopg2.connect(
                 host=self.host,
@@ -47,16 +58,29 @@ def connect(self):
             return DBHandlerStatus(status=False, error=str(e))
 
     def disconnect(self):
+        """
+        Close any existing connections.
+        """
         if self.connection:
             self.connection.close()
 
     def check_connection(self) -> DBHandlerStatus:
+        """
+        Check connection to the handler.
+        Returns:
+            DBHandlerStatus
+        """
         if self.connection:
             return DBHandlerStatus(status=True)
         else:
             return DBHandlerStatus(status=False, error="Not connected to the database.")
 
     def get_tables(self) -> DBHandlerResponse:
+        """
+        Return the list of tables in the database.
+        Returns:
+            DBHandlerResponse
+        """
         if not self.connection:
             return DBHandlerResponse(data=None, error="Not connected to the database.")
 
@@ -68,6 +92,13 @@ def get_tables(self) -> DBHandlerResponse:
             return DBHandlerResponse(data=None, error=str(e))
 
     def get_columns(self, table_name: str) -> DBHandlerResponse:
+        """
+        Returns the list of columns for the given table.
+        Args:
+            table_name (str): name of the table whose columns are to be retrieved.
+        Returns:
+            DBHandlerResponse
+        """
         if not self.connection:
             return DBHandlerResponse(data=None, error="Not connected to the database.")
 
@@ -101,6 +132,13 @@ def _fetch_results_as_df(self, cursor):
             raise e
 
     def execute_native_query(self, query_string: str) -> DBHandlerResponse:
+        """
+        Executes the native query on the database.
+        Args:
+            query_string (str): query in native format
+        Returns:
+            DBHandlerResponse
+        """
         if not self.connection:
             return DBHandlerResponse(data=None, error="Not connected to the database.")
 

evadb/third_party/databases/sqlite/__init__.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+# Copyright 2018-2023 EvaDB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""sqlite integration"""

evadb/third_party/databases/sqlite/sqlite_handler.py

@@ -0,0 +1,139 @@
+# coding=utf-8
+# Copyright 2018-2023 EvaDB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sqlite3
+
+import pandas as pd
+
+from evadb.third_party.databases.types import (
+    DBHandler,
+    DBHandlerResponse,
+    DBHandlerStatus,
+)
+
+
+class SQLiteHandler(DBHandler):
+    def __init__(self, name: str, **kwargs):
+        """
+        Initialize the handler.
+        Args:
+            name (str): name of the DB handler instance
+            **kwargs: arbitrary keyword arguments for establishing the connection.
+        """
+        super().__init__(name)
+        self.database = kwargs.get("database")
+        self.connection = None
+
+    def connect(self):
+        """
+        Set up the connection required by the handler.
+        Returns:
+            DBHandlerStatus
+        """
+        try:
+            self.connection = sqlite3.connect(
+                database=self.database, isolation_level=None  # Autocommit mode.
+            )
+            return DBHandlerStatus(status=True)
+        except sqlite3.Error as e:
+            return DBHandlerStatus(status=False, error=str(e))
+
+    def disconnect(self):
+        """
+        Close any existing connections.
+        """
+        if self.connection:
+            self.connection.close()
+
+    def check_connection(self) -> DBHandlerStatus:
+        """
+        Check connection to the handler.
+        Returns:
+            DBHandlerStatus
+        """
+        if self.connection:
+            return DBHandlerStatus(status=True)
+        else:
+            return DBHandlerStatus(status=False, error="Not connected to the database.")
+
+    def get_tables(self) -> DBHandlerResponse:
+        """
+        Return the list of tables in the database.
+        Returns:
+            DBHandlerResponse
+        """
+        if not self.connection:
+            return DBHandlerResponse(data=None, error="Not connected to the database.")
+
+        try:
+            query = "SELECT name AS table_name FROM sqlite_master WHERE type = 'table'"
+            tables_df = pd.read_sql_query(query, self.connection)
+            return DBHandlerResponse(data=tables_df)
+        except sqlite3.Error as e:
+            return DBHandlerResponse(data=None, error=str(e))
+
+    def get_columns(self, table_name: str) -> DBHandlerResponse:
+        """
+        Returns the list of columns for the given table.
+        Args:
+            table_name (str): name of the table whose columns are to be retrieved.
+        Returns:
+            DBHandlerResponse
+        """
+        if not self.connection:
+            return DBHandlerResponse(data=None, error="Not connected to the database.")
+        """
+        SQLite does not provide an in-built way to get the column names using a SELECT statement.
+        Hence we have to use the PRAGMA command and filter the required columns.
+        """
+        try:
+            query = f"PRAGMA table_info('{table_name}')"
+            pragma_df = pd.read_sql_query(query, self.connection)
+            columns_df = pragma_df[["name", "type"]].copy()
+            columns_df.rename(columns={"type": "dtype"}, inplace=True)
+            return DBHandlerResponse(data=columns_df)
+        except sqlite3.Error as e:
+            return DBHandlerResponse(data=None, error=str(e))
+
+    def _fetch_results_as_df(self, cursor):
+        try:
+            res = cursor.fetchall()
+            res_df = pd.DataFrame(
+                res,
+                columns=[desc[0] for desc in cursor.description]
+                if cursor.description
+                else [],
+            )
+            return res_df
+        except sqlite3.ProgrammingError as e:
+            if str(e) == "no results to fetch":
+                return pd.DataFrame({"status": ["success"]})
+            raise e
+
+    def execute_native_query(self, query_string: str) -> DBHandlerResponse:
+        """
+        Executes the native query on the database.
+        Args:
+            query_string (str): query in native format
+        Returns:
+            DBHandlerResponse
+        """
+        if not self.connection:
+            return DBHandlerResponse(data=None, error="Not connected to the database.")
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(query_string)
+            return DBHandlerResponse(data=self._fetch_results_as_df(cursor))
+        except sqlite3.Error as e:
+            return DBHandlerResponse(data=None, error=str(e))

test/third_party_tests/test_native_executor.py

@@ -112,6 +112,7 @@ def _execute_native_query(self):
             }""",
         )
         self.assertEqual(len(res_batch), 1)
+
         self.assertEqual(res_batch.frames["name"][0], "aa")
         self.assertEqual(res_batch.frames["age"][0], 1)
         self.assertEqual(res_batch.frames["comment"][0], "aaaa")
@@ -168,6 +169,20 @@ def test_should_run_query_in_postgres(self):
         self._raise_error_on_multiple_creation()
         self._raise_error_on_invalid_connection()
 
+    def test_should_run_query_in_sqlite(self):
+        # Create database.
+        params = {
+            "database": "evadb.db",
+        }
+        query = f"""CREATE DATABASE test_data_source
+                    WITH ENGINE = "sqlite",
+                    PARAMETERS = {params};"""
+        execute_query_fetch_all(self.evadb, query)
+
+        # Test executions.
+        self._execute_native_query()
+        self._execute_evadb_query()
+
     def test_should_run_query_in_mysql(self):
         # Create database.
         params = {