From 9bacacf52ebaeaa38476244601607cb959a33c7b Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 26 Jul 2023 11:38:05 +0300 Subject: [PATCH 1/4] Update README.md --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e47f3afd1ce..032f1935001 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,11 @@ For reference and tutorials, see the [Firebolt Python SDK reference](https://pyt ## Connection parameters These parameters are used to connect to a Firebolt database: -- **engine_url** - url for an engine to make requests to. Can be retrieved from Web UI, or from [engine](https://github.com/firebolt-db/firebolt-sdk/tree/main/src/firebolt/model/engine.py) attribute `endpoint` -- **database** - name of the database to make queries to -- **username** - account username -- **password** - account password - -Optional parameters -- **api_endpoint** - api hostname for logging in. Defaults to `api.app.firebolt.io`. +- **account_name** - name of firebolt account +- **client_id** - credentials client id +- **cliend_secret** - credentials client secret +- **database [Optional]** - name of the database to connect to +- **engine_name [Optional]** - name of the engine to connect to ## Examples See [PEP-249](https://www.python.org/dev/peps/pep-0249) for the DB API reference and specifications. An example [jupyter notebook](https://github.com/firebolt-db/firebolt-sdk/tree/main/examples/dbapi.ipynb) is included to illustrate the use of the Firebolt API. From 3e94f8599b29c97935f17be5da3e77462826d247 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 26 Jul 2023 13:55:44 +0300 Subject: [PATCH 2/4] update jupyter examples --- examples/dbapi.ipynb | 121 +++++++++++++---------------- examples/management.ipynb | 157 +++++++++++++------------------------- 2 files changed, 110 insertions(+), 168 deletions(-) diff --git a/examples/dbapi.ipynb b/examples/dbapi.ipynb index ee34671aa8a..21dbf89824c 100644 --- a/examples/dbapi.ipynb +++ b/examples/dbapi.ipynb @@ -15,8 +15,9 @@ "metadata": {}, "outputs": [], "source": [ - "from firebolt.db import connect\n", + "from firebolt.db import connect, OperationalError\n", "from firebolt.client import DEFAULT_API_URL\n", + "from firebolt.client.auth import ClientCredentials\n", "from datetime import datetime" ] }, @@ -36,15 +37,12 @@ "outputs": [], "source": [ "# Only one of these two parameters should be specified\n", - "engine_url = \"\"\n", - "assert bool(engine_url) != bool(\n", - " engine_name\n", - "), \"Specify only one of engine_name and engine_url\"\n", - "\n", - "database_name = \"\"\n", - "username = \"\"\n", - "password = \"\"\n", - "api_endpoint = DEFAULT_API_URL" + "client_id = \"\"\n", + "client_secret = \"\"\n", + "account_name = \"\"\n", + "engine_name = \"\" # Optional\n", + "database_name = \"\" # Optional\n", + "api_endpoint = DEFAULT_API_URL # Optional" ] }, { @@ -64,11 +62,10 @@ "source": [ "# create a connection based on provided credentials\n", "connection = connect(\n", - " engine_url=engine_url,\n", + " auth=ClientCredentials(client_id, client_secret),\n", + " account_name=account_name,\n", " engine_name=engine_name,\n", " database=database_name,\n", - " username=username,\n", - " password=password,\n", " api_endpoint=api_endpoint,\n", ")\n", "\n", @@ -218,7 +215,7 @@ " select * from test_table\n", " \"\"\"\n", " )\n", - "except:\n", + "except OperationalError:\n", " pass\n", "cursor.fetchall()" ] @@ -228,7 +225,8 @@ "id": "b1cd4ff2", "metadata": {}, "source": [ - "## Async interface" + "## Async interface\n", + "**NOTE**: In order to make async examples work in jupyter, you would need to install [trio-jupyter](https://github.com/mehaase/trio-jupyter) library and select **Python 3 Trio** kernel" ] }, { @@ -238,7 +236,7 @@ "metadata": {}, "outputs": [], "source": [ - "from firebolt.async_db import connect" + "from firebolt.async_db import connect as async_connect" ] }, { @@ -256,30 +254,17 @@ "metadata": {}, "outputs": [], "source": [ - "async def async_connect():\n", - " # create a connection based on provided credentials\n", - " connection = await connect(\n", - " engine_url=engine_url,\n", - " engine_name=engine_name,\n", - " database=database_name,\n", - " username=username,\n", - " password=password,\n", - " api_endpoint=api_endpoint,\n", - " )\n", + "# create a connection based on provided credentials\n", + "async_connection = await async_connect(\n", + " auth=ClientCredentials(client_id, client_secret),\n", + " account_name=account_name,\n", + " engine_name=engine_name,\n", + " database=database_name,\n", + " api_endpoint=api_endpoint,\n", + ")\n", "\n", - " # create a cursor for connection\n", - " cursor = connection.cursor()\n", - " return cursor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "58885c37", - "metadata": {}, - "outputs": [], - "source": [ - "async_cursor = await async_connect()" + "# create a cursor for connection\n", + "async_cursor = async_connection.cursor()" ] }, { @@ -297,26 +282,15 @@ "metadata": {}, "outputs": [], "source": [ - "async def run_queries(cursor):\n", - " await cursor.execute(\n", - " \"create fact table if not exists test_table (id int, name text, dt datetime) primary index id\"\n", - " )\n", - " await cursor.execute(\n", - " \"insert into test_table values (1, 'hello', '2021-01-01 01:01:01'),\"\n", - " \"(2, 'world', '2022-02-02 02:02:02'),\"\n", - " \"(3, '!', '2023-03-03 03:03:03')\"\n", - " )\n", - " await cursor.execute(\"select * from test_table\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "768fc91a", - "metadata": {}, - "outputs": [], - "source": [ - "await run_queries(async_cursor)" + "await async_cursor.execute(\n", + " \"create fact table if not exists test_table (id int, name text, dt datetime) primary index id\"\n", + ")\n", + "await async_cursor.execute(\n", + " \"insert into test_table values (1, 'hello', '2021-01-01 01:01:01'),\"\n", + " \"(2, 'world', '2022-02-02 02:02:02'),\"\n", + " \"(3, '!', '2023-03-03 03:03:03')\"\n", + ")\n", + "await async_cursor.execute(\"select * from test_table\")" ] }, { @@ -386,27 +360,42 @@ "source": [ "# manually\n", "connection.close()\n", + "print(connection.closed)\n", + "\n", + "await async_connection.aclose()\n", + "print(async_connection.closed)\n", "\n", "# using context manager\n", "with connect(\n", - " engine_url=engine_url,\n", + " auth=ClientCredentials(client_id, client_secret),\n", + " account_name=account_name,\n", " engine_name=engine_name,\n", " database=database_name,\n", - " username=username,\n", - " password=password,\n", " api_endpoint=api_endpoint,\n", ") as conn:\n", " # create cursors, perform database queries\n", " pass\n", - "conn.closed" + "print(conn.closed)\n", + "\n", + "# using context manager\n", + "async with await async_connect(\n", + " auth=ClientCredentials(client_id, client_secret),\n", + " account_name=account_name,\n", + " engine_name=engine_name,\n", + " database=database_name,\n", + " api_endpoint=api_endpoint,\n", + ") as async_conn:\n", + " # create cursors, perform database queries\n", + " pass\n", + "async_conn.closed" ] } ], "metadata": { "kernelspec": { - "display_name": "python37", + "display_name": "Python 3 Trio", "language": "python", - "name": "python37" + "name": "python3-trio" }, "language_info": { "codemirror_mode": { @@ -418,7 +407,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/examples/management.ipynb b/examples/management.ipynb index 6f51bb99bd3..9af308e5b44 100644 --- a/examples/management.ipynb +++ b/examples/management.ipynb @@ -17,24 +17,13 @@ "source": [ "# required imports\n", "from firebolt.service.manager import ResourceManager\n", + "from firebolt.client import DEFAULT_API_URL\n", + "from firebolt.client.auth import ClientCredentials\n", "\n", "# use debug function for pretty printing\n", "from devtools import debug" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "93073ef6", - "metadata": {}, - "outputs": [], - "source": [ - "# support async_to_sync in notebooks\n", - "import nest_asyncio\n", - "\n", - "nest_asyncio.apply()" - ] - }, { "cell_type": "markdown", "id": "443e8468", @@ -65,73 +54,28 @@ "# logging.getLogger(\"firebolt.model\").setLevel(\"WARNING\")\n", "\n", "# uncomment to suppress info messages from running queries\n", - "# logging.getLogger(\"firebolt.async_db\").setLevel(\"WARNING\")" + "# logging.getLogger(\"firebolt.db\").setLevel(\"WARNING\")" ] }, { "cell_type": "markdown", - "id": "bc5376bc", - "metadata": {}, - "source": [ - "# Configure ResourceManager" - ] - }, - { - "cell_type": "markdown", - "id": "4efaddf9", - "metadata": {}, - "source": [ - "**Option 1**: create a `.env` file with the following contents (fill in values):\n", - "```\n", - "FIREBOLT_USER=''\n", - "FIREBOLT_PASSWORD=''\n", - "FIREBOLT_SERVER=''\n", - "FIREBOLT_DEFAULT_REGION=''\n", - "```\n", - "(or ensure these env vars are already set)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bf3a14e2", - "metadata": {}, - "outputs": [], - "source": [ - "settings = None" - ] - }, - { - "cell_type": "markdown", - "id": "f1626c0e", + "id": "97388a98", "metadata": {}, "source": [ - "**Option 2**: Specify settings manually using the Settings object:" + "# Initialize ResourceManager" ] }, { "cell_type": "code", "execution_count": null, - "id": "e2c17737", + "id": "204d3ea8-84f3-4825-8940-e51405d3f451", "metadata": {}, "outputs": [], "source": [ - "from firebolt.common import Settings\n", - "\n", - "settings = Settings(\n", - " server=\"\", # api.app.firebolt.io\n", - " user=\"\",\n", - " password=\"\",\n", - " default_region=\"\", # us-east-1\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "97388a98", - "metadata": {}, - "source": [ - "# Initialize ResourceManager" + "client_id = \"\"\n", + "client_secret = \"\"\n", + "account_name = \"\"\n", + "api_endpoint = DEFAULT_API_URL # Optional" ] }, { @@ -143,13 +87,11 @@ }, "outputs": [], "source": [ - "rm = ResourceManager(settings=settings)\n", - "\n", - "# see your account_id\n", - "debug(rm.account_id)\n", - "\n", - "# see your settings\n", - "debug(rm.settings)" + "rm = ResourceManager(\n", + " auth=ClientCredentials(client_id, client_secret),\n", + " account_name=account_name,\n", + " api_endpoint=api_endpoint,\n", + ")" ] }, { @@ -170,8 +112,8 @@ "import time\n", "\n", "default_name = f\"temp_{int(time.time())}\"\n", - "database_name = \"\" or default_name\n", - "engine_name = \"\" or default_name" + "database_name = default_name # Change to use you own name for creating a new database\n", + "engine_name = default_name # Change to use you own name for creating a new engine" ] }, { @@ -216,7 +158,7 @@ "metadata": {}, "outputs": [], "source": [ - "database = rm.databases.get_by_name(name=database_name)\n", + "database = rm.databases.get(database_name)\n", "debug(database)" ] }, @@ -260,9 +202,7 @@ "cell_type": "code", "execution_count": null, "id": "9dad9f23", - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "engine = rm.engines.create(name=engine_name)\n", @@ -284,7 +224,7 @@ "metadata": {}, "outputs": [], "source": [ - "engine = rm.engines.get_by_name(name=engine_name)\n", + "engine = rm.engines.get(engine_name)\n", "debug(engine)" ] }, @@ -303,7 +243,11 @@ "metadata": {}, "outputs": [], "source": [ - "engine.attach_to_database(database=rm.databases.get_by_name(name=database_name))" + "# Attaching an already attached engine will raise an OperationalError\n", + "if not engine.database:\n", + " engine.attach_to_database(database=rm.databases.get(database_name))\n", + "\n", + "debug(engine.database)" ] }, { @@ -340,12 +284,10 @@ "metadata": {}, "outputs": [], "source": [ - "connection = engine.get_connection()\n", - "cursor = connection.cursor()\n", - "result = cursor.execute(\"select 1\")\n", - "debug(result)\n", - "\n", - "cursor.close()" + "with engine.get_connection() as connection:\n", + " with connection.cursor() as cursor:\n", + " result = cursor.execute(\"select 1\")\n", + " debug(result)" ] }, { @@ -360,9 +302,7 @@ "cell_type": "code", "execution_count": null, "id": "a069e9eb", - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "stopped_engine = engine.stop()\n", @@ -392,7 +332,7 @@ "id": "7639a535", "metadata": {}, "source": [ - "# Region, Instance Types" + "# Instance Types" ] }, { @@ -400,24 +340,37 @@ "id": "ddf4098d", "metadata": {}, "source": [ - "### Get default region, and an instance_type\n", - "An example of these might be:\n", - " * provider: AWS\n", - " * region: us-east-1\n", - " * instance_type: i3.4xlarge" + "### Get an instance type by name" ] }, { "cell_type": "code", "execution_count": null, "id": "8372e49f", - "metadata": { - "scrolled": false - }, + "metadata": {}, + "outputs": [], + "source": [ + "instance_type = rm.instance_types.get(\"B2\")\n", + "debug(instance_type)" + ] + }, + { + "cell_type": "markdown", + "id": "2d213b1a-bf08-424b-ba87-51866c8b65f4", + "metadata": {}, + "source": [ + "### Get cheapest available instance type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53ce76b8-62a9-4eaa-a730-6dd268aab588", + "metadata": {}, "outputs": [], "source": [ - "debug(rm.regions.default_region)\n", - "debug(rm.instance_types.get_by_name(instance_type_name=\"i3.4xlarge\"))" + "cheapest_instance_type = rm.instance_types.cheapest_instance\n", + "debug(cheapest_instance_type)" ] } ], @@ -437,7 +390,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.16" } }, "nbformat": 4, From 9118b72bd2b0e93efb57eaf46f9f36085b7a60b6 Mon Sep 17 00:00:00 2001 From: Stepan Burlakov Date: Wed, 26 Jul 2023 13:57:58 +0300 Subject: [PATCH 3/4] minor examples improvement --- examples/dbapi.ipynb | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/examples/dbapi.ipynb b/examples/dbapi.ipynb index 21dbf89824c..73b168bc7a6 100644 --- a/examples/dbapi.ipynb +++ b/examples/dbapi.ipynb @@ -327,20 +327,9 @@ "metadata": {}, "outputs": [], "source": [ - "async def print_results(cursor):\n", - " print(await cursor.fetchone())\n", - " print(await cursor.fetchmany(1))\n", - " print(await cursor.fetchall())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15f24a0a", - "metadata": {}, - "outputs": [], - "source": [ - "await print_results(async_cursor)" + "print(await cursor.fetchone())\n", + "print(await cursor.fetchmany(1))\n", + "print(await cursor.fetchall())" ] }, { From cefc9b77a5bfa50ad56a07bfc7a03628e6cfa7de Mon Sep 17 00:00:00 2001 From: Petro Tiurin <93913847+ptiurin@users.noreply.github.com> Date: Mon, 31 Jul 2023 17:17:38 +0100 Subject: [PATCH 4/4] Remove old comment --- examples/dbapi.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/dbapi.ipynb b/examples/dbapi.ipynb index 73b168bc7a6..0fb009903b9 100644 --- a/examples/dbapi.ipynb +++ b/examples/dbapi.ipynb @@ -36,7 +36,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Only one of these two parameters should be specified\n", "client_id = \"\"\n", "client_secret = \"\"\n", "account_name = \"\"\n",