From bfa095666115b676077f507a5f0291a785e58f78 Mon Sep 17 00:00:00 2001 From: marknoack Date: Tue, 14 Dec 2021 14:05:17 -0800 Subject: [PATCH 01/12] feat: RTD build added --- .gitignore | 11 ++--- .readthedocs.yaml | 6 +++ docs/requirements.txt | 5 +++ docsrc/Makefile | 24 +++++++++++ docsrc/conf.py | 61 ++++++++++++++++++++++++++ docsrc/firebolt.async_db.rst | 31 ++++++++++++++ docsrc/firebolt.client.rst | 36 ++++++++++++++++ docsrc/firebolt.common.rst | 35 +++++++++++++++ docsrc/firebolt.db.rst | 24 +++++++++++ docsrc/firebolt.model.rst | 59 +++++++++++++++++++++++++ docsrc/firebolt.service.rst | 81 +++++++++++++++++++++++++++++++++++ docsrc/index.rst | 83 ++++++++++++++++++++++++++++++++++++ docsrc/make.bat | 42 ++++++++++++++++++ 13 files changed, 491 insertions(+), 7 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt create mode 100644 docsrc/Makefile create mode 100644 docsrc/conf.py create mode 100644 docsrc/firebolt.async_db.rst create mode 100644 docsrc/firebolt.client.rst create mode 100644 docsrc/firebolt.common.rst create mode 100644 docsrc/firebolt.db.rst create mode 100644 docsrc/firebolt.model.rst create mode 100644 docsrc/firebolt.service.rst create mode 100644 docsrc/index.rst create mode 100644 docsrc/make.bat diff --git a/.gitignore b/.gitignore index 6dabcbf09b6..5784b334ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +docsrc/_build +docs/* +!docs/requirements.txt .idea # Created by https://www.toptal.com/developers/gitignore/api/python,pycharm,macos @@ -282,10 +285,4 @@ cython_debug/ # Emacs *~ -\#*\# - -coverage-reports/* - -# VS Code -.vscode/* -*.code-workspace +\#*\# \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000000..5da9876ac21 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,6 @@ +sphinx: + configuration: docsrc/conf.py + +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000000..639c04f6d34 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +pydantic +httpx +aiorwlock +async_property +readerwriterlock \ No newline at end of file diff --git a/docsrc/Makefile b/docsrc/Makefile new file mode 100644 index 00000000000..0957461a07c --- /dev/null +++ b/docsrc/Makefile @@ -0,0 +1,24 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = ./_build + +github: + @make html + @cp -a _build/html/. ../docs + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docsrc/conf.py b/docsrc/conf.py new file mode 100644 index 00000000000..57f46c9d1cf --- /dev/null +++ b/docsrc/conf.py @@ -0,0 +1,61 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path += [ + os.path.abspath("../src"), + os.path.abspath("../src/firebolt/"), + os.path.abspath("../src/firebolt/async_db"), + os.path.abspath("../src/firebolt/client"), + os.path.abspath("../src/firebolt/common"), + os.path.abspath("../src/firebolt/db"), + os.path.abspath("../src/firebolt/model"), + os.path.abspath("../src/firebolt/service") + ] +print(sys.path) + +# -- Project information ----------------------------------------------------- + +project = "firebolt-python-sdk" +copyright = "2021, Firebolt" +author = "Firebolt" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] diff --git a/docsrc/firebolt.async_db.rst b/docsrc/firebolt.async_db.rst new file mode 100644 index 00000000000..1b3e922fbbc --- /dev/null +++ b/docsrc/firebolt.async_db.rst @@ -0,0 +1,31 @@ +========================== +firebolt.async\_db package +========================== + + + +firebolt.async\_db.connection module +------------------------------------ + +.. automodule:: firebolt.async_db.connection + :members: + :exclude-members: BaseConnection + :undoc-members: + :show-inheritance: + +firebolt.async\_db.cursor module +-------------------------------- + +.. automodule:: firebolt.async_db.cursor + :members: + :exclude-members: BaseCursor + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: firebolt.async_db + :members: + :undoc-members: + :show-inheritance: diff --git a/docsrc/firebolt.client.rst b/docsrc/firebolt.client.rst new file mode 100644 index 00000000000..ef97c946df3 --- /dev/null +++ b/docsrc/firebolt.client.rst @@ -0,0 +1,36 @@ +======================= +firebolt.client package +======================= + + +firebolt.client.auth module +--------------------------- + +.. automodule:: firebolt.client.auth + :members: + :undoc-members: + :show-inheritance: + +firebolt.client.client module +----------------------------- + +.. automodule:: firebolt.client.client + :members: + :exclude-members: FireboltClientMixin + :undoc-members: + :show-inheritance: + +.. automodule:: firebolt.client.constants + :members: + :undoc-members: + :show-inheritance: + +firebolt.client.resource\_manager\_hooks module +----------------------------------------------- + +.. automodule:: firebolt.client.resource_manager_hooks + :members: + :undoc-members: + :show-inheritance: + + diff --git a/docsrc/firebolt.common.rst b/docsrc/firebolt.common.rst new file mode 100644 index 00000000000..92a982c18e0 --- /dev/null +++ b/docsrc/firebolt.common.rst @@ -0,0 +1,35 @@ +======================= +firebolt.common package +======================= + + +firebolt.common.exception module +-------------------------------- + +.. automodule:: firebolt.common.exception + :members: + :undoc-members: + :show-inheritance: + +firebolt.common.settings module +------------------------------- + +.. automodule:: firebolt.common.settings + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: firebolt.common.urls + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: firebolt.common.util + :exclude-members: async_to_sync, cached_property, fix_url_schema, mixin_for, prune_dict + :members: + :undoc-members: + :show-inheritance: + + + diff --git a/docsrc/firebolt.db.rst b/docsrc/firebolt.db.rst new file mode 100644 index 00000000000..ae2a231417e --- /dev/null +++ b/docsrc/firebolt.db.rst @@ -0,0 +1,24 @@ +=================== +firebolt.db package +=================== + + +firebolt.db.connection module +----------------------------- + +.. automodule:: firebolt.db.connection + :members: + :undoc-members: + :show-inheritance: + +firebolt.db.cursor module +------------------------- + +.. automodule:: firebolt.db.cursor + :members: + :undoc-members: + :show-inheritance: + + + + diff --git a/docsrc/firebolt.model.rst b/docsrc/firebolt.model.rst new file mode 100644 index 00000000000..c7a41bc6366 --- /dev/null +++ b/docsrc/firebolt.model.rst @@ -0,0 +1,59 @@ +====================== +firebolt.model package +====================== + + +firebolt.model.database module +------------------------------ + +.. automodule:: firebolt.model.database + :members: + :exclude-members: DatabaseKey + :undoc-members: + :show-inheritance: + +firebolt.model.engine module +---------------------------- + +.. automodule:: firebolt.model.engine + :members: + :exclude-members: EngineKey + :undoc-members: + :show-inheritance: + +firebolt.model.engine\_revision module +-------------------------------------- + +.. automodule:: firebolt.model.engine_revision + :members: + :exclude-members: EngineRevisionKey + :undoc-members: + :show-inheritance: + +firebolt.model.instance\_type module +------------------------------------ + +.. automodule:: firebolt.model.instance_type + :members: + :exclude-members: InstanceTypeKey + :undoc-members: + :show-inheritance: + +firebolt.model.provider module +------------------------------ + +.. automodule:: firebolt.model.provider + :members: + :undoc-members: + :show-inheritance: + +firebolt.model.region module +---------------------------- + +.. automodule:: firebolt.model.region + :members: + :exclude-members: RegionKey + :undoc-members: + :show-inheritance: + + diff --git a/docsrc/firebolt.service.rst b/docsrc/firebolt.service.rst new file mode 100644 index 00000000000..0f5db3cface --- /dev/null +++ b/docsrc/firebolt.service.rst @@ -0,0 +1,81 @@ +======================== +firebolt.service package +======================== + + +.. automodule:: firebolt.service.base + :members: + :exclude-members: BaseService + :undoc-members: + :show-inheritance: + +.. automodule:: firebolt.service.binding + :members: + :exclude-members: BindingService + :undoc-members: + :show-inheritance: + +firebolt.service.database module +-------------------------------- + +.. automodule:: firebolt.service.database + :members: + :undoc-members: + :show-inheritance: + +firebolt.service.engine module +------------------------------ + +.. automodule:: firebolt.service.engine + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: firebolt.service.engine_revision + :members: + :exclude-members: EngineRevisionService + :undoc-members: + :show-inheritance: + +firebolt.service.instance\_type module +-------------------------------------- + +.. automodule:: firebolt.service.instance_type + :members: + :exclude-members: InstanceTypeLookup + :undoc-members: + :show-inheritance: + +firebolt.service.manager module +------------------------------- + +.. automodule:: firebolt.service.manager + :members: + :undoc-members: + :show-inheritance: + +firebolt.service.provider module +-------------------------------- + +.. automodule:: firebolt.service.provider + :members: + :undoc-members: + :show-inheritance: + +firebolt.service.region module +------------------------------ + +.. automodule:: firebolt.service.region + :members: + :undoc-members: + :show-inheritance: + +firebolt.service.types module +----------------------------- + +.. automodule:: firebolt.service.types + :members: + :undoc-members: + :show-inheritance: + + diff --git a/docsrc/index.rst b/docsrc/index.rst new file mode 100644 index 00000000000..a384c7709e6 --- /dev/null +++ b/docsrc/index.rst @@ -0,0 +1,83 @@ +.. firebolt-sdk documentation master file, created by + sphinx-quickstart on Wed Nov 17 11:16:16 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +================= +**firebolt-sdk** +================= + +Welcome to firebolt-sdk's documentation! + +######################## +**Installation** +######################## + +* Requires Python ``>=3.7`` +* ``pip install firebolt-sdk`` + +########################## +**Connection parameters** +########################## + +These parameters are used to connect to a Firebolt database: + +* **engine_url** - url for a Firebolt engine to make requests to. This can be retrieved from our web interface, or from the `engine `_ attribute endpoint +* **database** - the name of the database to receive queries +* **username** - Firebolt account username +* **password** - Firebolt account password + +Optional parameters + +* **api_endpoint** - api hostname for logging in. Defaults to ``api.app.firebolt.io``. + +############### +**Examples** +############### + +See `PEP-249 `_ for the DB API reference and specifications. An example `jupyter notebook `_ is included to illustrate the use of the Firebolt API. + + +####################### +**Optional features** +####################### + +By default, firebolt-sdk uses ``datetime`` module to parse date and datetime values, which might be slow for a large amount of operations. In order to speed up datetime operations, it's possible to use `ciso8601 `_ package. + +To install firebolt-sdk with ``ciso8601`` support, run ``pip install firebolt-sdk[ciso8601]`` + +################### +**Contributing** +################### +See: `CONTRIBUTING.MD `_ + +################### +**License** +################### + +The Firebolt DB API is licensed under the `Apache License Version 2.0 `_ software license. + +.. note:: + + This project is under active development + +======================================== + +.. toctree:: + :maxdepth: 1 + + firebolt.async_db + firebolt.client + firebolt.common + firebolt.db + firebolt.model + firebolt.service + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docsrc/make.bat b/docsrc/make.bat new file mode 100644 index 00000000000..3331730fe0a --- /dev/null +++ b/docsrc/make.bat @@ -0,0 +1,42 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +if "%1" == "github" ( + %SPHINXBUILD% -M html %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + robocopy %BUILDDIR%/html ../docs /E > nul + echo.Generated files copied to ../docs + goto end +) + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd From ff7783170616e5a0a491f41568bd2d1e3f399555 Mon Sep 17 00:00:00 2001 From: Eric Gustavson Date: Tue, 14 Dec 2021 15:45:23 -0800 Subject: [PATCH 02/12] fix: black --- docsrc/conf.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docsrc/conf.py b/docsrc/conf.py index 57f46c9d1cf..ea4f0c8f283 100644 --- a/docsrc/conf.py +++ b/docsrc/conf.py @@ -14,15 +14,15 @@ import sys sys.path += [ - os.path.abspath("../src"), - os.path.abspath("../src/firebolt/"), - os.path.abspath("../src/firebolt/async_db"), - os.path.abspath("../src/firebolt/client"), - os.path.abspath("../src/firebolt/common"), - os.path.abspath("../src/firebolt/db"), - os.path.abspath("../src/firebolt/model"), - os.path.abspath("../src/firebolt/service") - ] + os.path.abspath("../src"), + os.path.abspath("../src/firebolt/"), + os.path.abspath("../src/firebolt/async_db"), + os.path.abspath("../src/firebolt/client"), + os.path.abspath("../src/firebolt/common"), + os.path.abspath("../src/firebolt/db"), + os.path.abspath("../src/firebolt/model"), + os.path.abspath("../src/firebolt/service"), +] print(sys.path) # -- Project information ----------------------------------------------------- From a56375a2e70f2d1370583a4b2887ed20f3b3466f Mon Sep 17 00:00:00 2001 From: Eric Gustavson Date: Wed, 15 Dec 2021 11:19:21 -0800 Subject: [PATCH 03/12] fix: gitignore --- .gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5784b334ad5..2e08a3ef5c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ docsrc/_build docs/* !docs/requirements.txt + .idea # Created by https://www.toptal.com/developers/gitignore/api/python,pycharm,macos @@ -285,4 +286,10 @@ cython_debug/ # Emacs *~ -\#*\# \ No newline at end of file +\#*\# + +coverage-reports/* + +# VS Code +.vscode/* +*.code-workspace From a2db7d4469eca42b00cef0c6e9f295069fb85e8a Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 10:11:39 -0800 Subject: [PATCH 04/12] Revisions to docstrings --- src/firebolt/async_db/connection.py | 26 +++--- src/firebolt/async_db/cursor.py | 66 ++++++------- src/firebolt/client/auth.py | 3 +- src/firebolt/client/client.py | 12 +-- src/firebolt/common/exception.py | 18 +--- src/firebolt/db/connection.py | 22 ++--- src/firebolt/db/cursor.py | 29 ++---- src/firebolt/model/engine.py | 2 +- src/firebolt/model/engine_revision.py | 2 +- src/firebolt/service/manager.py | 2 + src/firebolt/service/region.py | 4 +- src/firebolt/service/types.py | 128 ++++++++++++++++---------- 12 files changed, 144 insertions(+), 170 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index 62f25351b6b..f2aabf446e1 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -64,19 +64,17 @@ async def connect_inner( engine_url: Optional[str] = None, api_endpoint: str = DEFAULT_API_URL, ) -> Connection: - cleandoc( - """ - Connect to Firebolt database. - - Connection parameters: - database - name of the database to connect - username - user name to use for authentication - password - password to use for authentication - engine_name - name of the engine to connect to - engine_url - engine endpoint to use - note: either engine_name or engine_url should be provided, but not both - """ - ) + """ + Connect to Firebolt database. + :param database: name of the database to connect + :param username: user name to use for authentication + :param password: password to use for authentication + :param engine_name: The name of the engine to connect to + :type engine_name: optional + :param engine_url: The engine endpoint to use + :type engine_url: optional + .. note:: either `engine_name` or `engine_url` should be provided, but not both + """ if engine_name and engine_url: raise InterfaceError( @@ -198,7 +196,6 @@ def commit(self) -> None: class Connection(BaseConnection): - cleandoc( """ Firebolt asyncronous database connection class. Implements PEP-249. @@ -216,7 +213,6 @@ class Connection(BaseConnection): Firebolt currenly doesn't support transactions so commit and rollback methods are not implemented. """ - ) cursor_class = Cursor diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index b127a5fd715..4b84f93c9fb 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -118,19 +118,18 @@ def __del__(self) -> None: @property # type: ignore @check_not_closed def description(self) -> Optional[List[Column]]: - cleandoc( - """ - Provides information about a single result row of a query - Attributes: - - name - - type_code - - display_size - - internal_size - - precision - - scale - - null_ok - """ - ) + """ + Provides information about a single result row of a query + + Attributes: + * ``name`` + * ``type_code`` + * ``display_size`` + * ``internal_size`` + * ``precision`` + * ``scale`` + * ``null_ok`` + """ return self._descriptions @property # type: ignore @@ -265,12 +264,10 @@ async def execute( async def executemany( self, query: str, parameters_seq: Sequence[Sequence[ParameterType]] ) -> int: - cleandoc( """ Prepare and execute a database query against all parameter sequences provided. Return last query row count. """ - ) self._reset() resp = None for parameters in parameters_seq: @@ -318,12 +315,10 @@ def fetchone(self) -> Optional[List[ColType]]: @check_not_closed @check_query_executed def fetchmany(self, size: Optional[int] = None) -> List[List[ColType]]: - cleandoc( """ Fetch the next set of rows of a query result, cursor.arraysize is default size. """ - ) size = size if size is not None else self.arraysize left, right = self._get_next_range(size) assert self._rows is not None @@ -359,32 +354,17 @@ def __exit__( class Cursor(BaseCursor): - cleandoc( """ Class, responsible for executing asyncio queries to Firebolt Database. Should not be created directly, use connection.cursor() - Properties: - - description - information about a single result row - - rowcount - the number of rows produced by last query - - closed - True if connection is closed, False otherwise - - arraysize - Read/Write, specifies the number of rows to fetch at a time - with .fetchmany method - - Methods: - - close - terminate an ongoing query (if any) and mark connection as closed - - execute - prepare and execute a database query - - executemany - prepare and execute a database query against all parameter - sequences provided - - fetchone - fetch the next row of a query result set - - fetchmany - fetch the next set of rows of a query result, - size is cursor.arraysize by default - - fetchall - fetch all remaining rows of a query result - - setinputsizes - predefine memory areas for query parameters (does nothing) - - setoutputsize - set a column buffer size for fetches of large columns - (does nothing) + **Properties:** + + * ``description`` - information about a single result row + * ``rowcount`` - the number of rows produced by last query + * ``closed`` - True if connection is closed, False otherwise + * ``arraysize`` - Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method """ - ) __slots__ = BaseCursor.__slots__ + ("_async_query_lock",) @@ -401,6 +381,7 @@ async def execute( ) -> int: async with self._async_query_lock.writer: return await super().execute(query, parameters, set_parameters) + """Prepare and execute a database query""" @wraps(BaseCursor.executemany) async def executemany( @@ -408,21 +389,28 @@ async def executemany( ) -> int: async with self._async_query_lock.writer: return await super().executemany(query, parameters_seq) + """ + Prepare and execute a database query against all parameter + sequences provided + """ @wraps(BaseCursor.fetchone) async def fetchone(self) -> Optional[List[ColType]]: async with self._async_query_lock.reader: return super().fetchone() + """Fetch the next row of a query result set""" @wraps(BaseCursor.fetchmany) async def fetchmany(self, size: Optional[int] = None) -> List[List[ColType]]: async with self._async_query_lock.reader: return super().fetchmany(size) - + """fetch the next set of rows of a query result, + size is cursor.arraysize by default""" @wraps(BaseCursor.fetchall) async def fetchall(self) -> List[List[ColType]]: async with self._async_query_lock.reader: return super().fetchall() + """Fetch all remaining rows of a query result""" # Iteration support @check_not_closed diff --git a/src/firebolt/client/auth.py b/src/firebolt/client/auth.py index f8d3b785e01..2696e21aff5 100644 --- a/src/firebolt/client/auth.py +++ b/src/firebolt/client/auth.py @@ -12,12 +12,11 @@ class Auth(HttpxAuth): - cleandoc( + """ Authentication class for Firebolt database. Gets authentication token using provided credentials and updates it when it expires """ - ) __slots__ = ( "username", diff --git a/src/firebolt/client/client.py b/src/firebolt/client/client.py index f8713f85765..f0b55ea9ae1 100644 --- a/src/firebolt/client/client.py +++ b/src/firebolt/client/client.py @@ -40,7 +40,7 @@ def _build_auth(self, auth: _types.AuthTypes) -> Optional[Auth]: class Client(FireboltClientMixin, HttpxClient): - cleandoc( + """ An http client, based on httpx.Client, that handles the authentication for Firebolt database. @@ -49,9 +49,9 @@ class Client(FireboltClientMixin, HttpxClient): FireboltAuth instance httpx.Client: - """ + + (HttpxClient.__doc__ or "") - ) + """ @cached_property def account_id(self) -> str: @@ -59,7 +59,6 @@ def account_id(self) -> str: class AsyncClient(FireboltClientMixin, HttpxAsyncClient): - cleandoc( """ An http client, based on httpx.AsyncClient, that asyncronously handles authentication for Firebolt database. @@ -68,10 +67,9 @@ class AsyncClient(FireboltClientMixin, HttpxAsyncClient): FireboltAuth instance httpx.AsyncClient: - """ - + (HttpxAsyncClient.__doc__ or "") - ) + + (HttpxAsyncClient.__doc__ or "") + """ @async_cached_property async def account_id(self) -> str: return (await self.get(url=ACCOUNT_URL)).json()["account"]["id"] diff --git a/src/firebolt/common/exception.py b/src/firebolt/common/exception.py index 54ae21a67e3..c15761811a7 100644 --- a/src/firebolt/common/exception.py +++ b/src/firebolt/common/exception.py @@ -92,24 +92,22 @@ def __str__(self) -> str: class Warning(Exception): - cleandoc( """ Exception raised for important warnings like data truncations while inserting, etc. """ - ) + Error = FireboltError class InterfaceError(Error): - cleandoc( + """ Exception raised for errors that are related to the database interface rather than the database itself. """ - ) class DatabaseError(Error): @@ -117,56 +115,44 @@ class DatabaseError(Error): class DataError(DatabaseError): - cleandoc( """ Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range, etc. """ - ) class OperationalError(DatabaseError): - cleandoc( """ Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer, e.g. an unexpected disconnect occurs, the data source name is not found, a transaction could not be processed, a memory allocation error occurred during processing, etc. """ - ) class IntegrityError(DatabaseError): - cleandoc( """ Exception raised when the relational integrity of the database is affected, e.g. a foreign key check fails. """ - ) class InternalError(DatabaseError): - cleandoc( """ Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. """ - ) class ProgrammingError(DatabaseError): - cleandoc( """ Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. """ - ) class NotSupportedError(DatabaseError): - cleandoc( """ Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. """ - ) diff --git a/src/firebolt/db/connection.py b/src/firebolt/db/connection.py index 6c3303a0c55..5b5f89fdea2 100644 --- a/src/firebolt/db/connection.py +++ b/src/firebolt/db/connection.py @@ -18,25 +18,17 @@ class Connection(AsyncBaseConnection): - cleandoc( """ Firebolt database connection class. Implements PEP-249. - - Parameters: - engine_url - Firebolt database engine REST API url - database - Firebolt database name - username - Firebolt account username - password - Firebolt account password - api_endpoint(optional) - Firebolt API endpoint. Used for authentication - - Methods: - cursor - create new Cursor object - close - close the Connection and all it's cursors - - Firebolt currenly doesn't support transactions so commit and rollback methods + :param engine_url: Firebolt database engine REST API url + :param database: Firebolt database name + :param username: Firebolt account username + :param password: Firebolt account password + :param api_endpoint: Firebolt API endpoint. Used for authentication + :type api_endpoint: Optional + .. note:: Firebolt currenly doesn't support transactions so commit and rollback methods are not implemented. """ - ) __slots__ = AsyncBaseConnection.__slots__ + ("_closing_lock",) diff --git a/src/firebolt/db/cursor.py b/src/firebolt/db/cursor.py index e2ebb1f5347..67987e2c5e8 100644 --- a/src/firebolt/db/cursor.py +++ b/src/firebolt/db/cursor.py @@ -18,32 +18,15 @@ class Cursor(AsyncBaseCursor): - cleandoc( """ Class, responsible for executing queries to Firebolt Database. - Should not be created directly, use connection.cursor() - - Properties: - - description - information about a single result row - - rowcount - the number of rows produced by last query - - closed - True if connection is closed, False otherwise - - arraysize - Read/Write, specifies the number of rows to fetch at a time - with .fetchmany method - - Methods: - - close - terminate an ongoing query (if any) and mark connection as closed - - execute - prepare and execute a database query - - executemany - prepare and execute a database query against all parameter - sequences provided - - fetchone - fetch the next row of a query result set - - fetchmany - fetch the next set of rows of a query result, - size is cursor.arraysize by default - - fetchall - fetch all remaining rows of a query result - - setinputsizes - predefine memory areas for query parameters (does nothing) - - setoutputsize - set a column buffer size for fetches of large columns - (does nothing) + Should not be created directly, use ``connection.cursor()`` + **Properties** + * ``description`` - information about a single result row + * ``rowcount`` - the number of rows produced by last query + * ``closed`` - True if connection is closed, False otherwise + * ``arraysize`` - Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method """ - ) __slots__ = AsyncBaseCursor.__slots__ + ("_query_lock", "_idx_lock") diff --git a/src/firebolt/model/engine.py b/src/firebolt/model/engine.py index 0a4860fcc50..8b34610c4ed 100644 --- a/src/firebolt/model/engine.py +++ b/src/firebolt/model/engine.py @@ -42,7 +42,7 @@ class EngineSettings(FireboltBaseModel): """ Engine Settings. - See Also: engine_revision.Specification which also contains engine configuration. + See Also: ``engine_revision.Specification`` which also contains engine configuration. """ preset: str diff --git a/src/firebolt/model/engine_revision.py b/src/firebolt/model/engine_revision.py index 6756aa14442..6404a6acb85 100644 --- a/src/firebolt/model/engine_revision.py +++ b/src/firebolt/model/engine_revision.py @@ -21,7 +21,7 @@ class EngineRevisionSpecification(FireboltBaseModel): Notably, it determines which instance types and how many of them its Engine gets. - See Also: engine.Settings, which also contains engine configuration. + See Also: ``engine.Settings``, which also contains engine configuration. """ db_compute_instances_type_key: InstanceTypeKey = Field( diff --git a/src/firebolt/service/manager.py b/src/firebolt/service/manager.py index 8e7eddc9dc4..948fd5678fd 100644 --- a/src/firebolt/service/manager.py +++ b/src/firebolt/service/manager.py @@ -20,7 +20,9 @@ class ResourceManager: - engine revisions (versions of an engine) Also provides listings of: + - regions (AWS regions in which engines can run) + - instance types (AWS instance types which engines can use) """ diff --git a/src/firebolt/service/region.py b/src/firebolt/service/region.py index c05c37c3478..8571f4e7d88 100644 --- a/src/firebolt/service/region.py +++ b/src/firebolt/service/region.py @@ -25,12 +25,12 @@ def regions(self) -> List[Region]: @cached_property def regions_by_name(self) -> Dict[str, Region]: - """Dict of {RegionLookup: Region}""" + """Dict of {RegionLookup to Region}""" return {r.name: r for r in self.regions} @cached_property def regions_by_key(self) -> Dict[RegionKey, Region]: - """Dict of {RegionKey: Region}""" + """Dict of {RegionKey to Region}""" return {r.key: r for r in self.regions} @cached_property diff --git a/src/firebolt/service/types.py b/src/firebolt/service/types.py index de2df6a312e..0226d957858 100644 --- a/src/firebolt/service/types.py +++ b/src/firebolt/service/types.py @@ -37,76 +37,94 @@ class EngineStatus(Enum): ENGINE_STATUS_UNSPECIFIED = "ENGINE_STATUS_UNSPECIFIED" - # Logical record is created, however underlying infrastructure is not initialized. - # In other words this means that engine is stopped. + ''' Logical record is created, however underlying infrastructure is not initialized. + In other words this means that engine is stopped.''' ENGINE_STATUS_CREATED = "ENGINE_STATUS_CREATED" - # Engine initialization request was sent. + """Engine status was created.""" ENGINE_STATUS_PROVISIONING_PENDING = "ENGINE_STATUS_PROVISIONING_PENDING" - # Engine initialization request was received and initialization process started. + ''' Engine initialization request was sent.''' ENGINE_STATUS_PROVISIONING_STARTED = "ENGINE_STATUS_PROVISIONING_STARTED" - # Engine initialization was finished successfully. + ''' Engine initialization request was received and initialization process started.''' ENGINE_STATUS_PROVISIONING_FINISHED = "ENGINE_STATUS_PROVISIONING_FINISHED" - # Engine initialization failed due to error. + ''' Engine initialization was finished successfully.''' ENGINE_STATUS_PROVISIONING_FAILED = "ENGINE_STATUS_PROVISIONING_FAILED" - # Engine is initialized, but there are no running or starting engine revisions. + ''' Engine initialization failed due to error.''' + ENGINE_STATUS_RUNNING_IDLE = "ENGINE_STATUS_RUNNING_IDLE" - # Engine is initialized, there are no running engine revision but it's starting. + ''' Engine is initialized, but there are no running or starting engine revisions.''' + ENGINE_STATUS_RUNNING_REVISION_STARTING = "ENGINE_STATUS_RUNNING_REVISION_STARTING" - # Engine is initialized, initial revision is failed to provision or start. + ''' Engine is initialized, there are no running engine revision but it's starting.''' + ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED" ) - # Engine is ready (serves an engine revision). + ''' Engine is initialized, initial revision is failed to provision or start.''' + ENGINE_STATUS_RUNNING_REVISION_SERVING = "ENGINE_STATUS_RUNNING_REVISION_SERVING" - # Engine is ready (serves an engine revision), - # zero-downtime replacement revision is starting. + ''' Engine is ready (serves an engine revision). ''' + ENGINE_STATUS_RUNNING_REVISION_CHANGING = "ENGINE_STATUS_RUNNING_REVISION_CHANGING" - # Engine is ready (serves an engine revision), - # replacement revision failed to provision or start. + ''' Engine is ready (serves an engine revision), + zero-downtime replacement revision is starting.''' + ENGINE_STATUS_RUNNING_REVISION_CHANGE_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_CHANGE_FAILED" ) - # Engine is initialized, replacement of the revision with a downtime is in progress. + ''' Engine is ready (serves an engine revision), + replacement revision failed to provision or start.''' + ENGINE_STATUS_RUNNING_REVISION_RESTARTING = ( "ENGINE_STATUS_RUNNING_REVISION_RESTARTING" ) - # Engine is initialized, replacement revision failed to provision or start. + ''' Engine is initialized, replacement of the revision with a downtime is in progress.''' + ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED" ) - # Engine is initialized, all child revisions are being terminated. + ''' Engine is initialized, replacement revision failed to provision or start.''' + + ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING = ( "ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING" ) + ''' Engine is initialized, all child revisions are being terminated.''' + + # Engine termination request was sent. ENGINE_STATUS_TERMINATION_PENDING = "ENGINE_STATUS_TERMINATION_PENDING" - # Engine termination started. + ''' Engine termination request was sent.''' + ENGINE_STATUS_TERMINATION_ST = "ENGINE_STATUS_TERMINATION_STARTED" - # Engine termination finished. + ''' Engine termination started.''' + ENGINE_STATUS_TERMINATION_FIN = "ENGINE_STATUS_TERMINATION_FINISHED" - # Engine termination failed. + ''' Engine termination finished.''' + ENGINE_STATUS_TERMINATION_F = "ENGINE_STATUS_TERMINATION_FAILED" - # Engine is soft-deleted. + ''' Engine termination failed.''' + ENGINE_STATUS_DELETED = "ENGINE_STATUS_DELETED" + ''' Engine is soft-deleted.''' class EngineStatusSummary(Enum): """ @@ -117,62 +135,74 @@ class EngineStatusSummary(Enum): ENGINE_STATUS_SUMMARY_UNSPECIFIED = "ENGINE_STATUS_SUMMARY_UNSPECIFIED" - # Fully stopped. + """Status unspecified""" ENGINE_STATUS_SUMMARY_STOPPED = "ENGINE_STATUS_SUMMARY_STOPPED" - # Provisioning process is in progress. - # We are creating cloud infra for this engine. + ''' Fully stopped.''' + ENGINE_STATUS_SUMMARY_STARTING = "ENGINE_STATUS_SUMMARY_STARTING" - # Provisioning process is complete. - # We are now waiting for PackDB cluster to initialize and start. + ''' Provisioning process is in progress. + We are creating cloud infra for this engine.''' + ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING = ( "ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING" ) - # Fully started. - # Engine is ready to serve requests. + ''' Provisioning process is complete. + We are now waiting for PackDB cluster to initialize and start.''' + ENGINE_STATUS_SUMMARY_RUNNING = "ENGINE_STATUS_SUMMARY_RUNNING" - # Version of the PackDB is changing. - # This is zero downtime operation that does not affect engine work.s - # This status is reserved for future use (not used fow now). + ''' Fully started. + Engine is ready to serve requests.''' + ENGINE_STATUS_SUMMARY_UPGRADING = "ENGINE_STATUS_SUMMARY_UPGRADING" - # Hard restart (full stop/start cycle) is in progress. - # Underlying infrastructure is being recreated. + ''' Version of the PackDB is changing. + This is zero downtime operation that does not affect engine work.s + This status is reserved for future use (not used fow now).''' + ENGINE_STATUS_SUMMARY_RESTARTING = "ENGINE_STATUS_SUMMARY_RESTARTING" - # Hard restart (full stop/start cycle) is in progress. - # Underlying infrastructure is ready, waiting for - # PackDB cluster to initialize and start. - # This status is logically the same as ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING, - # but used during restart cycle. + ''' Hard restart (full stop/start cycle) is in progress. + Underlying infrastructure is being recreated.''' + ENGINE_STATUS_SUMMARY_RESTARTING_INITIALIZING = ( "ENGINE_STATUS_SUMMARY_RESTARTING_INITIALIZING" ) - # Underlying infrastructure has issues and is being repaired. - # Engine is still running, but it's not fully healthy and some queries may fail. + ''' Hard restart (full stop/start cycle) is in progress. + Underlying infrastructure is ready, waiting for + PackDB cluster to initialize and start. + This status is logically the same as ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING, + but used during restart cycle.''' + ENGINE_STATUS_SUMMARY_REPAIRING = "ENGINE_STATUS_SUMMARY_REPAIRING" - # Stop is in progress. + ''' Underlying infrastructure has issues and is being repaired. + Engine is still running, but it's not fully healthy and some queries may fail.''' + ENGINE_STATUS_SUMMARY_STOPPING = "ENGINE_STATUS_SUMMARY_STOPPING" - # Termination is in progress. - # All infrastructure that belongs to this engine will be completely destroyed. + ''' Stop is in progress.''' + ENGINE_STATUS_SUMMARY_DELETING = "ENGINE_STATUS_SUMMARY_DELETING" - # Infrastructure is terminated, engine data is deleted. + ''' Termination is in progress. + All infrastructure that belongs to this engine will be completely destroyed.''' + ENGINE_STATUS_SUMMARY_DELETED = "ENGINE_STATUS_SUMMARY_DELETED" - # Failed to start or stop. - # This status only indicates that there were issues during provisioning operations. - # If engine enters this status, - # all infrastructure should be stopped/terminated already. - ENGINE_STATUS_SUMMARY_FAILED = "ENGINE_STATUS_SUMMARY_FAILED" + ''' Infrastructure is terminated, engine data is deleted.''' + ENGINE_STATUS_SUMMARY_FAILED = "ENGINE_STATUS_SUMMARY_FAILED" + ''' Failed to start or stop. + This status only indicates that there were issues during provisioning operations. + If engine enters this status, + all infrastructure should be stopped/terminated already.''' + class EngineOrder(Enum): ENGINE_ORDER_UNSPECIFIED = "ENGINE_ORDER_UNSPECIFIED" ENGINE_ORDER_NAME_ASC = "ENGINE_ORDER_NAME_ASC" From a8ae5d07553caca8fa17e231aa610b6790bc465d Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 10:21:44 -0800 Subject: [PATCH 05/12] Changed ''' to """ in docstrings --- src/firebolt/service/types.py | 86 +++++++++++++++++------------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/firebolt/service/types.py b/src/firebolt/service/types.py index 0226d957858..72d7c365162 100644 --- a/src/firebolt/service/types.py +++ b/src/firebolt/service/types.py @@ -37,94 +37,94 @@ class EngineStatus(Enum): ENGINE_STATUS_UNSPECIFIED = "ENGINE_STATUS_UNSPECIFIED" - ''' Logical record is created, however underlying infrastructure is not initialized. - In other words this means that engine is stopped.''' + """ Logical record is created, however underlying infrastructure is not initialized. + In other words this means that engine is stopped.""" ENGINE_STATUS_CREATED = "ENGINE_STATUS_CREATED" """Engine status was created.""" ENGINE_STATUS_PROVISIONING_PENDING = "ENGINE_STATUS_PROVISIONING_PENDING" - ''' Engine initialization request was sent.''' + """ Engine initialization request was sent.""" ENGINE_STATUS_PROVISIONING_STARTED = "ENGINE_STATUS_PROVISIONING_STARTED" - ''' Engine initialization request was received and initialization process started.''' + """ Engine initialization request was received and initialization process started.""" ENGINE_STATUS_PROVISIONING_FINISHED = "ENGINE_STATUS_PROVISIONING_FINISHED" - ''' Engine initialization was finished successfully.''' + """ Engine initialization was finished successfully.""" ENGINE_STATUS_PROVISIONING_FAILED = "ENGINE_STATUS_PROVISIONING_FAILED" - ''' Engine initialization failed due to error.''' + """ Engine initialization failed due to error.""" ENGINE_STATUS_RUNNING_IDLE = "ENGINE_STATUS_RUNNING_IDLE" - ''' Engine is initialized, but there are no running or starting engine revisions.''' + """ Engine is initialized, but there are no running or starting engine revisions.""" ENGINE_STATUS_RUNNING_REVISION_STARTING = "ENGINE_STATUS_RUNNING_REVISION_STARTING" - ''' Engine is initialized, there are no running engine revision but it's starting.''' + """ Engine is initialized, there are no running engine revision but it's starting.""" ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED" ) - ''' Engine is initialized, initial revision is failed to provision or start.''' + """ Engine is initialized, initial revision is failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISION_SERVING = "ENGINE_STATUS_RUNNING_REVISION_SERVING" - ''' Engine is ready (serves an engine revision). ''' + """ Engine is ready (serves an engine revision). """ ENGINE_STATUS_RUNNING_REVISION_CHANGING = "ENGINE_STATUS_RUNNING_REVISION_CHANGING" - ''' Engine is ready (serves an engine revision), - zero-downtime replacement revision is starting.''' + """ Engine is ready (serves an engine revision), + zero-downtime replacement revision is starting.""" ENGINE_STATUS_RUNNING_REVISION_CHANGE_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_CHANGE_FAILED" ) - ''' Engine is ready (serves an engine revision), - replacement revision failed to provision or start.''' + """ Engine is ready (serves an engine revision), + replacement revision failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISION_RESTARTING = ( "ENGINE_STATUS_RUNNING_REVISION_RESTARTING" ) - ''' Engine is initialized, replacement of the revision with a downtime is in progress.''' + """ Engine is initialized, replacement of the revision with a downtime is in progress.""" ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED" ) - ''' Engine is initialized, replacement revision failed to provision or start.''' + """ Engine is initialized, replacement revision failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING = ( "ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING" ) - ''' Engine is initialized, all child revisions are being terminated.''' + """ Engine is initialized, all child revisions are being terminated.""" # Engine termination request was sent. ENGINE_STATUS_TERMINATION_PENDING = "ENGINE_STATUS_TERMINATION_PENDING" - ''' Engine termination request was sent.''' + """ Engine termination request was sent.""" ENGINE_STATUS_TERMINATION_ST = "ENGINE_STATUS_TERMINATION_STARTED" - ''' Engine termination started.''' + """ Engine termination started.""" ENGINE_STATUS_TERMINATION_FIN = "ENGINE_STATUS_TERMINATION_FINISHED" - ''' Engine termination finished.''' + """ Engine termination finished.""" ENGINE_STATUS_TERMINATION_F = "ENGINE_STATUS_TERMINATION_FAILED" - ''' Engine termination failed.''' + """ Engine termination failed.""" ENGINE_STATUS_DELETED = "ENGINE_STATUS_DELETED" - ''' Engine is soft-deleted.''' + """ Engine is soft-deleted.""" class EngineStatusSummary(Enum): """ @@ -138,70 +138,70 @@ class EngineStatusSummary(Enum): """Status unspecified""" ENGINE_STATUS_SUMMARY_STOPPED = "ENGINE_STATUS_SUMMARY_STOPPED" - ''' Fully stopped.''' + """ Fully stopped.""" ENGINE_STATUS_SUMMARY_STARTING = "ENGINE_STATUS_SUMMARY_STARTING" - ''' Provisioning process is in progress. - We are creating cloud infra for this engine.''' + """ Provisioning process is in progress. + We are creating cloud infra for this engine.""" ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING = ( "ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING" ) - ''' Provisioning process is complete. - We are now waiting for PackDB cluster to initialize and start.''' + """ Provisioning process is complete. + We are now waiting for PackDB cluster to initialize and start.""" ENGINE_STATUS_SUMMARY_RUNNING = "ENGINE_STATUS_SUMMARY_RUNNING" - ''' Fully started. - Engine is ready to serve requests.''' + """ Fully started. + Engine is ready to serve requests.""" ENGINE_STATUS_SUMMARY_UPGRADING = "ENGINE_STATUS_SUMMARY_UPGRADING" - ''' Version of the PackDB is changing. + """ Version of the PackDB is changing. This is zero downtime operation that does not affect engine work.s - This status is reserved for future use (not used fow now).''' + This status is reserved for future use (not used fow now).""" ENGINE_STATUS_SUMMARY_RESTARTING = "ENGINE_STATUS_SUMMARY_RESTARTING" - ''' Hard restart (full stop/start cycle) is in progress. - Underlying infrastructure is being recreated.''' + """ Hard restart (full stop/start cycle) is in progress. + Underlying infrastructure is being recreated.""" ENGINE_STATUS_SUMMARY_RESTARTING_INITIALIZING = ( "ENGINE_STATUS_SUMMARY_RESTARTING_INITIALIZING" ) - ''' Hard restart (full stop/start cycle) is in progress. + """ Hard restart (full stop/start cycle) is in progress. Underlying infrastructure is ready, waiting for PackDB cluster to initialize and start. This status is logically the same as ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING, - but used during restart cycle.''' + but used during restart cycle.""" ENGINE_STATUS_SUMMARY_REPAIRING = "ENGINE_STATUS_SUMMARY_REPAIRING" - ''' Underlying infrastructure has issues and is being repaired. - Engine is still running, but it's not fully healthy and some queries may fail.''' + """ Underlying infrastructure has issues and is being repaired. + Engine is still running, but it's not fully healthy and some queries may fail.""" ENGINE_STATUS_SUMMARY_STOPPING = "ENGINE_STATUS_SUMMARY_STOPPING" - ''' Stop is in progress.''' + """ Stop is in progress.""" ENGINE_STATUS_SUMMARY_DELETING = "ENGINE_STATUS_SUMMARY_DELETING" - ''' Termination is in progress. - All infrastructure that belongs to this engine will be completely destroyed.''' + """ Termination is in progress. + All infrastructure that belongs to this engine will be completely destroyed.""" ENGINE_STATUS_SUMMARY_DELETED = "ENGINE_STATUS_SUMMARY_DELETED" - ''' Infrastructure is terminated, engine data is deleted.''' + """ Infrastructure is terminated, engine data is deleted.""" ENGINE_STATUS_SUMMARY_FAILED = "ENGINE_STATUS_SUMMARY_FAILED" - ''' Failed to start or stop. + """ Failed to start or stop. This status only indicates that there were issues during provisioning operations. If engine enters this status, - all infrastructure should be stopped/terminated already.''' + all infrastructure should be stopped/terminated already.""" class EngineOrder(Enum): ENGINE_ORDER_UNSPECIFIED = "ENGINE_ORDER_UNSPECIFIED" From 8863afa3a5622989b41b3fb1dbe153a775ceca77 Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 10:41:40 -0800 Subject: [PATCH 06/12] Test for Google docstring --- src/firebolt/async_db/connection.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index f2aabf446e1..5929ad2eb2a 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -66,14 +66,19 @@ async def connect_inner( ) -> Connection: """ Connect to Firebolt database. - :param database: name of the database to connect - :param username: user name to use for authentication - :param password: password to use for authentication - :param engine_name: The name of the engine to connect to - :type engine_name: optional - :param engine_url: The engine endpoint to use - :type engine_url: optional - .. note:: either `engine_name` or `engine_url` should be provided, but not both + + Args: + + :param database: name of the database to connect + :param username: user name to use for authentication + :param password: password to use for authentication + :param engine_name: The name of the engine to connect to + :type engine_name: optional + :param engine_url: The engine endpoint to use + :type engine_url: optional + + Note: + either `engine_name` or `engine_url` should be provided, but not both """ if engine_name and engine_url: From 1c18df1e0aa58026e869cc3857cebd9becfe0c7a Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 10:56:51 -0800 Subject: [PATCH 07/12] Test for Google docstring #2 --- src/firebolt/async_db/connection.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index 5929ad2eb2a..e5f181ccc53 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -67,18 +67,15 @@ async def connect_inner( """ Connect to Firebolt database. - Args: - - :param database: name of the database to connect - :param username: user name to use for authentication - :param password: password to use for authentication - :param engine_name: The name of the engine to connect to - :type engine_name: optional - :param engine_url: The engine endpoint to use - :type engine_url: optional - - Note: - either `engine_name` or `engine_url` should be provided, but not both + :param database: name of the database to connect + :param username: user name to use for authentication + :param password: password to use for authentication + :param engine_name: The name of the engine to connect to + :type engine_name: optional + :param engine_url: The engine endpoint to use + :type engine_url: optional + .. note:: either `engine_name` or `engine_url` should be provided, but not both + """ if engine_name and engine_url: From e0323569dbc966a2cc736af354de49f24d014f1e Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 12:07:15 -0800 Subject: [PATCH 08/12] Converted all docstrings to Google format --- src/firebolt/async_db/connection.py | 42 +++++++++++++---------------- src/firebolt/async_db/cursor.py | 41 ++++++++++++++-------------- src/firebolt/client/auth.py | 5 ++-- src/firebolt/client/client.py | 8 +++--- src/firebolt/common/exception.py | 32 +++++++++++----------- src/firebolt/db/connection.py | 24 ++++++++++------- src/firebolt/db/cursor.py | 15 ++++++----- src/firebolt/service/engine.py | 40 +++++++++++++-------------- src/firebolt/service/types.py | 2 +- 9 files changed, 104 insertions(+), 105 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index e5f181ccc53..34ac264f392 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -67,14 +67,15 @@ async def connect_inner( """ Connect to Firebolt database. - :param database: name of the database to connect - :param username: user name to use for authentication - :param password: password to use for authentication - :param engine_name: The name of the engine to connect to - :type engine_name: optional - :param engine_url: The engine endpoint to use - :type engine_url: optional - .. note:: either `engine_name` or `engine_url` should be provided, but not both + Args: + database: name of the database to connect + username: user name to use for authentication + password: password to use for authentication + engine_name: Optional The name of the engine to connect to + engine_url: Optional. The engine endpoint to use + + Note: + either `engine_name` or `engine_url` should be provided, but not both """ @@ -198,24 +199,19 @@ def commit(self) -> None: class Connection(BaseConnection): - """ + """ Firebolt asyncronous database connection class. Implements PEP-249. - Parameters: - engine_url - Firebolt database engine REST API url - database - Firebolt database name - username - Firebolt account username - password - Firebolt account password - api_endpoint(optional) - Firebolt API endpoint. Used for authentication - - Methods: - cursor - create new Cursor object - close - close the Connection and all it's cursors - - Firebolt currenly doesn't support transactions so commit and rollback methods - are not implemented. - """ + Args: + engine_url: Firebolt database engine REST API url + database: Firebolt database name + username: Firebolt account username + password: Firebolt account password + api_endpoint: Optional. Firebolt API endpoint. Used for authentication + Note: + Firebolt currenly doesn't support transactions so commit and rollback methods are not implemented. + """ cursor_class = Cursor aclose = BaseConnection._aclose diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index 4b84f93c9fb..0b852544a41 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -122,13 +122,13 @@ def description(self) -> Optional[List[Column]]: Provides information about a single result row of a query Attributes: - * ``name`` - * ``type_code`` - * ``display_size`` - * ``internal_size`` - * ``precision`` - * ``scale`` - * ``null_ok`` + * ``name`` + * ``type_code`` + * ``display_size`` + * ``internal_size`` + * ``precision`` + * ``scale`` + * ``null_ok`` """ return self._descriptions @@ -264,10 +264,10 @@ async def execute( async def executemany( self, query: str, parameters_seq: Sequence[Sequence[ParameterType]] ) -> int: - """ + """ Prepare and execute a database query against all parameter sequences provided. Return last query row count. - """ + """ self._reset() resp = None for parameters in parameters_seq: @@ -315,10 +315,10 @@ def fetchone(self) -> Optional[List[ColType]]: @check_not_closed @check_query_executed def fetchmany(self, size: Optional[int] = None) -> List[List[ColType]]: - """ + """ Fetch the next set of rows of a query result, cursor.arraysize is default size. - """ + """ size = size if size is not None else self.arraysize left, right = self._get_next_range(size) assert self._rows is not None @@ -354,17 +354,16 @@ def __exit__( class Cursor(BaseCursor): - """ + """ Class, responsible for executing asyncio queries to Firebolt Database. - Should not be created directly, use connection.cursor() + Should not be created directly, use ``connection.cursor()`` - **Properties:** - - * ``description`` - information about a single result row - * ``rowcount`` - the number of rows produced by last query - * ``closed`` - True if connection is closed, False otherwise - * ``arraysize`` - Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method - """ + Args: + description: information about a single result row + rowcount: the number of rows produced by last query + closed: True if connection is closed, False otherwise + arraysize: Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method + """ __slots__ = BaseCursor.__slots__ + ("_async_query_lock",) @@ -404,7 +403,7 @@ async def fetchone(self) -> Optional[List[ColType]]: async def fetchmany(self, size: Optional[int] = None) -> List[List[ColType]]: async with self._async_query_lock.reader: return super().fetchmany(size) - """fetch the next set of rows of a query result, + """fetch the next set of rows of a query result, size is cursor.arraysize by default""" @wraps(BaseCursor.fetchall) async def fetchall(self) -> List[List[ColType]]: diff --git a/src/firebolt/client/auth.py b/src/firebolt/client/auth.py index 2696e21aff5..832aecb97bb 100644 --- a/src/firebolt/client/auth.py +++ b/src/firebolt/client/auth.py @@ -13,11 +13,10 @@ class Auth(HttpxAuth): - """ + """ Authentication class for Firebolt database. Gets authentication token using provided credentials and updates it when it expires - """ - + """ __slots__ = ( "username", "password", diff --git a/src/firebolt/client/client.py b/src/firebolt/client/client.py index f0b55ea9ae1..0545ada089f 100644 --- a/src/firebolt/client/client.py +++ b/src/firebolt/client/client.py @@ -41,7 +41,7 @@ def _build_auth(self, auth: _types.AuthTypes) -> Optional[Auth]: class Client(FireboltClientMixin, HttpxClient): - """ + """ An http client, based on httpx.Client, that handles the authentication for Firebolt database. @@ -51,7 +51,7 @@ class Client(FireboltClientMixin, HttpxClient): httpx.Client: + (HttpxClient.__doc__ or "") - """ + """ @cached_property def account_id(self) -> str: @@ -59,7 +59,7 @@ def account_id(self) -> str: class AsyncClient(FireboltClientMixin, HttpxAsyncClient): - """ + """ An http client, based on httpx.AsyncClient, that asyncronously handles authentication for Firebolt database. @@ -69,7 +69,7 @@ class AsyncClient(FireboltClientMixin, HttpxAsyncClient): httpx.AsyncClient: + (HttpxAsyncClient.__doc__ or "") - """ + """ @async_cached_property async def account_id(self) -> str: return (await self.get(url=ACCOUNT_URL)).json()["account"]["id"] diff --git a/src/firebolt/common/exception.py b/src/firebolt/common/exception.py index c15761811a7..a3304467ad0 100644 --- a/src/firebolt/common/exception.py +++ b/src/firebolt/common/exception.py @@ -92,10 +92,10 @@ def __str__(self) -> str: class Warning(Exception): - """ + """ Exception raised for important warnings like data truncations while inserting, etc. - """ + """ @@ -104,10 +104,10 @@ class Warning(Exception): class InterfaceError(Error): - """ + """ Exception raised for errors that are related to the database interface rather than the database itself. - """ + """ class DatabaseError(Error): @@ -115,44 +115,44 @@ class DatabaseError(Error): class DataError(DatabaseError): - """ + """ Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range, etc. - """ + """ class OperationalError(DatabaseError): - """ + """ Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer, e.g. an unexpected disconnect occurs, the data source name is not found, a transaction could not be processed, a memory allocation error occurred during processing, etc. - """ + """ class IntegrityError(DatabaseError): - """ + """ Exception raised when the relational integrity of the database is affected, e.g. a foreign key check fails. - """ + """ class InternalError(DatabaseError): - """ + """ Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. - """ + """ class ProgrammingError(DatabaseError): - """ + """ Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. - """ + """ class NotSupportedError(DatabaseError): - """ + """ Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. - """ + """ diff --git a/src/firebolt/db/connection.py b/src/firebolt/db/connection.py index 5b5f89fdea2..7e666f52c87 100644 --- a/src/firebolt/db/connection.py +++ b/src/firebolt/db/connection.py @@ -18,17 +18,21 @@ class Connection(AsyncBaseConnection): - """ + """ Firebolt database connection class. Implements PEP-249. - :param engine_url: Firebolt database engine REST API url - :param database: Firebolt database name - :param username: Firebolt account username - :param password: Firebolt account password - :param api_endpoint: Firebolt API endpoint. Used for authentication - :type api_endpoint: Optional - .. note:: Firebolt currenly doesn't support transactions so commit and rollback methods - are not implemented. - """ + + Args: + + engine_url: Firebolt database engine REST API url + database: Firebolt database name + username: Firebolt account username + password: Firebolt account password + api_endpoint: Optional. Firebolt API endpoint. Used for authentication + + Note: + Firebolt currenly doesn't support transactions so commit and rollback methods + are not implemented. + """ __slots__ = AsyncBaseConnection.__slots__ + ("_closing_lock",) diff --git a/src/firebolt/db/cursor.py b/src/firebolt/db/cursor.py index 67987e2c5e8..9d37b2a1d53 100644 --- a/src/firebolt/db/cursor.py +++ b/src/firebolt/db/cursor.py @@ -18,15 +18,16 @@ class Cursor(AsyncBaseCursor): - """ + """ Class, responsible for executing queries to Firebolt Database. Should not be created directly, use ``connection.cursor()`` - **Properties** - * ``description`` - information about a single result row - * ``rowcount`` - the number of rows produced by last query - * ``closed`` - True if connection is closed, False otherwise - * ``arraysize`` - Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method - """ + + Args: + description: Information about a single result row + rowcount: The number of rows produced by last query + closed: True if connection is closed, False otherwise + arraysize: Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method + """ __slots__ = AsyncBaseCursor.__slots__ + ("_query_lock", "_idx_lock") diff --git a/src/firebolt/service/engine.py b/src/firebolt/service/engine.py index 7a2f52ab1ae..131bd1bb594 100644 --- a/src/firebolt/service/engine.py +++ b/src/firebolt/service/engine.py @@ -110,26 +110,26 @@ def create( description: str = "", ) -> Engine: """ - Create a new Engine. - - Args: - name: An identifier that specifies the name of the engine. - region: The AWS region in which the engine runs. - engine_type: The engine type. GENERAL_PURPOSE or DATA_ANALYTICS - scale: The number of compute instances on the engine. - The scale can be any int from 1 to 128. - spec: The AWS EC2 instance type. - auto_stop: The amount of time (in minutes) after which - the engine automatically stops. - warmup: The warmup method that should be used. - MINIMAL: On-demand loading (both indexes and tables' data). - PRELOAD_INDEXES: Load indexes only. - PRELOAD_ALL_DATA: Full data auto-load - (both indexes and table data - full warmup). - description: A short description of the engine's purpose. - - Returns: - Engine with the specified settings. + Create a new Engine. + + Args: + name: An identifier that specifies the name of the engine. + region: The AWS region in which the engine runs. + engine_type: The engine type. GENERAL_PURPOSE or DATA_ANALYTICS + scale: The number of compute instances on the engine. + The scale can be any int from 1 to 128. + spec: The AWS EC2 instance type. + auto_stop: The amount of time (in minutes) after which + the engine automatically stops. + warmup: The warmup method that should be used. + MINIMAL: On-demand loading (both indexes and tables' data). + PRELOAD_INDEXES: Load indexes only. + PRELOAD_ALL_DATA: Full data auto-load + (both indexes and table data - full warmup). + description: A short description of the engine's purpose. + + Returns: + Engine with the specified settings. """ logger.info(f"Creating Engine (name={name})") diff --git a/src/firebolt/service/types.py b/src/firebolt/service/types.py index 72d7c365162..e9a387d2622 100644 --- a/src/firebolt/service/types.py +++ b/src/firebolt/service/types.py @@ -95,7 +95,7 @@ class EngineStatus(Enum): "ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED" ) - """ Engine is initialized, replacement revision failed to provision or start.""" + """ Engine is initialized, replacement revision failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING = ( From 4c3f9c45875516aec7e2c287c2066883d02b8c15 Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 13:26:00 -0800 Subject: [PATCH 09/12] Added cross references --- src/firebolt/async_db/cursor.py | 2 +- src/firebolt/model/engine.py | 4 ++-- src/firebolt/model/engine_revision.py | 2 +- src/firebolt/service/database.py | 2 +- src/firebolt/service/engine_revision.py | 6 ++++-- src/firebolt/service/instance_type.py | 4 ++-- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index 0b852544a41..564df158b85 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -362,7 +362,7 @@ class Cursor(BaseCursor): description: information about a single result row rowcount: the number of rows produced by last query closed: True if connection is closed, False otherwise - arraysize: Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method + arraysize: Read/Write, specifies the number of rows to fetch at a time with the :py:func:`fetchmany` method """ __slots__ = BaseCursor.__slots__ + ("_async_query_lock",) diff --git a/src/firebolt/model/engine.py b/src/firebolt/model/engine.py index 8b34610c4ed..07233a2bd71 100644 --- a/src/firebolt/model/engine.py +++ b/src/firebolt/model/engine.py @@ -42,7 +42,7 @@ class EngineSettings(FireboltBaseModel): """ Engine Settings. - See Also: ``engine_revision.Specification`` which also contains engine configuration. + See Also: :py:class:`firebolt.model.engine_revision.EngineRevisionSpecification ` which also contains engine configuration. """ preset: str @@ -90,7 +90,7 @@ class Engine(FireboltBaseModel): """ A Firebolt engine. Responsible for performing work (queries, data ingestion). - Engines are configured in Settings and in EngineRevisions. + Engines are configured in :py:class:`firebolt.model.engine.EngineSettings ` and in :py:class:`firebolt.model.engine_revision.EngineRevisionSpecification `. """ # internal diff --git a/src/firebolt/model/engine_revision.py b/src/firebolt/model/engine_revision.py index 6404a6acb85..e3370893a83 100644 --- a/src/firebolt/model/engine_revision.py +++ b/src/firebolt/model/engine_revision.py @@ -21,7 +21,7 @@ class EngineRevisionSpecification(FireboltBaseModel): Notably, it determines which instance types and how many of them its Engine gets. - See Also: ``engine.Settings``, which also contains engine configuration. + See Also: :py:class:`firebolt.model.engine.EngineSettings`, which also contains engine configuration. """ db_compute_instances_type_key: InstanceTypeKey = Field( diff --git a/src/firebolt/service/database.py b/src/firebolt/service/database.py index cde6cecc3e1..5a386741051 100644 --- a/src/firebolt/service/database.py +++ b/src/firebolt/service/database.py @@ -53,7 +53,7 @@ def get_many( attached_engine_name_eq: Filter for databases by an exact engine name. attached_engine_name_contains: Filter for databases by engines with a name containing this substring. - order_by: Method by which to order the results. See [DatabaseOrder]. + order_by: Method by which to order the results. See :py:class:`firebolt.service.types.DatabaseOrder`. Returns: A list of databases matching the filters. diff --git a/src/firebolt/service/engine_revision.py b/src/firebolt/service/engine_revision.py index 18cc6efef85..8ad733a3d9c 100644 --- a/src/firebolt/service/engine_revision.py +++ b/src/firebolt/service/engine_revision.py @@ -5,7 +5,9 @@ class EngineRevisionService(BaseService): def get_by_id(self, engine_id: str, engine_revision_id: str) -> EngineRevision: - """Get an EngineRevision from Firebolt by engine_id and engine_revision_id.""" + """ + Get an EngineRevision from Firebolt by engine_id and engine_revision_id. + """ return self.get_by_key( EngineRevisionKey( account_id=self.account_id, @@ -16,7 +18,7 @@ def get_by_id(self, engine_id: str, engine_revision_id: str) -> EngineRevision: def get_by_key(self, key: EngineRevisionKey) -> EngineRevision: """ - Fetch an EngineRevision from Firebolt by it's key. + Fetch an EngineRevision from Firebolt by its key. Args: key: Key of the desired EngineRevision. diff --git a/src/firebolt/service/instance_type.py b/src/firebolt/service/instance_type.py index 96841c92a92..32e2ff171fc 100644 --- a/src/firebolt/service/instance_type.py +++ b/src/firebolt/service/instance_type.py @@ -22,12 +22,12 @@ def instance_types(self) -> List[InstanceType]: @cached_property def instance_types_by_key(self) -> Dict[InstanceTypeKey, InstanceType]: - """Dict of {InstanceTypeKey: InstanceType}""" + """Dict of {InstanceTypeKey to InstanceType}""" return {i.key: i for i in self.instance_types} @cached_property def instance_types_by_name(self) -> Dict[InstanceTypeLookup, InstanceType]: - """Dict of {InstanceTypeLookup: InstanceType}""" + """Dict of {InstanceTypeLookup to InstanceType}""" return { InstanceTypeLookup( region_name=self.resource_manager.regions.get_by_id( From 7388e3de57820ff05a5910da70f31048548d586b Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 14:57:02 -0800 Subject: [PATCH 10/12] Finished cross-references and other cleanup --- src/firebolt/async_db/connection.py | 13 ++++++- src/firebolt/async_db/cursor.py | 5 ++- src/firebolt/client/auth.py | 5 ++- src/firebolt/client/client.py | 1 + src/firebolt/client/resource_manager_hooks.py | 1 + src/firebolt/common/util.py | 1 + src/firebolt/db/cursor.py | 4 +- src/firebolt/model/database.py | 3 ++ src/firebolt/model/engine.py | 4 +- src/firebolt/model/engine_revision.py | 2 +- src/firebolt/service/binding.py | 3 ++ src/firebolt/service/database.py | 4 ++ src/firebolt/service/engine.py | 18 ++++++--- src/firebolt/service/engine_revision.py | 2 + src/firebolt/service/instance_type.py | 5 +++ src/firebolt/service/region.py | 7 ++++ src/firebolt/service/types.py | 39 +++---------------- 17 files changed, 69 insertions(+), 48 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index 34ac264f392..6bb919c23f8 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -156,7 +156,10 @@ def __init__( self._is_closed = False def cursor(self) -> BaseCursor: - """Create new cursor object.""" + """ + Create new cursor object. + """ + if self.closed: raise ConnectionClosedError("Unable to create cursor: connection closed") @@ -194,13 +197,14 @@ def _remove_cursor(self, cursor: Cursor) -> None: def commit(self) -> None: """Does nothing since Firebolt doesn't have transactions""" + if self.closed: raise ConnectionClosedError("Unable to commit: connection closed") class Connection(BaseConnection): """ - Firebolt asyncronous database connection class. Implements PEP-249. + Firebolt asyncronous database connection class. Implements `PEP 249`_. Args: engine_url: Firebolt database engine REST API url @@ -211,7 +215,12 @@ class Connection(BaseConnection): Note: Firebolt currenly doesn't support transactions so commit and rollback methods are not implemented. + + .. _PEP 249: + https://www.python.org/dev/peps/pep-0249/ + """ + cursor_class = Cursor aclose = BaseConnection._aclose diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index 564df158b85..99dfc107d70 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -155,16 +155,19 @@ def arraysize(self, value: int) -> None: @property def closed(self) -> bool: """True if connection is closed, False otherwise.""" + return self._state == CursorState.CLOSED def close(self) -> None: """Terminate an ongoing query (if any) and mark connection as closed.""" + self._state = CursorState.CLOSED # remove typecheck skip after connection is implemented self.connection._remove_cursor(self) # type: ignore def _store_query_data(self, response: Response) -> None: """Store information about executed query from httpx response.""" + # Empty response is returned for insert query if response.headers.get("content-length", "") == "0": return @@ -356,7 +359,7 @@ def __exit__( class Cursor(BaseCursor): """ Class, responsible for executing asyncio queries to Firebolt Database. - Should not be created directly, use ``connection.cursor()`` + Should not be created directly, use :py:func:`connection.cursor ` Args: description: information about a single result row diff --git a/src/firebolt/client/auth.py b/src/firebolt/client/auth.py index 832aecb97bb..2dc212e663f 100644 --- a/src/firebolt/client/auth.py +++ b/src/firebolt/client/auth.py @@ -17,6 +17,7 @@ class Auth(HttpxAuth): Authentication class for Firebolt database. Gets authentication token using provided credentials and updates it when it expires """ + __slots__ = ( "username", "password", @@ -50,6 +51,7 @@ def expired(self) -> Optional[int]: def get_new_token_generator(self) -> Generator[Request, Response, None]: """Get new token using username and password""" + try: response = yield Request( "POST", @@ -71,7 +73,8 @@ def get_new_token_generator(self) -> Generator[Request, Response, None]: raise AuthenticationError(repr(e), self._api_endpoint) def auth_flow(self, request: Request) -> Generator[Request, Response, None]: - """Add authorization token to request headers. Overrides httpx.Auth.auth_flow""" + """Add authorization token to request headers. Overrides ``httpx.Auth.auth_flow``""" + if not self.token or self.expired: yield from self.get_new_token_generator() request.headers["Authorization"] = f"Bearer {self.token}" diff --git a/src/firebolt/client/client.py b/src/firebolt/client/client.py index 0545ada089f..563c939389a 100644 --- a/src/firebolt/client/client.py +++ b/src/firebolt/client/client.py @@ -70,6 +70,7 @@ class AsyncClient(FireboltClientMixin, HttpxAsyncClient): + (HttpxAsyncClient.__doc__ or "") """ + @async_cached_property async def account_id(self) -> str: return (await self.get(url=ACCOUNT_URL)).json()["account"]["id"] diff --git a/src/firebolt/client/resource_manager_hooks.py b/src/firebolt/client/resource_manager_hooks.py index f2afdde336e..3d09ea2b87a 100644 --- a/src/firebolt/client/resource_manager_hooks.py +++ b/src/firebolt/client/resource_manager_hooks.py @@ -28,6 +28,7 @@ def raise_on_4xx_5xx(response: Response) -> None: If an error is message is found raise as an ApiError """ + try: response.raise_for_status() except RequestError as exc: diff --git a/src/firebolt/common/util.py b/src/firebolt/common/util.py index 102193395b4..1789e70abb0 100644 --- a/src/firebolt/common/util.py +++ b/src/firebolt/common/util.py @@ -27,6 +27,7 @@ class ReadonlyMixin(mixin_for(BaseClass))): ... ``` """ + if TYPE_CHECKING: return baseclass return object diff --git a/src/firebolt/db/cursor.py b/src/firebolt/db/cursor.py index 9d37b2a1d53..05d2dbc9371 100644 --- a/src/firebolt/db/cursor.py +++ b/src/firebolt/db/cursor.py @@ -20,13 +20,13 @@ class Cursor(AsyncBaseCursor): """ Class, responsible for executing queries to Firebolt Database. - Should not be created directly, use ``connection.cursor()`` + Should not be created directly, use :py:func:`connection.cursor ` Args: description: Information about a single result row rowcount: The number of rows produced by last query closed: True if connection is closed, False otherwise - arraysize: Read/Write, specifies the number of rows to fetch at a time with ``.fetchmany`` method + arraysize: Read/Write, specifies the number of rows to fetch at a time with :py:func:`fetchmany` method """ __slots__ = AsyncBaseCursor.__slots__ + ("_query_lock", "_idx_lock") diff --git a/src/firebolt/model/database.py b/src/firebolt/model/database.py index 77eb2269bf7..b1ef8fa14b1 100644 --- a/src/firebolt/model/database.py +++ b/src/firebolt/model/database.py @@ -72,6 +72,7 @@ def database_id(self) -> Optional[str]: def get_attached_engines(self) -> List[Engine]: """Get a list of engines that are attached to this database.""" + return self._service.resource_manager.bindings.get_engines_bound_to_database( # noqa: E501 database=self ) @@ -89,6 +90,7 @@ def attach_to_engine( Only one engine can be set as default for a single database. This will overwrite any existing default. """ + return self._service.resource_manager.bindings.create( engine=engine, database=self, is_default_engine=is_default_engine ) @@ -99,6 +101,7 @@ def delete(self) -> Database: Raises an error if there are any attached engines. """ + for engine in self.get_attached_engines(): if engine.current_status_summary in { EngineStatusSummary.ENGINE_STATUS_SUMMARY_STARTING, diff --git a/src/firebolt/model/engine.py b/src/firebolt/model/engine.py index 07233a2bd71..b5fbad446a2 100644 --- a/src/firebolt/model/engine.py +++ b/src/firebolt/model/engine.py @@ -42,7 +42,7 @@ class EngineSettings(FireboltBaseModel): """ Engine Settings. - See Also: :py:class:`firebolt.model.engine_revision.EngineRevisionSpecification ` which also contains engine configuration. + See Also: :py:class:`EngineRevisionSpecification ` which also contains engine configuration. """ preset: str @@ -90,7 +90,7 @@ class Engine(FireboltBaseModel): """ A Firebolt engine. Responsible for performing work (queries, data ingestion). - Engines are configured in :py:class:`firebolt.model.engine.EngineSettings ` and in :py:class:`firebolt.model.engine_revision.EngineRevisionSpecification `. + Engines are configured in :py:class:`Settings ` and in :py:class:`EngineRevisionSpecification `. """ # internal diff --git a/src/firebolt/model/engine_revision.py b/src/firebolt/model/engine_revision.py index e3370893a83..ec2a4ae5aca 100644 --- a/src/firebolt/model/engine_revision.py +++ b/src/firebolt/model/engine_revision.py @@ -21,7 +21,7 @@ class EngineRevisionSpecification(FireboltBaseModel): Notably, it determines which instance types and how many of them its Engine gets. - See Also: :py:class:`firebolt.model.engine.EngineSettings`, which also contains engine configuration. + See Also: :py:class:`Settings `, which also contains engine configuration. """ db_compute_instances_type_key: InstanceTypeKey = Field( diff --git a/src/firebolt/service/binding.py b/src/firebolt/service/binding.py index 8a7122988dd..43652fc8238 100644 --- a/src/firebolt/service/binding.py +++ b/src/firebolt/service/binding.py @@ -52,6 +52,7 @@ def get_many( Returns: List of bindings matching the filter parameters. """ + response = self.client.get( url=ACCOUNT_BINDINGS_URL.format(account_id=self.account_id), params=prune_dict( @@ -78,6 +79,7 @@ def get_database_bound_to_engine(self, engine: Engine) -> Optional[Database]: def get_engines_bound_to_database(self, database: Database) -> List[Engine]: """Get a list of engines that are bound to a database.""" + bindings = self.get_many(database_id=database.database_id) return self.resource_manager.engines.get_by_ids( ids=[b.engine_id for b in bindings] @@ -100,6 +102,7 @@ def create( Returns: New binding between the engine and database. """ + existing_database = self.get_database_bound_to_engine(engine=engine) if existing_database is not None: raise AlreadyBoundError( diff --git a/src/firebolt/service/database.py b/src/firebolt/service/database.py index 5a386741051..87af9c973e1 100644 --- a/src/firebolt/service/database.py +++ b/src/firebolt/service/database.py @@ -17,6 +17,7 @@ class DatabaseService(BaseService): def get(self, id_: str) -> Database: """Get a Database from Firebolt by its id.""" + response = self.client.get( url=ACCOUNT_DATABASE_URL.format(account_id=self.account_id, database_id=id_) ) @@ -26,11 +27,13 @@ def get(self, id_: str) -> Database: def get_by_name(self, name: str) -> Database: """Get a Database from Firebolt by its name.""" + database_id = self.get_id_by_name(name=name) return self.get(id_=database_id) def get_id_by_name(self, name: str) -> str: """Get a Database id from Firebolt by its name.""" + response = self.client.get( url=ACCOUNT_DATABASE_BY_NAME_URL.format(account_id=self.account_id), params={"database_name": name}, @@ -58,6 +61,7 @@ def get_many( Returns: A list of databases matching the filters. """ + if isinstance(order_by, str): order_by = DatabaseOrder[order_by] response = self.client.get( diff --git a/src/firebolt/service/engine.py b/src/firebolt/service/engine.py index 131bd1bb594..5676e10891c 100644 --- a/src/firebolt/service/engine.py +++ b/src/firebolt/service/engine.py @@ -24,6 +24,7 @@ class EngineService(BaseService): def get(self, id_: str) -> Engine: """Get an Engine from Firebolt by its id.""" + response = self.client.get( url=ACCOUNT_ENGINE_URL.format(account_id=self.account_id, engine_id=id_), ) @@ -48,6 +49,7 @@ def get_by_ids(self, ids: List[str]) -> List[Engine]: def get_by_name(self, name: str) -> Engine: """Get an Engine from Firebolt by its name.""" + response = self.client.get( url=ACCOUNT_ENGINE_BY_NAME_URL.format(account_id=self.account_id), params={"engine_name": name}, @@ -76,6 +78,7 @@ def get_many( Returns: A list of engines matching the filters. """ + if isinstance(order_by, str): order_by = EngineOrder[order_by] response = self.client.get( @@ -119,18 +122,21 @@ def create( scale: The number of compute instances on the engine. The scale can be any int from 1 to 128. spec: The AWS EC2 instance type. - auto_stop: The amount of time (in minutes) after which - the engine automatically stops. + auto_stop: The amount of time (in minutes) after which the engine automatically stops. warmup: The warmup method that should be used. - MINIMAL: On-demand loading (both indexes and tables' data). - PRELOAD_INDEXES: Load indexes only. - PRELOAD_ALL_DATA: Full data auto-load - (both indexes and table data - full warmup). + + `MINIMAL` - On-demand loading (both indexes and tables' data). + + `PRELOAD_INDEXES` - Load indexes only. + + `PRELOAD_ALL_DATA` - Full data auto-load + (both indexes and table data - full warmup). description: A short description of the engine's purpose. Returns: Engine with the specified settings. """ + logger.info(f"Creating Engine (name={name})") if isinstance(engine_type, str): diff --git a/src/firebolt/service/engine_revision.py b/src/firebolt/service/engine_revision.py index 8ad733a3d9c..d8c860ff043 100644 --- a/src/firebolt/service/engine_revision.py +++ b/src/firebolt/service/engine_revision.py @@ -8,6 +8,7 @@ def get_by_id(self, engine_id: str, engine_revision_id: str) -> EngineRevision: """ Get an EngineRevision from Firebolt by engine_id and engine_revision_id. """ + return self.get_by_key( EngineRevisionKey( account_id=self.account_id, @@ -26,6 +27,7 @@ def get_by_key(self, key: EngineRevisionKey) -> EngineRevision: Returns: The requested EngineRevision """ + response = self.client.get( url=ACCOUNT_ENGINE_REVISION_URL.format( account_id=key.account_id, diff --git a/src/firebolt/service/instance_type.py b/src/firebolt/service/instance_type.py index 32e2ff171fc..8ce68f0bdf1 100644 --- a/src/firebolt/service/instance_type.py +++ b/src/firebolt/service/instance_type.py @@ -17,17 +17,20 @@ class InstanceTypeService(BaseService): @cached_property def instance_types(self) -> List[InstanceType]: """List of instance types available on Firebolt.""" + response = self.client.get(url=INSTANCE_TYPES_URL, params={"page.first": 5000}) return [InstanceType.parse_obj(i["node"]) for i in response.json()["edges"]] @cached_property def instance_types_by_key(self) -> Dict[InstanceTypeKey, InstanceType]: """Dict of {InstanceTypeKey to InstanceType}""" + return {i.key: i for i in self.instance_types} @cached_property def instance_types_by_name(self) -> Dict[InstanceTypeLookup, InstanceType]: """Dict of {InstanceTypeLookup to InstanceType}""" + return { InstanceTypeLookup( region_name=self.resource_manager.regions.get_by_id( @@ -40,6 +43,7 @@ def instance_types_by_name(self) -> Dict[InstanceTypeLookup, InstanceType]: def get_by_key(self, instance_type_key: InstanceTypeKey) -> InstanceType: """Get an instance type by key.""" + return self.instance_types_by_key[instance_type_key] def get_by_name( @@ -59,6 +63,7 @@ def get_by_name( Returns: The requested instance type. """ + # Will raise an error if neither set region_name = region_name or self.resource_manager.regions.default_region.name return self.instance_types_by_name[ diff --git a/src/firebolt/service/region.py b/src/firebolt/service/region.py index 8571f4e7d88..748295ad65a 100644 --- a/src/firebolt/service/region.py +++ b/src/firebolt/service/region.py @@ -15,22 +15,26 @@ def __init__(self, resource_manager: ResourceManager): Args: resource_manager: Resource manager to use. """ + super().__init__(resource_manager=resource_manager) @cached_property def regions(self) -> List[Region]: """List of available AWS Regions on Firebolt.""" + response = self.client.get(url=REGIONS_URL, params={"page.first": 5000}) return [Region.parse_obj(i["node"]) for i in response.json()["edges"]] @cached_property def regions_by_name(self) -> Dict[str, Region]: """Dict of {RegionLookup to Region}""" + return {r.name: r for r in self.regions} @cached_property def regions_by_key(self) -> Dict[RegionKey, Region]: """Dict of {RegionKey to Region}""" + return {r.key: r for r in self.regions} @cached_property @@ -45,14 +49,17 @@ def default_region(self) -> Region: def get_by_name(self, name: str) -> Region: """Get an AWS region by its name (eg. us-east-1).""" + return self.regions_by_name[name] def get_by_key(self, key: RegionKey) -> Region: """Get an AWS Region by its key.""" + return self.regions_by_key[key] def get_by_id(self, id_: str) -> Region: """Get an AWS Region by region_id.""" + return self.get_by_key( RegionKey(provider_id=self.resource_manager.provider_id, region_id=id_) ) diff --git a/src/firebolt/service/types.py b/src/firebolt/service/types.py index e9a387d2622..8b599399287 100644 --- a/src/firebolt/service/types.py +++ b/src/firebolt/service/types.py @@ -36,94 +36,79 @@ class EngineStatus(Enum): """ ENGINE_STATUS_UNSPECIFIED = "ENGINE_STATUS_UNSPECIFIED" - """ Logical record is created, however underlying infrastructure is not initialized. In other words this means that engine is stopped.""" - ENGINE_STATUS_CREATED = "ENGINE_STATUS_CREATED" + ENGINE_STATUS_CREATED = "ENGINE_STATUS_CREATED" """Engine status was created.""" - ENGINE_STATUS_PROVISIONING_PENDING = "ENGINE_STATUS_PROVISIONING_PENDING" + ENGINE_STATUS_PROVISIONING_PENDING = "ENGINE_STATUS_PROVISIONING_PENDING" """ Engine initialization request was sent.""" - ENGINE_STATUS_PROVISIONING_STARTED = "ENGINE_STATUS_PROVISIONING_STARTED" + ENGINE_STATUS_PROVISIONING_STARTED = "ENGINE_STATUS_PROVISIONING_STARTED" """ Engine initialization request was received and initialization process started.""" - ENGINE_STATUS_PROVISIONING_FINISHED = "ENGINE_STATUS_PROVISIONING_FINISHED" + ENGINE_STATUS_PROVISIONING_FINISHED = "ENGINE_STATUS_PROVISIONING_FINISHED" """ Engine initialization was finished successfully.""" - ENGINE_STATUS_PROVISIONING_FAILED = "ENGINE_STATUS_PROVISIONING_FAILED" + ENGINE_STATUS_PROVISIONING_FAILED = "ENGINE_STATUS_PROVISIONING_FAILED" """ Engine initialization failed due to error.""" ENGINE_STATUS_RUNNING_IDLE = "ENGINE_STATUS_RUNNING_IDLE" - """ Engine is initialized, but there are no running or starting engine revisions.""" ENGINE_STATUS_RUNNING_REVISION_STARTING = "ENGINE_STATUS_RUNNING_REVISION_STARTING" - """ Engine is initialized, there are no running engine revision but it's starting.""" ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED" ) - """ Engine is initialized, initial revision is failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISION_SERVING = "ENGINE_STATUS_RUNNING_REVISION_SERVING" - """ Engine is ready (serves an engine revision). """ ENGINE_STATUS_RUNNING_REVISION_CHANGING = "ENGINE_STATUS_RUNNING_REVISION_CHANGING" - """ Engine is ready (serves an engine revision), zero-downtime replacement revision is starting.""" ENGINE_STATUS_RUNNING_REVISION_CHANGE_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_CHANGE_FAILED" ) - """ Engine is ready (serves an engine revision), replacement revision failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISION_RESTARTING = ( "ENGINE_STATUS_RUNNING_REVISION_RESTARTING" ) - """ Engine is initialized, replacement of the revision with a downtime is in progress.""" ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED" ) - """ Engine is initialized, replacement revision failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING = ( "ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING" ) - """ Engine is initialized, all child revisions are being terminated.""" # Engine termination request was sent. ENGINE_STATUS_TERMINATION_PENDING = "ENGINE_STATUS_TERMINATION_PENDING" - """ Engine termination request was sent.""" ENGINE_STATUS_TERMINATION_ST = "ENGINE_STATUS_TERMINATION_STARTED" - """ Engine termination started.""" ENGINE_STATUS_TERMINATION_FIN = "ENGINE_STATUS_TERMINATION_FINISHED" - """ Engine termination finished.""" ENGINE_STATUS_TERMINATION_F = "ENGINE_STATUS_TERMINATION_FAILED" - """ Engine termination failed.""" ENGINE_STATUS_DELETED = "ENGINE_STATUS_DELETED" - """ Engine is soft-deleted.""" class EngineStatusSummary(Enum): @@ -134,44 +119,37 @@ class EngineStatusSummary(Enum): """ ENGINE_STATUS_SUMMARY_UNSPECIFIED = "ENGINE_STATUS_SUMMARY_UNSPECIFIED" - """Status unspecified""" - ENGINE_STATUS_SUMMARY_STOPPED = "ENGINE_STATUS_SUMMARY_STOPPED" + ENGINE_STATUS_SUMMARY_STOPPED = "ENGINE_STATUS_SUMMARY_STOPPED" """ Fully stopped.""" ENGINE_STATUS_SUMMARY_STARTING = "ENGINE_STATUS_SUMMARY_STARTING" - """ Provisioning process is in progress. We are creating cloud infra for this engine.""" ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING = ( "ENGINE_STATUS_SUMMARY_STARTING_INITIALIZING" ) - """ Provisioning process is complete. We are now waiting for PackDB cluster to initialize and start.""" ENGINE_STATUS_SUMMARY_RUNNING = "ENGINE_STATUS_SUMMARY_RUNNING" - """ Fully started. Engine is ready to serve requests.""" ENGINE_STATUS_SUMMARY_UPGRADING = "ENGINE_STATUS_SUMMARY_UPGRADING" - """ Version of the PackDB is changing. This is zero downtime operation that does not affect engine work.s This status is reserved for future use (not used fow now).""" ENGINE_STATUS_SUMMARY_RESTARTING = "ENGINE_STATUS_SUMMARY_RESTARTING" - """ Hard restart (full stop/start cycle) is in progress. Underlying infrastructure is being recreated.""" ENGINE_STATUS_SUMMARY_RESTARTING_INITIALIZING = ( "ENGINE_STATUS_SUMMARY_RESTARTING_INITIALIZING" ) - """ Hard restart (full stop/start cycle) is in progress. Underlying infrastructure is ready, waiting for PackDB cluster to initialize and start. @@ -179,25 +157,20 @@ class EngineStatusSummary(Enum): but used during restart cycle.""" ENGINE_STATUS_SUMMARY_REPAIRING = "ENGINE_STATUS_SUMMARY_REPAIRING" - """ Underlying infrastructure has issues and is being repaired. Engine is still running, but it's not fully healthy and some queries may fail.""" ENGINE_STATUS_SUMMARY_STOPPING = "ENGINE_STATUS_SUMMARY_STOPPING" - """ Stop is in progress.""" ENGINE_STATUS_SUMMARY_DELETING = "ENGINE_STATUS_SUMMARY_DELETING" - """ Termination is in progress. All infrastructure that belongs to this engine will be completely destroyed.""" ENGINE_STATUS_SUMMARY_DELETED = "ENGINE_STATUS_SUMMARY_DELETED" - """ Infrastructure is terminated, engine data is deleted.""" ENGINE_STATUS_SUMMARY_FAILED = "ENGINE_STATUS_SUMMARY_FAILED" - """ Failed to start or stop. This status only indicates that there were issues during provisioning operations. If engine enters this status, From f79d75389706d196bece1166bc086c4a88e0d13b Mon Sep 17 00:00:00 2001 From: marknoack Date: Thu, 16 Dec 2021 15:29:26 -0800 Subject: [PATCH 11/12] docstring line wrapping --- src/firebolt/async_db/connection.py | 3 ++- src/firebolt/async_db/cursor.py | 7 +++++-- src/firebolt/client/auth.py | 3 ++- src/firebolt/db/cursor.py | 7 +++++-- src/firebolt/model/engine.py | 9 +++++++-- src/firebolt/model/engine_revision.py | 7 +++++-- src/firebolt/service/database.py | 3 ++- src/firebolt/service/engine.py | 3 ++- src/firebolt/service/types.py | 24 ++++++++++++++++-------- 9 files changed, 46 insertions(+), 20 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index 6bb919c23f8..0fbc4e81522 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -214,7 +214,8 @@ class Connection(BaseConnection): api_endpoint: Optional. Firebolt API endpoint. Used for authentication Note: - Firebolt currenly doesn't support transactions so commit and rollback methods are not implemented. + Firebolt currenly doesn't support transactions + so commit and rollback methods are not implemented. .. _PEP 249: https://www.python.org/dev/peps/pep-0249/ diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index 99dfc107d70..1ceb120b82f 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -359,13 +359,16 @@ def __exit__( class Cursor(BaseCursor): """ Class, responsible for executing asyncio queries to Firebolt Database. - Should not be created directly, use :py:func:`connection.cursor ` + Should not be created directly, + use :py:func:`connection.cursor ` Args: description: information about a single result row rowcount: the number of rows produced by last query closed: True if connection is closed, False otherwise - arraysize: Read/Write, specifies the number of rows to fetch at a time with the :py:func:`fetchmany` method + arraysize: Read/Write, specifies the number of rows to fetch at a time + with the :py:func:`fetchmany` method + """ __slots__ = BaseCursor.__slots__ + ("_async_query_lock",) diff --git a/src/firebolt/client/auth.py b/src/firebolt/client/auth.py index 2dc212e663f..b020b6c7918 100644 --- a/src/firebolt/client/auth.py +++ b/src/firebolt/client/auth.py @@ -73,7 +73,8 @@ def get_new_token_generator(self) -> Generator[Request, Response, None]: raise AuthenticationError(repr(e), self._api_endpoint) def auth_flow(self, request: Request) -> Generator[Request, Response, None]: - """Add authorization token to request headers. Overrides ``httpx.Auth.auth_flow``""" + """Add authorization token to request headers. + Overrides ``httpx.Auth.auth_flow``""" if not self.token or self.expired: yield from self.get_new_token_generator() diff --git a/src/firebolt/db/cursor.py b/src/firebolt/db/cursor.py index 05d2dbc9371..fe28450dd3c 100644 --- a/src/firebolt/db/cursor.py +++ b/src/firebolt/db/cursor.py @@ -20,13 +20,16 @@ class Cursor(AsyncBaseCursor): """ Class, responsible for executing queries to Firebolt Database. - Should not be created directly, use :py:func:`connection.cursor ` + Should not be created directly, + use :py:func:`connection.cursor ` Args: description: Information about a single result row rowcount: The number of rows produced by last query closed: True if connection is closed, False otherwise - arraysize: Read/Write, specifies the number of rows to fetch at a time with :py:func:`fetchmany` method + arraysize: Read/Write, + specifies the number of rows to fetch at a time + with :py:func:`fetchmany` method """ __slots__ = AsyncBaseCursor.__slots__ + ("_query_lock", "_idx_lock") diff --git a/src/firebolt/model/engine.py b/src/firebolt/model/engine.py index b5fbad446a2..efedacab2fc 100644 --- a/src/firebolt/model/engine.py +++ b/src/firebolt/model/engine.py @@ -42,7 +42,9 @@ class EngineSettings(FireboltBaseModel): """ Engine Settings. - See Also: :py:class:`EngineRevisionSpecification ` which also contains engine configuration. + See Also: :py:class:`EngineRevisionSpecification + ` + which also contains engine configuration. """ preset: str @@ -90,7 +92,10 @@ class Engine(FireboltBaseModel): """ A Firebolt engine. Responsible for performing work (queries, data ingestion). - Engines are configured in :py:class:`Settings ` and in :py:class:`EngineRevisionSpecification `. + Engines are configured in :py:class:`Settings + ` + and in :py:class:`EngineRevisionSpecification + `. """ # internal diff --git a/src/firebolt/model/engine_revision.py b/src/firebolt/model/engine_revision.py index ec2a4ae5aca..24c7333dc63 100644 --- a/src/firebolt/model/engine_revision.py +++ b/src/firebolt/model/engine_revision.py @@ -21,7 +21,9 @@ class EngineRevisionSpecification(FireboltBaseModel): Notably, it determines which instance types and how many of them its Engine gets. - See Also: :py:class:`Settings `, which also contains engine configuration. + See Also: :py:class:`Settings + `, + which also contains engine configuration. """ db_compute_instances_type_key: InstanceTypeKey = Field( @@ -37,7 +39,8 @@ class EngineRevisionSpecification(FireboltBaseModel): class EngineRevision(FireboltBaseModel): """ - A Firebolt Engine revision, which contains a Specification (instance types, counts). + A Firebolt Engine revision, + which contains a Specification (instance types, counts). As engines are updated with new settings, revisions are created. """ diff --git a/src/firebolt/service/database.py b/src/firebolt/service/database.py index 87af9c973e1..e2ffcc21d00 100644 --- a/src/firebolt/service/database.py +++ b/src/firebolt/service/database.py @@ -56,7 +56,8 @@ def get_many( attached_engine_name_eq: Filter for databases by an exact engine name. attached_engine_name_contains: Filter for databases by engines with a name containing this substring. - order_by: Method by which to order the results. See :py:class:`firebolt.service.types.DatabaseOrder`. + order_by: Method by which to order the results. + See :py:class:`firebolt.service.types.DatabaseOrder`. Returns: A list of databases matching the filters. diff --git a/src/firebolt/service/engine.py b/src/firebolt/service/engine.py index 5676e10891c..c7a16c88ae7 100644 --- a/src/firebolt/service/engine.py +++ b/src/firebolt/service/engine.py @@ -122,7 +122,8 @@ def create( scale: The number of compute instances on the engine. The scale can be any int from 1 to 128. spec: The AWS EC2 instance type. - auto_stop: The amount of time (in minutes) after which the engine automatically stops. + auto_stop: The amount of time (in minutes) + after which the engine automatically stops. warmup: The warmup method that should be used. `MINIMAL` - On-demand loading (both indexes and tables' data). diff --git a/src/firebolt/service/types.py b/src/firebolt/service/types.py index 8b599399287..07c63b0247e 100644 --- a/src/firebolt/service/types.py +++ b/src/firebolt/service/types.py @@ -36,7 +36,8 @@ class EngineStatus(Enum): """ ENGINE_STATUS_UNSPECIFIED = "ENGINE_STATUS_UNSPECIFIED" - """ Logical record is created, however underlying infrastructure is not initialized. + """ Logical record is created, however underlying infrastructure + is not initialized. In other words this means that engine is stopped.""" ENGINE_STATUS_CREATED = "ENGINE_STATUS_CREATED" @@ -46,7 +47,8 @@ class EngineStatus(Enum): """ Engine initialization request was sent.""" ENGINE_STATUS_PROVISIONING_STARTED = "ENGINE_STATUS_PROVISIONING_STARTED" - """ Engine initialization request was received and initialization process started.""" + """ Engine initialization request was received + and initialization process started.""" ENGINE_STATUS_PROVISIONING_FINISHED = "ENGINE_STATUS_PROVISIONING_FINISHED" """ Engine initialization was finished successfully.""" @@ -55,15 +57,18 @@ class EngineStatus(Enum): """ Engine initialization failed due to error.""" ENGINE_STATUS_RUNNING_IDLE = "ENGINE_STATUS_RUNNING_IDLE" - """ Engine is initialized, but there are no running or starting engine revisions.""" + """ Engine is initialized, + but there are no running or starting engine revisions.""" ENGINE_STATUS_RUNNING_REVISION_STARTING = "ENGINE_STATUS_RUNNING_REVISION_STARTING" - """ Engine is initialized, there are no running engine revision but it's starting.""" + """ Engine is initialized, + there are no running engine revision but it's starting.""" ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED" ) - """ Engine is initialized, initial revision is failed to provision or start.""" + """ Engine is initialized, + initial revision is failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISION_SERVING = "ENGINE_STATUS_RUNNING_REVISION_SERVING" """ Engine is ready (serves an engine revision). """ @@ -81,18 +86,21 @@ class EngineStatus(Enum): ENGINE_STATUS_RUNNING_REVISION_RESTARTING = ( "ENGINE_STATUS_RUNNING_REVISION_RESTARTING" ) - """ Engine is initialized, replacement of the revision with a downtime is in progress.""" + """ Engine is initialized, + replacement of the revision with a downtime is in progress.""" ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED" ) - """ Engine is initialized, replacement revision failed to provision or start.""" + """ Engine is initialized, + replacement revision failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING = ( "ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING" ) - """ Engine is initialized, all child revisions are being terminated.""" + """ Engine is initialized, + all child revisions are being terminated.""" # Engine termination request was sent. From 8e64734adab0b4417c27759f7f03ac441775fda1 Mon Sep 17 00:00:00 2001 From: Eric Gustavson Date: Fri, 17 Dec 2021 09:55:39 -0800 Subject: [PATCH 12/12] fix pre-commit formatting issues --- src/firebolt/async_db/connection.py | 37 ++++++++------- src/firebolt/async_db/cursor.py | 31 +++++++------ src/firebolt/client/auth.py | 7 ++- src/firebolt/client/client.py | 29 ++++++------ src/firebolt/client/resource_manager_hooks.py | 2 +- src/firebolt/common/exception.py | 40 ++++++++-------- src/firebolt/common/util.py | 2 +- src/firebolt/db/connection.py | 27 ++++++----- src/firebolt/db/cursor.py | 21 ++++----- src/firebolt/model/database.py | 2 +- src/firebolt/model/engine.py | 10 ++-- src/firebolt/model/engine_revision.py | 6 +-- src/firebolt/service/binding.py | 2 +- src/firebolt/service/database.py | 4 +- src/firebolt/service/engine.py | 46 +++++++++---------- src/firebolt/service/engine_revision.py | 4 +- src/firebolt/service/instance_type.py | 2 +- src/firebolt/service/manager.py | 2 +- src/firebolt/service/region.py | 2 +- src/firebolt/service/types.py | 24 +++++----- 20 files changed, 146 insertions(+), 154 deletions(-) diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index 0fbc4e81522..a96d8783c18 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -1,6 +1,5 @@ from __future__ import annotations -from inspect import cleandoc from json import JSONDecodeError from types import TracebackType from typing import Callable, List, Optional, Type @@ -67,14 +66,14 @@ async def connect_inner( """ Connect to Firebolt database. - Args: + Args: database: name of the database to connect username: user name to use for authentication password: password to use for authentication engine_name: Optional The name of the engine to connect to engine_url: Optional. The engine endpoint to use - - Note: + + Note: either `engine_name` or `engine_url` should be provided, but not both """ @@ -157,7 +156,7 @@ def __init__( def cursor(self) -> BaseCursor: """ - Create new cursor object. + Create new cursor object. """ if self.closed: @@ -204,21 +203,21 @@ def commit(self) -> None: class Connection(BaseConnection): """ - Firebolt asyncronous database connection class. Implements `PEP 249`_. + Firebolt asyncronous database connection class. Implements `PEP 249`_. - Args: - engine_url: Firebolt database engine REST API url - database: Firebolt database name - username: Firebolt account username - password: Firebolt account password - api_endpoint: Optional. Firebolt API endpoint. Used for authentication - - Note: - Firebolt currenly doesn't support transactions - so commit and rollback methods are not implemented. - - .. _PEP 249: - https://www.python.org/dev/peps/pep-0249/ + Args: + engine_url: Firebolt database engine REST API url + database: Firebolt database name + username: Firebolt account username + password: Firebolt account password + api_endpoint: Optional. Firebolt API endpoint. Used for authentication + + Note: + Firebolt currenly doesn't support transactions + so commit and rollback methods are not implemented. + + .. _PEP 249: + https://www.python.org/dev/peps/pep-0249/ """ diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index 1ceb120b82f..76a401d45e6 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -120,7 +120,7 @@ def __del__(self) -> None: def description(self) -> Optional[List[Column]]: """ Provides information about a single result row of a query - + Attributes: * ``name`` * ``type_code`` @@ -167,7 +167,7 @@ def close(self) -> None: def _store_query_data(self, response: Response) -> None: """Store information about executed query from httpx response.""" - + # Empty response is returned for insert query if response.headers.get("content-length", "") == "0": return @@ -268,8 +268,8 @@ async def executemany( self, query: str, parameters_seq: Sequence[Sequence[ParameterType]] ) -> int: """ - Prepare and execute a database query against all parameter - sequences provided. Return last query row count. + Prepare and execute a database query against all parameter + sequences provided. Return last query row count. """ self._reset() resp = None @@ -319,8 +319,8 @@ def fetchone(self) -> Optional[List[ColType]]: @check_query_executed def fetchmany(self, size: Optional[int] = None) -> List[List[ColType]]: """ - Fetch the next set of rows of a query result, - cursor.arraysize is default size. + Fetch the next set of rows of a query result, + cursor.arraysize is default size. """ size = size if size is not None else self.arraysize left, right = self._get_next_range(size) @@ -358,16 +358,16 @@ def __exit__( class Cursor(BaseCursor): """ - Class, responsible for executing asyncio queries to Firebolt Database. - Should not be created directly, - use :py:func:`connection.cursor ` + Class, responsible for executing asyncio queries to Firebolt Database. + Should not be created directly, + use :py:func:`connection.cursor ` - Args: - description: information about a single result row - rowcount: the number of rows produced by last query - closed: True if connection is closed, False otherwise - arraysize: Read/Write, specifies the number of rows to fetch at a time - with the :py:func:`fetchmany` method + Args: + description: information about a single result row + rowcount: the number of rows produced by last query + closed: True if connection is closed, False otherwise + arraysize: Read/Write, specifies the number of rows to fetch at a time + with the :py:func:`fetchmany` method """ @@ -411,6 +411,7 @@ async def fetchmany(self, size: Optional[int] = None) -> List[List[ColType]]: return super().fetchmany(size) """fetch the next set of rows of a query result, size is cursor.arraysize by default""" + @wraps(BaseCursor.fetchall) async def fetchall(self) -> List[List[ColType]]: async with self._async_query_lock.reader: diff --git a/src/firebolt/client/auth.py b/src/firebolt/client/auth.py index b020b6c7918..7eaedfe6f9f 100644 --- a/src/firebolt/client/auth.py +++ b/src/firebolt/client/auth.py @@ -1,4 +1,3 @@ -from inspect import cleandoc from time import time from typing import Generator, Optional @@ -14,8 +13,8 @@ class Auth(HttpxAuth): """ - Authentication class for Firebolt database. Gets authentication token using - provided credentials and updates it when it expires + Authentication class for Firebolt database. Gets authentication token using + provided credentials and updates it when it expires """ __slots__ = ( @@ -73,7 +72,7 @@ def get_new_token_generator(self) -> Generator[Request, Response, None]: raise AuthenticationError(repr(e), self._api_endpoint) def auth_flow(self, request: Request) -> Generator[Request, Response, None]: - """Add authorization token to request headers. + """Add authorization token to request headers. Overrides ``httpx.Auth.auth_flow``""" if not self.token or self.expired: diff --git a/src/firebolt/client/client.py b/src/firebolt/client/client.py index 563c939389a..135d15b0f07 100644 --- a/src/firebolt/client/client.py +++ b/src/firebolt/client/client.py @@ -1,4 +1,3 @@ -from inspect import cleandoc from typing import Any, Optional from async_property import async_cached_property # type: ignore @@ -42,15 +41,15 @@ def _build_auth(self, auth: _types.AuthTypes) -> Optional[Auth]: class Client(FireboltClientMixin, HttpxClient): """ - An http client, based on httpx.Client, that handles the authentication - for Firebolt database. + An http client, based on httpx.Client, that handles the authentication + for Firebolt database. - Authentication can be passed through auth keyword as a tuple or as a - FireboltAuth instance + Authentication can be passed through auth keyword as a tuple or as a + FireboltAuth instance - httpx.Client: - - + (HttpxClient.__doc__ or "") + httpx.Client: + + + (HttpxClient.__doc__ or "") """ @cached_property @@ -60,17 +59,17 @@ def account_id(self) -> str: class AsyncClient(FireboltClientMixin, HttpxAsyncClient): """ - An http client, based on httpx.AsyncClient, that asyncronously handles - authentication for Firebolt database. + An http client, based on httpx.AsyncClient, that asyncronously handles + authentication for Firebolt database. - Authentication can be passed through auth keyword as a tuple or as a - FireboltAuth instance + Authentication can be passed through auth keyword as a tuple or as a + FireboltAuth instance - httpx.AsyncClient: + httpx.AsyncClient: - + (HttpxAsyncClient.__doc__ or "") + + (HttpxAsyncClient.__doc__ or "") """ - + @async_cached_property async def account_id(self) -> str: return (await self.get(url=ACCOUNT_URL)).json()["account"]["id"] diff --git a/src/firebolt/client/resource_manager_hooks.py b/src/firebolt/client/resource_manager_hooks.py index 3d09ea2b87a..1682bbb1fc7 100644 --- a/src/firebolt/client/resource_manager_hooks.py +++ b/src/firebolt/client/resource_manager_hooks.py @@ -28,7 +28,7 @@ def raise_on_4xx_5xx(response: Response) -> None: If an error is message is found raise as an ApiError """ - + try: response.raise_for_status() except RequestError as exc: diff --git a/src/firebolt/common/exception.py b/src/firebolt/common/exception.py index a3304467ad0..86bfa548181 100644 --- a/src/firebolt/common/exception.py +++ b/src/firebolt/common/exception.py @@ -1,6 +1,3 @@ -from inspect import cleandoc - - class FireboltError(Exception): pass @@ -93,20 +90,19 @@ def __str__(self) -> str: class Warning(Exception): """ - Exception raised for important warnings - like data truncations while inserting, etc. + Exception raised for important warnings + like data truncations while inserting, etc. """ - Error = FireboltError class InterfaceError(Error): """ - Exception raised for errors that are related to the database interface - rather than the database itself. + Exception raised for errors that are related to the database interface + rather than the database itself. """ @@ -116,43 +112,43 @@ class DatabaseError(Error): class DataError(DatabaseError): """ - Exception raised for errors that are due to problems with the processed data - like division by zero, numeric value out of range, etc. + Exception raised for errors that are due to problems with the processed data + like division by zero, numeric value out of range, etc. """ class OperationalError(DatabaseError): """ - Exception raised for errors that are related to the database's operation - and not necessarily under the control of the programmer, e.g. an unexpected - disconnect occurs, the data source name is not found, a transaction could not - be processed, a memory allocation error occurred during processing, etc. + Exception raised for errors that are related to the database's operation + and not necessarily under the control of the programmer, e.g. an unexpected + disconnect occurs, the data source name is not found, a transaction could not + be processed, a memory allocation error occurred during processing, etc. """ class IntegrityError(DatabaseError): """ - Exception raised when the relational integrity of the database is affected, - e.g. a foreign key check fails. + Exception raised when the relational integrity of the database is affected, + e.g. a foreign key check fails. """ class InternalError(DatabaseError): """ - Exception raised when the database encounters an internal error, - e.g. the cursor is not valid anymore, the transaction is out of sync, etc. + Exception raised when the database encounters an internal error, + e.g. the cursor is not valid anymore, the transaction is out of sync, etc. """ class ProgrammingError(DatabaseError): """ - Exception raised when the database encounters an internal error, - e.g. the cursor is not valid anymore, the transaction is out of sync, etc. + Exception raised when the database encounters an internal error, + e.g. the cursor is not valid anymore, the transaction is out of sync, etc. """ class NotSupportedError(DatabaseError): """ - Exception raised when the database encounters an internal error, - e.g. the cursor is not valid anymore, the transaction is out of sync, etc. + Exception raised when the database encounters an internal error, + e.g. the cursor is not valid anymore, the transaction is out of sync, etc. """ diff --git a/src/firebolt/common/util.py b/src/firebolt/common/util.py index 1789e70abb0..dfeb7313b40 100644 --- a/src/firebolt/common/util.py +++ b/src/firebolt/common/util.py @@ -27,7 +27,7 @@ class ReadonlyMixin(mixin_for(BaseClass))): ... ``` """ - + if TYPE_CHECKING: return baseclass return object diff --git a/src/firebolt/db/connection.py b/src/firebolt/db/connection.py index 7e666f52c87..ad447126fee 100644 --- a/src/firebolt/db/connection.py +++ b/src/firebolt/db/connection.py @@ -1,7 +1,6 @@ from __future__ import annotations from functools import wraps -from inspect import cleandoc from types import TracebackType from typing import Any from warnings import warn @@ -19,19 +18,19 @@ class Connection(AsyncBaseConnection): """ - Firebolt database connection class. Implements PEP-249. - - Args: - - engine_url: Firebolt database engine REST API url - database: Firebolt database name - username: Firebolt account username - password: Firebolt account password - api_endpoint: Optional. Firebolt API endpoint. Used for authentication - - Note: - Firebolt currenly doesn't support transactions so commit and rollback methods - are not implemented. + Firebolt database connection class. Implements PEP-249. + + Args: + + engine_url: Firebolt database engine REST API url + database: Firebolt database name + username: Firebolt account username + password: Firebolt account password + api_endpoint: Optional. Firebolt API endpoint. Used for authentication + + Note: + Firebolt currenly doesn't support transactions so commit and rollback methods + are not implemented. """ __slots__ = AsyncBaseConnection.__slots__ + ("_closing_lock",) diff --git a/src/firebolt/db/cursor.py b/src/firebolt/db/cursor.py index fe28450dd3c..e7086597856 100644 --- a/src/firebolt/db/cursor.py +++ b/src/firebolt/db/cursor.py @@ -1,7 +1,6 @@ from __future__ import annotations from functools import wraps -from inspect import cleandoc from threading import Lock from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple @@ -19,17 +18,17 @@ class Cursor(AsyncBaseCursor): """ - Class, responsible for executing queries to Firebolt Database. - Should not be created directly, - use :py:func:`connection.cursor ` + Class, responsible for executing queries to Firebolt Database. + Should not be created directly, + use :py:func:`connection.cursor ` - Args: - description: Information about a single result row - rowcount: The number of rows produced by last query - closed: True if connection is closed, False otherwise - arraysize: Read/Write, - specifies the number of rows to fetch at a time - with :py:func:`fetchmany` method + Args: + description: Information about a single result row + rowcount: The number of rows produced by last query + closed: True if connection is closed, False otherwise + arraysize: Read/Write, + specifies the number of rows to fetch at a time + with :py:func:`fetchmany` method """ __slots__ = AsyncBaseCursor.__slots__ + ("_query_lock", "_idx_lock") diff --git a/src/firebolt/model/database.py b/src/firebolt/model/database.py index b1ef8fa14b1..2f397ef65c5 100644 --- a/src/firebolt/model/database.py +++ b/src/firebolt/model/database.py @@ -101,7 +101,7 @@ def delete(self) -> Database: Raises an error if there are any attached engines. """ - + for engine in self.get_attached_engines(): if engine.current_status_summary in { EngineStatusSummary.ENGINE_STATUS_SUMMARY_STARTING, diff --git a/src/firebolt/model/engine.py b/src/firebolt/model/engine.py index efedacab2fc..bd17cedeef1 100644 --- a/src/firebolt/model/engine.py +++ b/src/firebolt/model/engine.py @@ -42,8 +42,8 @@ class EngineSettings(FireboltBaseModel): """ Engine Settings. - See Also: :py:class:`EngineRevisionSpecification - ` + See Also: :py:class:`EngineRevisionSpecification + ` which also contains engine configuration. """ @@ -92,9 +92,9 @@ class Engine(FireboltBaseModel): """ A Firebolt engine. Responsible for performing work (queries, data ingestion). - Engines are configured in :py:class:`Settings - ` - and in :py:class:`EngineRevisionSpecification + Engines are configured in :py:class:`Settings + ` + and in :py:class:`EngineRevisionSpecification `. """ diff --git a/src/firebolt/model/engine_revision.py b/src/firebolt/model/engine_revision.py index 24c7333dc63..640a470cc09 100644 --- a/src/firebolt/model/engine_revision.py +++ b/src/firebolt/model/engine_revision.py @@ -21,8 +21,8 @@ class EngineRevisionSpecification(FireboltBaseModel): Notably, it determines which instance types and how many of them its Engine gets. - See Also: :py:class:`Settings - `, + See Also: :py:class:`Settings + `, which also contains engine configuration. """ @@ -39,7 +39,7 @@ class EngineRevisionSpecification(FireboltBaseModel): class EngineRevision(FireboltBaseModel): """ - A Firebolt Engine revision, + A Firebolt Engine revision, which contains a Specification (instance types, counts). As engines are updated with new settings, revisions are created. diff --git a/src/firebolt/service/binding.py b/src/firebolt/service/binding.py index 43652fc8238..aefbcdf7e0f 100644 --- a/src/firebolt/service/binding.py +++ b/src/firebolt/service/binding.py @@ -102,7 +102,7 @@ def create( Returns: New binding between the engine and database. """ - + existing_database = self.get_database_bound_to_engine(engine=engine) if existing_database is not None: raise AlreadyBoundError( diff --git a/src/firebolt/service/database.py b/src/firebolt/service/database.py index e2ffcc21d00..451f93f3fc6 100644 --- a/src/firebolt/service/database.py +++ b/src/firebolt/service/database.py @@ -56,13 +56,13 @@ def get_many( attached_engine_name_eq: Filter for databases by an exact engine name. attached_engine_name_contains: Filter for databases by engines with a name containing this substring. - order_by: Method by which to order the results. + order_by: Method by which to order the results. See :py:class:`firebolt.service.types.DatabaseOrder`. Returns: A list of databases matching the filters. """ - + if isinstance(order_by, str): order_by = DatabaseOrder[order_by] response = self.client.get( diff --git a/src/firebolt/service/engine.py b/src/firebolt/service/engine.py index c7a16c88ae7..2c1b82ba096 100644 --- a/src/firebolt/service/engine.py +++ b/src/firebolt/service/engine.py @@ -113,29 +113,29 @@ def create( description: str = "", ) -> Engine: """ - Create a new Engine. - - Args: - name: An identifier that specifies the name of the engine. - region: The AWS region in which the engine runs. - engine_type: The engine type. GENERAL_PURPOSE or DATA_ANALYTICS - scale: The number of compute instances on the engine. - The scale can be any int from 1 to 128. - spec: The AWS EC2 instance type. - auto_stop: The amount of time (in minutes) - after which the engine automatically stops. - warmup: The warmup method that should be used. - - `MINIMAL` - On-demand loading (both indexes and tables' data). - - `PRELOAD_INDEXES` - Load indexes only. - - `PRELOAD_ALL_DATA` - Full data auto-load - (both indexes and table data - full warmup). - description: A short description of the engine's purpose. - - Returns: - Engine with the specified settings. + Create a new Engine. + + Args: + name: An identifier that specifies the name of the engine. + region: The AWS region in which the engine runs. + engine_type: The engine type. GENERAL_PURPOSE or DATA_ANALYTICS + scale: The number of compute instances on the engine. + The scale can be any int from 1 to 128. + spec: The AWS EC2 instance type. + auto_stop: The amount of time (in minutes) + after which the engine automatically stops. + warmup: The warmup method that should be used. + + `MINIMAL` - On-demand loading (both indexes and tables' data). + + `PRELOAD_INDEXES` - Load indexes only. + + `PRELOAD_ALL_DATA` - Full data auto-load + (both indexes and table data - full warmup). + description: A short description of the engine's purpose. + + Returns: + Engine with the specified settings. """ logger.info(f"Creating Engine (name={name})") diff --git a/src/firebolt/service/engine_revision.py b/src/firebolt/service/engine_revision.py index d8c860ff043..f8f76ce566e 100644 --- a/src/firebolt/service/engine_revision.py +++ b/src/firebolt/service/engine_revision.py @@ -6,9 +6,9 @@ class EngineRevisionService(BaseService): def get_by_id(self, engine_id: str, engine_revision_id: str) -> EngineRevision: """ - Get an EngineRevision from Firebolt by engine_id and engine_revision_id. + Get an EngineRevision from Firebolt by engine_id and engine_revision_id. """ - + return self.get_by_key( EngineRevisionKey( account_id=self.account_id, diff --git a/src/firebolt/service/instance_type.py b/src/firebolt/service/instance_type.py index 8ce68f0bdf1..ff50fb9673a 100644 --- a/src/firebolt/service/instance_type.py +++ b/src/firebolt/service/instance_type.py @@ -17,7 +17,7 @@ class InstanceTypeService(BaseService): @cached_property def instance_types(self) -> List[InstanceType]: """List of instance types available on Firebolt.""" - + response = self.client.get(url=INSTANCE_TYPES_URL, params={"page.first": 5000}) return [InstanceType.parse_obj(i["node"]) for i in response.json()["edges"]] diff --git a/src/firebolt/service/manager.py b/src/firebolt/service/manager.py index 948fd5678fd..3c4875532e4 100644 --- a/src/firebolt/service/manager.py +++ b/src/firebolt/service/manager.py @@ -22,7 +22,7 @@ class ResourceManager: Also provides listings of: - regions (AWS regions in which engines can run) - + - instance types (AWS instance types which engines can use) """ diff --git a/src/firebolt/service/region.py b/src/firebolt/service/region.py index 748295ad65a..32ef72a0faf 100644 --- a/src/firebolt/service/region.py +++ b/src/firebolt/service/region.py @@ -59,7 +59,7 @@ def get_by_key(self, key: RegionKey) -> Region: def get_by_id(self, id_: str) -> Region: """Get an AWS Region by region_id.""" - + return self.get_by_key( RegionKey(provider_id=self.resource_manager.provider_id, region_id=id_) ) diff --git a/src/firebolt/service/types.py b/src/firebolt/service/types.py index 07c63b0247e..b46f6104d49 100644 --- a/src/firebolt/service/types.py +++ b/src/firebolt/service/types.py @@ -36,7 +36,7 @@ class EngineStatus(Enum): """ ENGINE_STATUS_UNSPECIFIED = "ENGINE_STATUS_UNSPECIFIED" - """ Logical record is created, however underlying infrastructure + """ Logical record is created, however underlying infrastructure is not initialized. In other words this means that engine is stopped.""" @@ -47,7 +47,7 @@ class EngineStatus(Enum): """ Engine initialization request was sent.""" ENGINE_STATUS_PROVISIONING_STARTED = "ENGINE_STATUS_PROVISIONING_STARTED" - """ Engine initialization request was received + """ Engine initialization request was received and initialization process started.""" ENGINE_STATUS_PROVISIONING_FINISHED = "ENGINE_STATUS_PROVISIONING_FINISHED" @@ -57,17 +57,17 @@ class EngineStatus(Enum): """ Engine initialization failed due to error.""" ENGINE_STATUS_RUNNING_IDLE = "ENGINE_STATUS_RUNNING_IDLE" - """ Engine is initialized, + """ Engine is initialized, but there are no running or starting engine revisions.""" ENGINE_STATUS_RUNNING_REVISION_STARTING = "ENGINE_STATUS_RUNNING_REVISION_STARTING" - """ Engine is initialized, + """ Engine is initialized, there are no running engine revision but it's starting.""" ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_STARTUP_FAILED" ) - """ Engine is initialized, + """ Engine is initialized, initial revision is failed to provision or start.""" ENGINE_STATUS_RUNNING_REVISION_SERVING = "ENGINE_STATUS_RUNNING_REVISION_SERVING" @@ -86,23 +86,21 @@ class EngineStatus(Enum): ENGINE_STATUS_RUNNING_REVISION_RESTARTING = ( "ENGINE_STATUS_RUNNING_REVISION_RESTARTING" ) - """ Engine is initialized, + """ Engine is initialized, replacement of the revision with a downtime is in progress.""" ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED = ( "ENGINE_STATUS_RUNNING_REVISION_RESTART_FAILED" ) - """ Engine is initialized, + """ Engine is initialized, replacement revision failed to provision or start.""" - ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING = ( "ENGINE_STATUS_RUNNING_REVISIONS_TERMINATING" ) - """ Engine is initialized, + """ Engine is initialized, all child revisions are being terminated.""" - # Engine termination request was sent. ENGINE_STATUS_TERMINATION_PENDING = "ENGINE_STATUS_TERMINATION_PENDING" """ Engine termination request was sent.""" @@ -119,6 +117,7 @@ class EngineStatus(Enum): ENGINE_STATUS_DELETED = "ENGINE_STATUS_DELETED" """ Engine is soft-deleted.""" + class EngineStatusSummary(Enum): """ Engine summary status. @@ -148,7 +147,7 @@ class EngineStatusSummary(Enum): ENGINE_STATUS_SUMMARY_UPGRADING = "ENGINE_STATUS_SUMMARY_UPGRADING" """ Version of the PackDB is changing. - This is zero downtime operation that does not affect engine work.s + This is zero downtime operation that does not affect engine work. This status is reserved for future use (not used fow now).""" ENGINE_STATUS_SUMMARY_RESTARTING = "ENGINE_STATUS_SUMMARY_RESTARTING" @@ -183,7 +182,8 @@ class EngineStatusSummary(Enum): This status only indicates that there were issues during provisioning operations. If engine enters this status, all infrastructure should be stopped/terminated already.""" - + + class EngineOrder(Enum): ENGINE_ORDER_UNSPECIFIED = "ENGINE_ORDER_UNSPECIFIED" ENGINE_ORDER_NAME_ASC = "ENGINE_ORDER_NAME_ASC"