From 785f2c167c820bd9f058b29474f766b073263734 Mon Sep 17 00:00:00 2001 From: Carson Moore Date: Tue, 15 Nov 2022 12:40:38 -0600 Subject: [PATCH 1/6] Get metadata --- .../clients/dataframe/_data_frame_client.py | 17 +++++++++--- .../clients/dataframe/models/__init__.py | 4 +++ .../dataframe/models/_column_metadata.py | 15 +++++++++++ .../clients/dataframe/models/_column_type.py | 18 +++++++++++++ .../clients/dataframe/models/_data_type.py | 26 +++++++++++++++++++ .../dataframe/models/_table_metadata.py | 22 ++++++++++++++++ 6 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 nisystemlink/clients/dataframe/models/_column_metadata.py create mode 100644 nisystemlink/clients/dataframe/models/_column_type.py create mode 100644 nisystemlink/clients/dataframe/models/_data_type.py create mode 100644 nisystemlink/clients/dataframe/models/_table_metadata.py diff --git a/nisystemlink/clients/dataframe/_data_frame_client.py b/nisystemlink/clients/dataframe/_data_frame_client.py index 9620fc87..e054e203 100644 --- a/nisystemlink/clients/dataframe/_data_frame_client.py +++ b/nisystemlink/clients/dataframe/_data_frame_client.py @@ -7,7 +7,7 @@ from nisystemlink.clients import core from nisystemlink.clients.core._uplink._base_client import BaseClient -from uplink import get, returns +from uplink import get from . import models @@ -28,8 +28,19 @@ def __init__(self, configuration: Optional[core.HttpConfiguration] = None): super().__init__(configuration) - @returns.json() @get(_BASE_PATH) def api_info(self) -> models.ApiInfo: """Returns information about available API operations.""" - pass + ... + + @get(_BASE_PATH + "/tables/{id}") + def get_table_metadata(self, id: str) -> models.TableMetadata: + """Retrieves the metadata and column information for a single table identified by its ID. + + Args: + id (str): Unique ID of a DataFrame table. + + Returns: + models.TableMetadata: The metadata for the table. + """ + ... diff --git a/nisystemlink/clients/dataframe/models/__init__.py b/nisystemlink/clients/dataframe/models/__init__.py index 4dcf979c..6b66da60 100644 --- a/nisystemlink/clients/dataframe/models/__init__.py +++ b/nisystemlink/clients/dataframe/models/__init__.py @@ -1,3 +1,7 @@ from ._api_info import ApiInfo, Operation, OperationsV1 +from ._column_metadata import ColumnMetadata +from ._column_type import ColumnType +from ._data_type import DataType +from ._table_metadata import TableMetadata # flake8: noqa diff --git a/nisystemlink/clients/dataframe/models/_column_metadata.py b/nisystemlink/clients/dataframe/models/_column_metadata.py new file mode 100644 index 00000000..3cb244ee --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_column_metadata.py @@ -0,0 +1,15 @@ +from typing import Dict + +from nisystemlink.clients.core._uplink._json_model import JsonModel + +from ._column_type import ColumnType +from ._data_type import DataType + + +class ColumnMetadata(JsonModel): + """Defines a single column in a table.""" + + name: str #: The column name, which must be unique across all columns in the table. + data_type: DataType #: The data type of the column. + column_type: ColumnType #: The column type. Defaults to ColumnType.Normal. + properties: Dict #: User-defined properties associated with the column. diff --git a/nisystemlink/clients/dataframe/models/_column_type.py b/nisystemlink/clients/dataframe/models/_column_type.py new file mode 100644 index 00000000..2c052b00 --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_column_type.py @@ -0,0 +1,18 @@ +from enum import Enum + + +class ColumnType(str, Enum): + """Represents the different column types for a table column.""" + + Normal = "NORMAL" + """The column has no special properties. This is the default.""" + + Index = "INDEX" + """The column provides a unique value per row. Each table must provide + exactly one INDEX column. The column's :class:`.DataType` must be INT32, + INT64, or TIMESTAMP.""" + Nullable = "NULLABLE" + + """Rows may contain null values for this column. When appending rows, + NULLABLE columns may be left out entirely, in which case all rows being + appended will use null values for that column.""" diff --git a/nisystemlink/clients/dataframe/models/_data_type.py b/nisystemlink/clients/dataframe/models/_data_type.py new file mode 100644 index 00000000..e18d618f --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_data_type.py @@ -0,0 +1,26 @@ +from enum import Enum + + +class DataType(str, Enum): + """Represents the different data types for a table column.""" + + Bool = "BOOL" + """32-bit IEEE 754 floating-point number.""" + + Float32 = "FLOAT32" + """32-bit IEEE 754 floating-point number.""" + + Float64 = "FLOAT64" + """64-bit IEEE 754 floating-point number.""" + + Int32 = "INT32" + """32-bit signed integers.""" + + Int64 = "INT64" + """64-bit signed integers.""" + + String = "STRING" + """Arbitrary string data.""" + + Timestamp = "TIMESTAMP" + """Date and time represented in UTC with millisecond precision.""" diff --git a/nisystemlink/clients/dataframe/models/_table_metadata.py b/nisystemlink/clients/dataframe/models/_table_metadata.py new file mode 100644 index 00000000..f53d2aa3 --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_table_metadata.py @@ -0,0 +1,22 @@ +from datetime import datetime +from typing import Dict, List + +from nisystemlink.clients.core._uplink._json_model import JsonModel + +from ._column_metadata import ColumnMetadata + + +class TableMetadata(JsonModel): + """Contains information about a table, including its properties and column definitions.""" + + columns: List[ColumnMetadata] #: The list of columns in the table. + created_at: datetime #: The date and time the table was created. + id: str #: The table's unique identifier. + metadata_modified_at: datetime #: The date and time the table's metadata was last modified. + metadata_revision: int #: The table's metadata revision number, incremented each time the metadata is modified. + name: str #: The name associated with the table. + properties: Dict #: User-defined properties associated with the table. + row_count: int #: The number of rows in the table. + rows_modified_at: datetime #: The date and time the table's data was last modified. + supports_append: bool #: Whether the table supports appending additional rows of data. + workspace: str #: The workspace the table belongs to. From c12a27ef984d32f54702d99394a82d32f5fb28a3 Mon Sep 17 00:00:00 2001 From: Carson Moore Date: Tue, 15 Nov 2022 15:18:53 -0600 Subject: [PATCH 2/6] Fix up comments --- docs/conf.py | 1 + .../clients/dataframe/models/_api_info.py | 30 +++++++++---- .../dataframe/models/_column_metadata.py | 15 +++++-- .../clients/dataframe/models/_column_type.py | 2 +- .../dataframe/models/_table_metadata.py | 43 ++++++++++++++----- 5 files changed, 67 insertions(+), 24 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index b8a12f01..b435092b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,3 +57,4 @@ # Configuration for https://autodoc-pydantic.readthedocs.io autodoc_pydantic_model_show_config_summary = False +autodoc_pydantic_field_show_alias = False diff --git a/nisystemlink/clients/dataframe/models/_api_info.py b/nisystemlink/clients/dataframe/models/_api_info.py index 57d3822f..38006bf8 100644 --- a/nisystemlink/clients/dataframe/models/_api_info.py +++ b/nisystemlink/clients/dataframe/models/_api_info.py @@ -4,19 +4,33 @@ class Operation(JsonModel): """Represents an operation that can be performed on a data frame.""" - available: bool #: Whether or not the operation is available to the caller (e.g. due to permissions). - version: int #: The version of the available operation. + available: bool + """Whether or not the operation is available to the caller (e.g. due to permissions).""" + + version: int + """The version of the available operation.""" class OperationsV1(JsonModel): """The operations available in the routes provided by the v1 HTTP API.""" - create_tables: Operation #: The ability to create new DataFrame tables. - delete_tables: Operation #: The ability to delete tables and all of their data. - modify_metadata: Operation #: The ability to modify metadata for tables. - list_tables: Operation #: The ability to locate and read metadata for tables. - read_data: Operation #: The ability to query and read data from tables. - write_data: Operation #: The ability to append rows of data to tables. + create_tables: Operation + """The ability to create new DataFrame tables.""" + + delete_tables: Operation + """The ability to delete tables and all of their data.""" + + modify_metadata: Operation + """The ability to modify metadata for tables.""" + + list_tables: Operation + """The ability to locate and read metadata for tables.""" + + read_data: Operation + """The ability to query and read data from tables.""" + + write_data: Operation + """The ability to append rows of data to tables.""" class ApiInfo(JsonModel): diff --git a/nisystemlink/clients/dataframe/models/_column_metadata.py b/nisystemlink/clients/dataframe/models/_column_metadata.py index 3cb244ee..01a091be 100644 --- a/nisystemlink/clients/dataframe/models/_column_metadata.py +++ b/nisystemlink/clients/dataframe/models/_column_metadata.py @@ -9,7 +9,14 @@ class ColumnMetadata(JsonModel): """Defines a single column in a table.""" - name: str #: The column name, which must be unique across all columns in the table. - data_type: DataType #: The data type of the column. - column_type: ColumnType #: The column type. Defaults to ColumnType.Normal. - properties: Dict #: User-defined properties associated with the column. + name: str + """The column name, which must be unique across all columns in the table.""" + + data_type: DataType + """The data type of the column.""" + + column_type: ColumnType + """The column type. Defaults to ColumnType.Normal.""" + + properties: Dict + """User-defined properties associated with the column.""" diff --git a/nisystemlink/clients/dataframe/models/_column_type.py b/nisystemlink/clients/dataframe/models/_column_type.py index 2c052b00..cecd1efe 100644 --- a/nisystemlink/clients/dataframe/models/_column_type.py +++ b/nisystemlink/clients/dataframe/models/_column_type.py @@ -11,8 +11,8 @@ class ColumnType(str, Enum): """The column provides a unique value per row. Each table must provide exactly one INDEX column. The column's :class:`.DataType` must be INT32, INT64, or TIMESTAMP.""" - Nullable = "NULLABLE" + Nullable = "NULLABLE" """Rows may contain null values for this column. When appending rows, NULLABLE columns may be left out entirely, in which case all rows being appended will use null values for that column.""" diff --git a/nisystemlink/clients/dataframe/models/_table_metadata.py b/nisystemlink/clients/dataframe/models/_table_metadata.py index f53d2aa3..bf872375 100644 --- a/nisystemlink/clients/dataframe/models/_table_metadata.py +++ b/nisystemlink/clients/dataframe/models/_table_metadata.py @@ -9,14 +9,35 @@ class TableMetadata(JsonModel): """Contains information about a table, including its properties and column definitions.""" - columns: List[ColumnMetadata] #: The list of columns in the table. - created_at: datetime #: The date and time the table was created. - id: str #: The table's unique identifier. - metadata_modified_at: datetime #: The date and time the table's metadata was last modified. - metadata_revision: int #: The table's metadata revision number, incremented each time the metadata is modified. - name: str #: The name associated with the table. - properties: Dict #: User-defined properties associated with the table. - row_count: int #: The number of rows in the table. - rows_modified_at: datetime #: The date and time the table's data was last modified. - supports_append: bool #: Whether the table supports appending additional rows of data. - workspace: str #: The workspace the table belongs to. + columns: List[ColumnMetadata] + """The list of columns in the table.""" + + created_at: datetime + """The date and time the table was created.""" + + id: str + """The table's unique identifier.""" + + metadata_modified_at: datetime + """The date and time the table's metadata was last modified.""" + + metadata_revision: int + """The table's metadata revision number, incremented each time the metadata is modified.""" + + name: str + """The name associated with the table.""" + + properties: Dict + """User-defined properties associated with the table.""" + + row_count: int + """The number of rows in the table.""" + + rows_modified_at: datetime + """The date and time the table's data was last modified.""" + + supports_append: bool + """Whether the table supports appending additional rows of data.""" + + workspace: str + """The workspace the table belongs to.""" From 18a3cb6d788385fbca7649b88a1208a7051abdd0 Mon Sep 17 00:00:00 2001 From: Carson Moore Date: Tue, 15 Nov 2022 18:28:10 -0600 Subject: [PATCH 3/6] Implement create and delete --- mypy.ini | 1 - .../clients/core/_uplink/_base_client.py | 18 +++++-- .../clients/dataframe/_data_frame_client.py | 25 +++++++++- .../clients/dataframe/models/__init__.py | 3 +- .../{_column_metadata.py => _column.py} | 8 +-- .../dataframe/models/_create_table_request.py | 22 ++++++++ .../dataframe/models/_table_metadata.py | 6 +-- tests/integration/dataframe/test_dataframe.py | 50 ++++++++++++++++++- 8 files changed, 118 insertions(+), 15 deletions(-) rename nisystemlink/clients/dataframe/models/{_column_metadata.py => _column.py} (74%) create mode 100644 nisystemlink/clients/dataframe/models/_create_table_request.py diff --git a/mypy.ini b/mypy.ini index 3705a916..80cd27f8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -13,7 +13,6 @@ strict_equality=True [mypy-tests.*] disallow_untyped_calls=True -disallow_incomplete_defs=True disallow_untyped_decorators=True strict_equality=True diff --git a/nisystemlink/clients/core/_uplink/_base_client.py b/nisystemlink/clients/core/_uplink/_base_client.py index b812d428..171fde05 100644 --- a/nisystemlink/clients/core/_uplink/_base_client.py +++ b/nisystemlink/clients/core/_uplink/_base_client.py @@ -1,10 +1,12 @@ # mypy: disable-error-code = misc -from typing import Optional +from typing import Dict, Optional, Type from nisystemlink.clients import core from requests import JSONDecodeError, Response -from uplink import Consumer, response_handler +from uplink import Consumer, dumps, response_handler + +from ._json_model import JsonModel @response_handler @@ -36,6 +38,12 @@ def _handle_http_status(response: Response) -> Optional[Response]: raise core.ApiException(msg, http_status_code=response.status_code) +@dumps.to_json(JsonModel) +def _deserialize_model(model_cls: Type[JsonModel], model_instance: JsonModel) -> Dict: + """Turns a :class:`.JsonModel` instance into a dictionary for serialization.""" + return model_instance.dict(by_alias=True, exclude_unset=True) + + class BaseClient(Consumer): """Base class for SystemLink clients, built on top of `Uplink `_.""" @@ -45,6 +53,10 @@ def __init__(self, configuration: core.HttpConfiguration): Args: configuration: Defines the web server to connect to and information about how to connect. """ - super().__init__(base_url=configuration.server_uri, hooks=[_handle_http_status]) + super().__init__( + base_url=configuration.server_uri, + converter=_deserialize_model, + hooks=[_handle_http_status], + ) if configuration.api_keys: self.session.headers.update(configuration.api_keys) diff --git a/nisystemlink/clients/dataframe/_data_frame_client.py b/nisystemlink/clients/dataframe/_data_frame_client.py index e054e203..06918098 100644 --- a/nisystemlink/clients/dataframe/_data_frame_client.py +++ b/nisystemlink/clients/dataframe/_data_frame_client.py @@ -7,7 +7,7 @@ from nisystemlink.clients import core from nisystemlink.clients.core._uplink._base_client import BaseClient -from uplink import get +from uplink import Body, delete, get, json, post, returns from . import models @@ -33,6 +33,20 @@ def api_info(self) -> models.ApiInfo: """Returns information about available API operations.""" ... + @json + @returns.json(key="id") + @post(_BASE_PATH + "/tables", args=(Body,)) + def create_table(self, table: models.CreateTableRequest) -> str: + """Create a new table with the provided metadata and column definitions. + + Args: + table: The request create the table. + + Returns: + The ID of the newly created table. + """ + ... + @get(_BASE_PATH + "/tables/{id}") def get_table_metadata(self, id: str) -> models.TableMetadata: """Retrieves the metadata and column information for a single table identified by its ID. @@ -44,3 +58,12 @@ def get_table_metadata(self, id: str) -> models.TableMetadata: models.TableMetadata: The metadata for the table. """ ... + + @delete(_BASE_PATH + "/tables/{id}") + def delete_table(self, id: str) -> None: + """Deletes a table. + + Args: + id (str): Unique ID of a DataFrame table. + """ + ... diff --git a/nisystemlink/clients/dataframe/models/__init__.py b/nisystemlink/clients/dataframe/models/__init__.py index 6b66da60..5e8e4db5 100644 --- a/nisystemlink/clients/dataframe/models/__init__.py +++ b/nisystemlink/clients/dataframe/models/__init__.py @@ -1,5 +1,6 @@ from ._api_info import ApiInfo, Operation, OperationsV1 -from ._column_metadata import ColumnMetadata +from ._create_table_request import CreateTableRequest +from ._column import Column from ._column_type import ColumnType from ._data_type import DataType from ._table_metadata import TableMetadata diff --git a/nisystemlink/clients/dataframe/models/_column_metadata.py b/nisystemlink/clients/dataframe/models/_column.py similarity index 74% rename from nisystemlink/clients/dataframe/models/_column_metadata.py rename to nisystemlink/clients/dataframe/models/_column.py index 01a091be..6c56fc32 100644 --- a/nisystemlink/clients/dataframe/models/_column_metadata.py +++ b/nisystemlink/clients/dataframe/models/_column.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Optional from nisystemlink.clients.core._uplink._json_model import JsonModel @@ -6,7 +6,7 @@ from ._data_type import DataType -class ColumnMetadata(JsonModel): +class Column(JsonModel): """Defines a single column in a table.""" name: str @@ -15,8 +15,8 @@ class ColumnMetadata(JsonModel): data_type: DataType """The data type of the column.""" - column_type: ColumnType + column_type: Optional[ColumnType] = ColumnType.Normal """The column type. Defaults to ColumnType.Normal.""" - properties: Dict + properties: Optional[Dict[str, str]] = None """User-defined properties associated with the column.""" diff --git a/nisystemlink/clients/dataframe/models/_create_table_request.py b/nisystemlink/clients/dataframe/models/_create_table_request.py new file mode 100644 index 00000000..865f3409 --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_create_table_request.py @@ -0,0 +1,22 @@ +from typing import Dict, List, Optional + +from nisystemlink.clients.core._uplink._json_model import JsonModel + +from ._column import Column + + +class CreateTableRequest(JsonModel): + """Contains information needed to create a table, including its properties and column definitions.""" + + columns: List[Column] + """The list of columns in the table. Exactly one column must have a :class:`.ColumnType` of INDEX.""" + + name: Optional[str] = None + """The name to associate with the table. When not specified, a name will be + assigned from the table's ID.""" + + properties: Optional[Dict[str, str]] = None + """User-defined properties to associate with the table.""" + + workspace: Optional[str] = None + """The workspace to create the table in. Uses the default workspace when not specified.""" diff --git a/nisystemlink/clients/dataframe/models/_table_metadata.py b/nisystemlink/clients/dataframe/models/_table_metadata.py index bf872375..18abeec6 100644 --- a/nisystemlink/clients/dataframe/models/_table_metadata.py +++ b/nisystemlink/clients/dataframe/models/_table_metadata.py @@ -3,13 +3,13 @@ from nisystemlink.clients.core._uplink._json_model import JsonModel -from ._column_metadata import ColumnMetadata +from ._column import Column class TableMetadata(JsonModel): """Contains information about a table, including its properties and column definitions.""" - columns: List[ColumnMetadata] + columns: List[Column] """The list of columns in the table.""" created_at: datetime @@ -27,7 +27,7 @@ class TableMetadata(JsonModel): name: str """The name associated with the table.""" - properties: Dict + properties: Dict[str, str] """User-defined properties associated with the table.""" row_count: int diff --git a/tests/integration/dataframe/test_dataframe.py b/tests/integration/dataframe/test_dataframe.py index 5c9a21f6..f0dcf7ed 100644 --- a/tests/integration/dataframe/test_dataframe.py +++ b/tests/integration/dataframe/test_dataframe.py @@ -1,15 +1,61 @@ # -*- coding: utf-8 -*- +from typing import Iterator import pytest # type: ignore from nisystemlink.clients.dataframe import DataFrameClient +from nisystemlink.clients.dataframe import models @pytest.mark.enterprise @pytest.mark.integration class TestDataFrame: - def test__api_info__returns(self, enterprise_config): - client = DataFrameClient(enterprise_config) + @pytest.fixture + def client(self, enterprise_config): + return DataFrameClient(enterprise_config) + @pytest.fixture + def table_id(self, client: DataFrameClient) -> Iterator[str]: + id = client.create_table( + models.CreateTableRequest( + columns=[ + models.Column( + name="time", + data_type=models.DataType.Timestamp, + column_type=models.ColumnType.Index, + properties={"cat": "meow"}, + ), + models.Column(name="value", data_type=models.DataType.Int32), + ], + name="Python API test table (delete me)", + properties={"dog": "woof"}, + ) + ) + yield id + client.delete_table(id) + + def test__api_info__returns(self, client): response = client.api_info() assert len(response.dict()) != 0 + + def test__create_table__metadata_is_corect( + self, client: DataFrameClient, table_id: str + ): + table_metadata = client.get_table_metadata(table_id) + + assert table_metadata.name == "Python API test table (delete me)" + assert table_metadata.properties == {"dog": "woof"} + assert table_metadata.columns == [ + models.Column( + name="time", + data_type=models.DataType.Timestamp, + column_type=models.ColumnType.Index, + properties={"cat": "meow"}, + ), + models.Column( + name="value", + data_type=models.DataType.Int32, + column_type=models.ColumnType.Normal, + properties={}, + ), + ] From 20f3bbfe980a5793277b0fc11d41e886d4b69c10 Mon Sep 17 00:00:00 2001 From: Carson Moore Date: Wed, 16 Nov 2022 17:03:50 -0600 Subject: [PATCH 4/6] Add list_tables method --- .../clients/core/_uplink/_paged_result.py | 15 ++ .../clients/dataframe/_data_frame_client.py | 39 +++++- .../clients/dataframe/models/__init__.py | 2 + .../clients/dataframe/models/_column.py | 2 +- .../clients/dataframe/models/_order_by.py | 14 ++ .../clients/dataframe/models/_paged_tables.py | 12 ++ tests/integration/dataframe/test_dataframe.py | 128 ++++++++++++++---- 7 files changed, 182 insertions(+), 30 deletions(-) create mode 100644 nisystemlink/clients/core/_uplink/_paged_result.py create mode 100644 nisystemlink/clients/dataframe/models/_order_by.py create mode 100644 nisystemlink/clients/dataframe/models/_paged_tables.py diff --git a/nisystemlink/clients/core/_uplink/_paged_result.py b/nisystemlink/clients/core/_uplink/_paged_result.py new file mode 100644 index 00000000..0eae3589 --- /dev/null +++ b/nisystemlink/clients/core/_uplink/_paged_result.py @@ -0,0 +1,15 @@ +from typing import Optional + +from ._json_model import JsonModel + + +class PagedResult(JsonModel): + continuation_token: Optional[str] + """A token which allows the user to resume a query at the next item in the matching results. + + When querying, a token will be returned if a query may be + continued. To obtain the next page of results, pass the token to the service + on a subsequent request.The service will respond with a new continuation + token. To paginate results, continue sending requests with the newest + continuation token provided by the service, until this value is null. + """ diff --git a/nisystemlink/clients/dataframe/_data_frame_client.py b/nisystemlink/clients/dataframe/_data_frame_client.py index 06918098..c3c59561 100644 --- a/nisystemlink/clients/dataframe/_data_frame_client.py +++ b/nisystemlink/clients/dataframe/_data_frame_client.py @@ -3,11 +3,11 @@ """Implementation of DataFrameClient.""" -from typing import Optional +from typing import List, Optional from nisystemlink.clients import core from nisystemlink.clients.core._uplink._base_client import BaseClient -from uplink import Body, delete, get, json, post, returns +from uplink import Body, delete, get, json, post, Query, returns from . import models @@ -33,6 +33,41 @@ def api_info(self) -> models.ApiInfo: """Returns information about available API operations.""" ... + @get( + _BASE_PATH + "/tables", + args=( + Query("take"), + Query("id"), + Query("orderBy"), + Query("orderByDescending"), + Query("continuationToken"), + Query("workspace"), + ), + ) + def list_tables( + self, + take: Optional[int] = None, + id: Optional[List[str]] = None, + order_by: Optional[models.OrderBy] = None, + order_by_descending: Optional[bool] = None, + continuation_token: Optional[str] = None, + workspace: Optional[str] = None, + ) -> models.PagedTables: + """Lists available tables on the SystemLink DataFrame service. + + Args: + take: Limits the returned list to the specified number of results. Defaults to 1000. + id: List of table IDs to filter by. + order_by: The sort order of the returned list of tables. + order_by_descending: Whether to sort descending instead of ascending. Defaults to false. + continuation_token: The token used to paginate results. + workspace: List of workspace IDs to filter by. + + Returns: + models.PagedTables: The list of tables with a continuation token. + """ + ... + @json @returns.json(key="id") @post(_BASE_PATH + "/tables", args=(Body,)) diff --git a/nisystemlink/clients/dataframe/models/__init__.py b/nisystemlink/clients/dataframe/models/__init__.py index 5e8e4db5..ba3b2f72 100644 --- a/nisystemlink/clients/dataframe/models/__init__.py +++ b/nisystemlink/clients/dataframe/models/__init__.py @@ -3,6 +3,8 @@ from ._column import Column from ._column_type import ColumnType from ._data_type import DataType +from ._order_by import OrderBy +from ._paged_tables import PagedTables from ._table_metadata import TableMetadata # flake8: noqa diff --git a/nisystemlink/clients/dataframe/models/_column.py b/nisystemlink/clients/dataframe/models/_column.py index 6c56fc32..ea1d9757 100644 --- a/nisystemlink/clients/dataframe/models/_column.py +++ b/nisystemlink/clients/dataframe/models/_column.py @@ -15,7 +15,7 @@ class Column(JsonModel): data_type: DataType """The data type of the column.""" - column_type: Optional[ColumnType] = ColumnType.Normal + column_type: ColumnType = ColumnType.Normal """The column type. Defaults to ColumnType.Normal.""" properties: Optional[Dict[str, str]] = None diff --git a/nisystemlink/clients/dataframe/models/_order_by.py b/nisystemlink/clients/dataframe/models/_order_by.py new file mode 100644 index 00000000..7ba3dd94 --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_order_by.py @@ -0,0 +1,14 @@ +from typing import Literal + + +OrderBy = Literal[ + "CREATED_AT", "METADATA_MODIFIED_AT", "NAME", "NUMBER_OF_ROWS", "ROWS_MODIFIED_AT" +] +"""Possible options for sorting when querying tables. + +* ``CREATED_AT``: The date and time the table was created. +* ``METADATA_MODIFIED_AT``: The date and time the table's metadata properties were modified. +* ``NAME``: The name of the table. +* ``NUMBER_OF_ROWS``: The number of rows of data in the table. +* ``ROWS_MODIFIED_AT``: Date and time rows were most recently appended to the table. +""" diff --git a/nisystemlink/clients/dataframe/models/_paged_tables.py b/nisystemlink/clients/dataframe/models/_paged_tables.py new file mode 100644 index 00000000..420512d9 --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_paged_tables.py @@ -0,0 +1,12 @@ +from typing import List + +from nisystemlink.clients.core._uplink._paged_result import PagedResult + +from ._table_metadata import TableMetadata + + +class PagedTables(PagedResult): + """The response for a table query containing the matched tables.""" + + tables: List[TableMetadata] + """The list of tables returned by the query.""" diff --git a/tests/integration/dataframe/test_dataframe.py b/tests/integration/dataframe/test_dataframe.py index f0dcf7ed..24647aca 100644 --- a/tests/integration/dataframe/test_dataframe.py +++ b/tests/integration/dataframe/test_dataframe.py @@ -1,49 +1,76 @@ # -*- coding: utf-8 -*- -from typing import Iterator +from datetime import datetime +from typing import List import pytest # type: ignore +from nisystemlink.clients.core import ApiException from nisystemlink.clients.dataframe import DataFrameClient from nisystemlink.clients.dataframe import models -@pytest.mark.enterprise -@pytest.mark.integration -class TestDataFrame: - @pytest.fixture - def client(self, enterprise_config): - return DataFrameClient(enterprise_config) +@pytest.fixture(scope="class") +def client(enterprise_config): + """Fixture to create a DataFrameClient instance.""" + return DataFrameClient(enterprise_config) - @pytest.fixture - def table_id(self, client: DataFrameClient) -> Iterator[str]: - id = client.create_table( - models.CreateTableRequest( - columns=[ - models.Column( - name="time", - data_type=models.DataType.Timestamp, - column_type=models.ColumnType.Index, - properties={"cat": "meow"}, - ), - models.Column(name="value", data_type=models.DataType.Int32), - ], - name="Python API test table (delete me)", - properties={"dog": "woof"}, + +@pytest.fixture(scope="class") +def create_table(client: DataFrameClient): + """Fixture to return a factory that creates tables.""" + tables = [] + + def _create_table(table: models.CreateTableRequest) -> str: + id = client.create_table(table) + print(f"Created table {id}") + tables.append(id) + return id + + yield _create_table + + for id in tables: + client.delete_table(id) + print(f"Deleted table {id}") + + +@pytest.fixture(scope="class") +def test_tables(create_table): + """Fixture to create a set of test tables.""" + ids = [] + for i in range(1, 4): + ids.append( + create_table( + models.CreateTableRequest( + columns=[ + models.Column( + name="time", + data_type=models.DataType.Timestamp, + column_type=models.ColumnType.Index, + properties={"cat": "meow"}, + ), + models.Column(name="value", data_type=models.DataType.Int32), + ], + name=f"Python API test table {i} (delete me)", + properties={"dog": "woof"}, + ) ) ) - yield id - client.delete_table(id) + return ids + +@pytest.mark.enterprise +@pytest.mark.integration +class TestDataFrame: def test__api_info__returns(self, client): response = client.api_info() assert len(response.dict()) != 0 def test__create_table__metadata_is_corect( - self, client: DataFrameClient, table_id: str + self, client: DataFrameClient, test_tables: List[str] ): - table_metadata = client.get_table_metadata(table_id) + table_metadata = client.get_table_metadata(test_tables[0]) - assert table_metadata.name == "Python API test table (delete me)" + assert table_metadata.name == "Python API test table 1 (delete me)" assert table_metadata.properties == {"dog": "woof"} assert table_metadata.columns == [ models.Column( @@ -59,3 +86,50 @@ def test__create_table__metadata_is_corect( properties={}, ), ] + + def test__get_table__correct_timestamp(self, client: DataFrameClient, create_table): + id = create_table( + models.CreateTableRequest( + columns=[ + models.Column( + name="index", + data_type=models.DataType.Int32, + column_type=models.ColumnType.Index, + ) + ] + ) + ) + table = client.get_table_metadata(id) + + now = datetime.now().timestamp() + # Assert that timestamp is within 10 seconds of now + assert table.created_at.timestamp() == pytest.approx(now, abs=10) + + def test__get_table_invalid_id__raises(self, client: DataFrameClient): + with pytest.raises(ApiException, match="invalid table ID"): + client.get_table_metadata("invalid_id") + + def test__list_tables__returns( + self, client: DataFrameClient, test_tables: List[str] + ): + take = len(test_tables) - 1 + first_page = client.list_tables( + take=take, + id=test_tables, + order_by="NAME", + order_by_descending=True, + ) + + assert len(first_page.tables) == take + assert first_page.tables[0].id == test_tables[-1] # Asserts descending order + assert first_page.continuation_token is not None + + second_page = client.list_tables( + id=test_tables, + order_by="NAME", + order_by_descending=True, + continuation_token=first_page.continuation_token, + ) + + assert len(second_page.tables) == 1 + assert second_page.continuation_token is None From a526a0f7334c57f06dcecf3a8f080292d7633759 Mon Sep 17 00:00:00 2001 From: Carson Date: Fri, 18 Nov 2022 10:18:32 -0600 Subject: [PATCH 5/6] Fix whitespace in comment Co-authored-by: Paul Spangler --- nisystemlink/clients/core/_uplink/_paged_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nisystemlink/clients/core/_uplink/_paged_result.py b/nisystemlink/clients/core/_uplink/_paged_result.py index 0eae3589..185bb371 100644 --- a/nisystemlink/clients/core/_uplink/_paged_result.py +++ b/nisystemlink/clients/core/_uplink/_paged_result.py @@ -9,7 +9,7 @@ class PagedResult(JsonModel): When querying, a token will be returned if a query may be continued. To obtain the next page of results, pass the token to the service - on a subsequent request.The service will respond with a new continuation + on a subsequent request. The service will respond with a new continuation token. To paginate results, continue sending requests with the newest continuation token provided by the service, until this value is null. """ From c5bfb578c710da3534262f802f1eccdc05413288 Mon Sep 17 00:00:00 2001 From: Carson Moore Date: Fri, 18 Nov 2022 12:21:00 -0600 Subject: [PATCH 6/6] PR feedback --- nisystemlink/clients/dataframe/_data_frame_client.py | 2 +- nisystemlink/clients/dataframe/models/_order_by.py | 2 +- tests/integration/dataframe/test_dataframe.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nisystemlink/clients/dataframe/_data_frame_client.py b/nisystemlink/clients/dataframe/_data_frame_client.py index c3c59561..6c272599 100644 --- a/nisystemlink/clients/dataframe/_data_frame_client.py +++ b/nisystemlink/clients/dataframe/_data_frame_client.py @@ -51,7 +51,7 @@ def list_tables( order_by: Optional[models.OrderBy] = None, order_by_descending: Optional[bool] = None, continuation_token: Optional[str] = None, - workspace: Optional[str] = None, + workspace: Optional[List[str]] = None, ) -> models.PagedTables: """Lists available tables on the SystemLink DataFrame service. diff --git a/nisystemlink/clients/dataframe/models/_order_by.py b/nisystemlink/clients/dataframe/models/_order_by.py index 7ba3dd94..dac98a88 100644 --- a/nisystemlink/clients/dataframe/models/_order_by.py +++ b/nisystemlink/clients/dataframe/models/_order_by.py @@ -1,6 +1,6 @@ from typing import Literal - +# TODO: Migrate to Enum when this change is released: https://github.com/prkumar/uplink/pull/282 OrderBy = Literal[ "CREATED_AT", "METADATA_MODIFIED_AT", "NAME", "NUMBER_OF_ROWS", "ROWS_MODIFIED_AT" ] diff --git a/tests/integration/dataframe/test_dataframe.py b/tests/integration/dataframe/test_dataframe.py index 24647aca..7f1de8f6 100644 --- a/tests/integration/dataframe/test_dataframe.py +++ b/tests/integration/dataframe/test_dataframe.py @@ -21,15 +21,14 @@ def create_table(client: DataFrameClient): def _create_table(table: models.CreateTableRequest) -> str: id = client.create_table(table) - print(f"Created table {id}") tables.append(id) return id yield _create_table for id in tables: + # TODO: Use multi-delete when implemented. client.delete_table(id) - print(f"Deleted table {id}") @pytest.fixture(scope="class")