From d7a7642f12543be4c1393435c9fd11604288a17f Mon Sep 17 00:00:00 2001 From: Carson Date: Mon, 5 Dec 2022 16:52:12 -0600 Subject: [PATCH] feat: Read and append table data (#36) --- .../clients/dataframe/_data_frame_client.py | 53 +++++++++++++- .../clients/dataframe/models/__init__.py | 3 + .../models/_append_table_data_request.py | 17 +++++ .../clients/dataframe/models/_data_frame.py | 57 +++++++++++++++ .../dataframe/models/_paged_table_rows.py | 13 ++++ poetry.lock | 47 +++++++++++- pyproject.toml | 15 ++-- tests/integration/dataframe/test_dataframe.py | 71 +++++++++++++++++++ 8 files changed, 268 insertions(+), 8 deletions(-) create mode 100644 nisystemlink/clients/dataframe/models/_append_table_data_request.py create mode 100644 nisystemlink/clients/dataframe/models/_data_frame.py create mode 100644 nisystemlink/clients/dataframe/models/_paged_table_rows.py diff --git a/nisystemlink/clients/dataframe/_data_frame_client.py b/nisystemlink/clients/dataframe/_data_frame_client.py index 7b6eb0e..f82d879 100644 --- a/nisystemlink/clients/dataframe/_data_frame_client.py +++ b/nisystemlink/clients/dataframe/_data_frame_client.py @@ -31,14 +31,14 @@ def api_info(self) -> models.ApiInfo: @get( "tables", - args=( + args=[ Query("take"), Query("id"), Query("orderBy"), Query("orderByDescending"), Query("continuationToken"), Query("workspace"), - ), + ], ) def list_tables( self, @@ -148,3 +148,52 @@ def modify_tables( tables were modified successfully. """ ... + + @get( + "tables/{id}/data", + args=[ + Path("id"), + Query("columns"), + Query("orderBy"), + Query("orderByDescending"), + Query("take"), + Query("continuationToken"), + ], + ) + def get_table_data( + self, + id: str, + columns: Optional[List[str]] = None, + order_by: Optional[List[str]] = None, + order_by_descending: Optional[bool] = None, + take: Optional[int] = None, + continuation_token: Optional[str] = None, + ) -> models.PagedTableRows: + """Reads raw data from the table identified by its ID. + + Args: + id: Unique ID of a DataFrame table. + columns: Columns to include in the response. Data will be returned in the same order as + the columns. If not specified, all columns are returned. + order_by: List of columns to sort by. Multiple columns may be specified to order rows + that have the same value for prior columns. The columns used for ordering do not + need to be included in the columns list, in which case they are not returned. If + not specified, then the order in which results are returned is undefined. + order_by_descending: Whether to sort descending instead of ascending. Defaults to false. + take: Limits the returned list to the specified number of results. Defaults to 500. + continuation_token: The token used to paginate results. + + Returns: + models.PagedTableRows: The table data and total number of rows with a continuation token. + """ + ... + + @post("tables/{id}/data", args=[Path, Body]) + def append_table_data(self, id: str, data: models.AppendTableDataRequest) -> None: + """Appends one or more rows of data to the table identified by its ID. + + Args: + id: Unique ID of a DataFrame table. + data: The rows of data to append and any additional options. + """ + ... diff --git a/nisystemlink/clients/dataframe/models/__init__.py b/nisystemlink/clients/dataframe/models/__init__.py index 20f20fc..432e1b4 100644 --- a/nisystemlink/clients/dataframe/models/__init__.py +++ b/nisystemlink/clients/dataframe/models/__init__.py @@ -1,7 +1,9 @@ +from ._append_table_data_request import AppendTableDataRequest from ._api_info import ApiInfo, Operation, OperationsV1 from ._create_table_request import CreateTableRequest from ._column import Column from ._column_type import ColumnType +from ._data_frame import DataFrame from ._data_type import DataType from ._delete_tables_partial_success import DeleteTablesPartialSuccess from ._modify_tables_partial_success import ModifyTablesPartialSuccess @@ -9,6 +11,7 @@ from ._modify_tables_request import ModifyTablesRequest, TableMetdataModification from ._order_by import OrderBy from ._paged_tables import PagedTables +from ._paged_table_rows import PagedTableRows from ._query_tables_request import QueryTablesRequest from ._table_metadata import TableMetadata diff --git a/nisystemlink/clients/dataframe/models/_append_table_data_request.py b/nisystemlink/clients/dataframe/models/_append_table_data_request.py new file mode 100644 index 0000000..ae08690 --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_append_table_data_request.py @@ -0,0 +1,17 @@ +from typing import Optional + +from nisystemlink.clients.core._uplink._json_model import JsonModel + +from ._data_frame import DataFrame + + +class AppendTableDataRequest(JsonModel): + """Contains the rows to append and optional flags. The ``frame`` field is + required unless ``endOfData`` is true. + """ + + frame: Optional[DataFrame] = None + """The data frame containing the rows to append.""" + + end_of_data: Optional[bool] = None + """Whether the table should expect any additional rows to be appended in future requests.""" diff --git a/nisystemlink/clients/dataframe/models/_data_frame.py b/nisystemlink/clients/dataframe/models/_data_frame.py new file mode 100644 index 0000000..c34c41f --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_data_frame.py @@ -0,0 +1,57 @@ +from typing import List, Optional + +from nisystemlink.clients.core._uplink._json_model import JsonModel + + +class DataFrame(JsonModel): + """Data read from or to be written to a table. + + Values may be ``None`` (if the column is of type ``NULLABLE``) or encoded as + a string in a format according to each column's datatype: + + * BOOL: One of ``"true"`` or ``"false"``, case-insensitive. + * INT32: Any integer number in the range [-2147483648, 2147483647], + surrounded by quotes. + * INT64: Any integer number in the range [-9223372036854775808, + 9223372036854775807], surrounded by quotes. + * FLOAT32: A decimal number using a period for the decimal point, optionally + in scientific notation, in the range [-3.40282347E+38, 3.40282347E+38], + surrounded by quotes. Not all values within the range can be represented + with 32 bits. To preserve the exact binary encoding of the value when + converting to a string, clients should serialize 9 digits after the + decimal. Instead of a number, the value may be ``"NaN"`` (not a number), + ``"Infinity"`` (positive infinity), or ``"-Infinity"`` (negative + infinity), case-sensitive. + * FLOAT64: A decimal number using a period for the decimal point, optionally + in scientific notation, in the range [-1.7976931348623157E+308, + 1.7976931348623157E+308], surrounded by quotes. Not all values within the + range can be represented with 64 bits. To preserve the exact binary + encoding of the value when converting to a string, clients should + serialize 17 digits after the decimal. Instead of a number, the value may + be ``"NaN"`` (not a number), ``"Infinity"`` (positive infinity), or + ``"-Infinity"`` (negative infinity), case-sensitive. + * STRING: Any quoted string. + * TIMESTAMP: A date and time with millisecond precision in ISO-8601 format + and time zone. For example: ``"2022-08-19T16:17:30.123Z"``. If a time zone + is not provided, UTC is assumed. If a time zone other than UTC is + provided, the value will be converted to UTC. If more than three digits of + fractional seconds are provided, the time will be truncated to three + digits (i.e. milliseconds). + + The format is the same as a serialized Pandas DataFrame with orient="split" + and index=False. See + https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html. + + When providing a DataFrame for appending rows, any of the table's columns + not specified will receive a value of ``None``. If any such columns aren't + nullable, an error will be returned. If the entire columns property is left + out, each row is assumed to contain all columns in the order specified when + the table was created. + """ + + columns: Optional[List[str]] = None + """The names and order of the columns included in the data frame.""" + + data: List[List[Optional[str]]] + """The data for each row with the order specified in the columns property. + Must contain a value for each column in the columns property.""" diff --git a/nisystemlink/clients/dataframe/models/_paged_table_rows.py b/nisystemlink/clients/dataframe/models/_paged_table_rows.py new file mode 100644 index 0000000..f16474d --- /dev/null +++ b/nisystemlink/clients/dataframe/models/_paged_table_rows.py @@ -0,0 +1,13 @@ +from nisystemlink.clients.core._uplink._with_paging import WithPaging + +from ._data_frame import DataFrame + + +class PagedTableRows(WithPaging): + """Contains the result of a query for rows of data.""" + + frame: DataFrame + """The data frame containing the rows of data.""" + + total_row_count: int + """The total number of rows matched by the query across all pages of results.""" diff --git a/poetry.lock b/poetry.lock index 1e226b1..dac5a4d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -423,6 +423,23 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + [[package]] name = "rfc3986" version = "1.5.0" @@ -474,6 +491,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "tomli" version = "2.0.1" @@ -493,6 +518,14 @@ python-versions = "*" [package.dependencies] types-urllib3 = "<1.27" +[[package]] +name = "types-toml" +version = "0.10.8.1" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-urllib3" version = "1.26.25.4" @@ -554,7 +587,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "e51db838bd40f553ff32f3fd38c63d9d3f5b49c9856cd6aa673d20f64c0bd4b1" +content-hash = "6497fc0d588477e74b42201860dfa2055b4fe5b37188385f62d50cddeac68fd5" [metadata.files] aenum = [ @@ -767,6 +800,10 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] +responses = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] rfc3986 = [ {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, @@ -787,6 +824,10 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, @@ -795,6 +836,10 @@ types-requests = [ {file = "types-requests-2.28.11.5.tar.gz", hash = "sha256:a7df37cc6fb6187a84097da951f8e21d335448aa2501a6b0a39cbd1d7ca9ee2a"}, {file = "types_requests-2.28.11.5-py3-none-any.whl", hash = "sha256:091d4a5a33c1b4f20d8b1b952aa8fa27a6e767c44c3cf65e56580df0b05fd8a9"}, ] +types-toml = [ + {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, + {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, +] types-urllib3 = [ {file = "types-urllib3-1.26.25.4.tar.gz", hash = "sha256:eec5556428eec862b1ac578fb69aab3877995a99ffec9e5a12cf7fbd0cc9daee"}, {file = "types_urllib3-1.26.25.4-py3-none-any.whl", hash = "sha256:ed6b9e8a8be488796f72306889a06a3fc3cb1aa99af02ab8afb50144d7317e49"}, diff --git a/pyproject.toml b/pyproject.toml index 7c7176e..77afa0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,11 @@ name = "nisystemlink-clients" version = "0.7.0" description = "NI-SystemLink Python API" authors = ["National Instruments"] -maintainers = ["Carson Moore ", "Paul Spangler ", "Cameron Waterman "] +maintainers = [ + "Carson Moore ", + "Paul Spangler ", + "Cameron Waterman ", +] keywords = ["nisystemlink", "systemlink"] license = "MIT" readme = "README.rst" @@ -42,6 +46,7 @@ mypy = "^0.982" flake8-docstrings = "^1.6.0" poethepoet = "^0.16.4" types-requests = "^2.28.11.4" +responses = "^0.22.0" [tool.poe.tasks] test = "pytest tests -m \"(not slow) and (not cloud) and (not enterprise)\"" @@ -67,10 +72,10 @@ markers = [ exclude = ".*\\.pyi" [tool.semantic_release] -branch = "master" -version_toml = ["pyproject.toml:tool.poetry.version"] -build_command = "poetry build" -major_on_zero = false +branch = "master" +version_toml = ["pyproject.toml:tool.poetry.version"] +build_command = "poetry build" +major_on_zero = false [build-system] requires = ["poetry-core"] diff --git a/tests/integration/dataframe/test_dataframe.py b/tests/integration/dataframe/test_dataframe.py index 24db1eb..1b7bf6a 100644 --- a/tests/integration/dataframe/test_dataframe.py +++ b/tests/integration/dataframe/test_dataframe.py @@ -3,6 +3,7 @@ from typing import List, Optional import pytest # type: ignore +import responses from nisystemlink.clients.core import ApiException from nisystemlink.clients.dataframe import DataFrameClient from nisystemlink.clients.dataframe import models @@ -276,3 +277,73 @@ def test__modify_tables__returns_partial_success(self, client: DataFrameClient): assert response.modified_table_ids == [id] assert response.failed_modifications == [updates[1]] assert len(response.error.inner_errors) == 1 + + def test__read_and_write_data__works(self, client: DataFrameClient, create_table): + id = create_table( + models.CreateTableRequest( + columns=[ + models.Column( + name="index", + data_type=models.DataType.Int32, + column_type=models.ColumnType.Index, + ), + models.Column( + name="value", + data_type=models.DataType.Float64, + column_type=models.ColumnType.Nullable, + ), + models.Column( + name="ignore_me", + data_type=models.DataType.Bool, + column_type=models.ColumnType.Nullable, + ), + ] + ) + ) + + frame = models.DataFrame( + columns=["index", "value", "ignore_me"], + data=[["1", "3.3", "True"], ["2", None, "False"], ["3", "1.1", "True"]], + ) + + client.append_table_data( + id, models.AppendTableDataRequest(frame=frame, end_of_data=True) + ) + + # TODO: Remove mock when service supports flushing + with responses.RequestsMock() as rsps: + rsps.add( + responses.GET, + f"{client.session.base_url}tables/{id}/data", + json={ + "frame": { + "columns": ["index", "value"], + "data": [["3", "1.1"], ["1", "3.3"], ["2", None]], + }, + "totalRowCount": 3, + "continuationToken": None, + }, + ) + + response = client.get_table_data( + id, columns=["index", "value"], order_by=["value"] + ) + + assert response.total_row_count == 3 + assert response.frame == models.DataFrame( + columns=["index", "value"], + data=[["3", "1.1"], ["1", "3.3"], ["2", None]], + ) + + def test__write_invalid_data__raises( + self, client: DataFrameClient, test_tables: List[str] + ): + id = test_tables[0] + + frame = models.DataFrame( + columns=["index", "non_existent_column"], + data=[["1", "2"], ["2", "2"], ["3", "3"]], + ) + + with pytest.raises(ApiException, match="400 Bad Request"): + client.append_table_data(id, models.AppendTableDataRequest(frame=frame))