diff --git a/awswrangler/timestream.py b/awswrangler/timestream.py index b1c44861f..e2b248ae4 100644 --- a/awswrangler/timestream.py +++ b/awswrangler/timestream.py @@ -234,3 +234,199 @@ def query(sql: str, boto3_session: Optional[boto3.Session] = None) -> pd.DataFra if col["type"] == "VARCHAR": df[col["name"]] = df[col["name"]].astype("string") return df + + +def create_database( + database: str, + kms_key_id: Optional[str] = None, + tags: Optional[Dict[str, str]] = None, + boto3_session: Optional[boto3.Session] = None, +) -> str: + """Create a new Timestream database. + + Note + ---- + If the KMS key is not specified, the database will be encrypted with a + Timestream managed KMS key located in your account. + + Parameters + ---------- + database: str + Database name. + kms_key_id: Optional[str] + The KMS key for the database. If the KMS key is not specified, + the database will be encrypted with a Timestream managed KMS key located in your account. + tags: Optional[Dict[str, str]] + Key/Value dict to put on the database. + Tags enable you to categorize databases and/or tables, for example, + by purpose, owner, or environment. + e.g. {"foo": "boo", "bar": "xoo"}) + boto3_session : boto3.Session(), optional + Boto3 Session. The default boto3 Session will be used if boto3_session receive None. + + Returns + ------- + str + The Amazon Resource Name that uniquely identifies this database. (ARN) + + Examples + -------- + Creating a database. + + >>> import awswrangler as wr + >>> arn = wr.timestream.create_database("MyDatabase") + + """ + client: boto3.client = _utils.client(service_name="timestream-write", session=boto3_session) + args: Dict[str, Any] = {"DatabaseName": database} + if kms_key_id is not None: + args["KmsKeyId"] = kms_key_id + if tags is not None: + args["Tags"] = [{"Key": k, "Value": v} for k, v in tags.items()] + response: Dict[str, Dict[str, Any]] = client.create_database(**args) + return cast(str, response["Database"]["Arn"]) + + +def delete_database( + database: str, + boto3_session: Optional[boto3.Session] = None, +) -> None: + """Delete a given Timestream database. This is an irreversible operation. + + After a database is deleted, the time series data from its tables cannot be recovered. + + All tables in the database must be deleted first, or a ValidationException error will be thrown. + + Due to the nature of distributed retries, + the operation can return either success or a ResourceNotFoundException. + Clients should consider them equivalent. + + Parameters + ---------- + database: str + Database name. + boto3_session : boto3.Session(), optional + Boto3 Session. The default boto3 Session will be used if boto3_session receive None. + + Returns + ------- + None + None. + + Examples + -------- + Deleting a database + + >>> import awswrangler as wr + >>> arn = wr.timestream.delete_database("MyDatabase") + + """ + client: boto3.client = _utils.client(service_name="timestream-write", session=boto3_session) + client.delete_database(DatabaseName=database) + + +def create_table( + database: str, + table: str, + memory_retention_hours: int, + magnetic_retention_days: int, + tags: Optional[Dict[str, str]] = None, + boto3_session: Optional[boto3.Session] = None, +) -> str: + """Create a new Timestream database. + + Note + ---- + If the KMS key is not specified, the database will be encrypted with a + Timestream managed KMS key located in your account. + + Parameters + ---------- + database: str + Database name. + table: str + Table name. + memory_retention_hours: int + The duration for which data must be stored in the memory store. + magnetic_retention_days: int + The duration for which data must be stored in the magnetic store. + tags: Optional[Dict[str, str]] + Key/Value dict to put on the table. + Tags enable you to categorize databases and/or tables, for example, + by purpose, owner, or environment. + e.g. {"foo": "boo", "bar": "xoo"}) + boto3_session : boto3.Session(), optional + Boto3 Session. The default boto3 Session will be used if boto3_session receive None. + + Returns + ------- + str + The Amazon Resource Name that uniquely identifies this database. (ARN) + + Examples + -------- + Creating a table. + + >>> import awswrangler as wr + >>> arn = wr.timestream.create_table( + ... database="MyDatabase", + ... table="MyTable", + ... memory_retention_hours=3, + ... magnetic_retention_days=7 + ... ) + + """ + client: boto3.client = _utils.client(service_name="timestream-write", session=boto3_session) + args: Dict[str, Any] = { + "DatabaseName": database, + "TableName": table, + "RetentionProperties": { + "MemoryStoreRetentionPeriodInHours": memory_retention_hours, + "MagneticStoreRetentionPeriodInDays": magnetic_retention_days, + }, + } + if tags is not None: + args["Tags"] = [{"Key": k, "Value": v} for k, v in tags.items()] + response: Dict[str, Dict[str, Any]] = client.create_table(**args) + return cast(str, response["Table"]["Arn"]) + + +def delete_table( + database: str, + table: str, + boto3_session: Optional[boto3.Session] = None, +) -> None: + """Delete a given Timestream table. + + This is an irreversible operation. + + After a Timestream database table is deleted, the time series data stored in the table cannot be recovered. + + Due to the nature of distributed retries, + the operation can return either success or a ResourceNotFoundException. + Clients should consider them equivalent. + + Parameters + ---------- + database: str + Database name. + table: str + Table name. + boto3_session : boto3.Session(), optional + Boto3 Session. The default boto3 Session will be used if boto3_session receive None. + + Returns + ------- + None + None. + + Examples + -------- + Deleting a table + + >>> import awswrangler as wr + >>> arn = wr.timestream.delete_table("MyDatabase", "MyTable") + + """ + client: boto3.client = _utils.client(service_name="timestream-write", session=boto3_session) + client.delete_table(DatabaseName=database, TableName=table) diff --git a/docs/source/api.rst b/docs/source/api.rst index 11c805415..708f19673 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -159,8 +159,12 @@ Amazon Timestream .. autosummary:: :toctree: stubs - write + create_database + create_table + delete_database + delete_table query + write Amazon EMR ---------- diff --git a/tests/conftest.py b/tests/conftest.py index 33e4c9b08..9267bb49f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -231,3 +231,14 @@ def mysql_table(): with con.cursor() as cursor: cursor.execute(f"DROP TABLE IF EXISTS test.{name}") con.close() + + +@pytest.fixture(scope="function") +def timestream_database_and_table(): + name = f"tbl_{get_time_str_with_random_suffix()}" + print(f"Timestream name: {name}") + wr.timestream.create_database(name) + wr.timestream.create_table(name, name, 1, 1) + yield name + wr.timestream.delete_table(name, name) + wr.timestream.delete_database(name) diff --git a/tests/test_timestream.py b/tests/test_timestream.py index 2d75930de..769c6fea2 100644 --- a/tests/test_timestream.py +++ b/tests/test_timestream.py @@ -8,7 +8,8 @@ logging.getLogger("awswrangler").setLevel(logging.DEBUG) -def test_write_simple(): +def test_basic_scenario(timestream_database_and_table): + name = timestream_database_and_table df = pd.DataFrame( { "time": [datetime.now(), datetime.now(), datetime.now()], @@ -19,16 +20,34 @@ def test_write_simple(): ) rejected_records = wr.timestream.write( df=df, - database="sampleDB", - table="sampleTable", + database=name, + table=name, time_col="time", measure_col="measure", dimensions_cols=["dim0", "dim1"], ) assert len(rejected_records) == 0 + df = wr.timestream.query( + f""" + SELECT + 1 as col_int, + try_cast(now() as time) as col_time, + TRUE as col_bool, + current_date as col_date, + 'foo' as col_str, + measure_value::double, + measure_name, + time + FROM "{name}"."{name}" + ORDER BY time + DESC LIMIT 10 + """ + ) + assert df.shape == (3, 8) -def test_write_real(): +def test_real_csv_load_scenario(timestream_database_and_table): + name = timestream_database_and_table df = pd.read_csv( "https://raw.githubusercontent.com/awslabs/amazon-timestream-tools/master/sample_apps/data/sample.csv", names=[ @@ -52,8 +71,8 @@ def test_write_real(): df_memory = df[df.measure_kind == "memory_utilization"] rejected_records = wr.timestream.write( df=df_cpu, - database="sampleDB", - table="sampleTable", + database=name, + table=name, time_col="time", measure_col="measure", dimensions_cols=["index", "region", "az", "hostname"], @@ -61,30 +80,12 @@ def test_write_real(): assert len(rejected_records) == 0 rejected_records = wr.timestream.write( df=df_memory, - database="sampleDB", - table="sampleTable", + database=name, + table=name, time_col="time", measure_col="measure", dimensions_cols=["index", "region", "az", "hostname"], ) assert len(rejected_records) == 0 - - -def test_query(): - df = wr.timestream.query( - """ - SELECT - 1 as col_int, - try_cast(now() as time) as col_time, - TRUE as col_bool, - current_date as col_date, - 'foo' as col_str, - measure_value::double, - measure_name, - time - FROM "sampleDB"."sampleTable" - ORDER BY time - DESC LIMIT 10 - """ - ) - assert df.shape == (10, 8) + df = wr.timestream.query(f'SELECT COUNT(*) AS counter FROM "{name}"."{name}"') + assert df["counter"].iloc[0] == 126_000 diff --git a/tutorials/026 - Amazon Timestream.ipynb b/tutorials/026 - Amazon Timestream.ipynb index 6d79541dd..1e4d745f8 100644 --- a/tutorials/026 - Amazon Timestream.ipynb +++ b/tutorials/026 - Amazon Timestream.ipynb @@ -1,12 +1,33 @@ { "cells": [ { - "cell_type": "markdown", - "metadata": {}, "source": [ "[![AWS Data Wrangler](_static/logo.png \"AWS Data Wrangler\")](https://github.com/awslabs/aws-data-wrangler)\n", "\n", "# 26 - Amazon Timestream" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "source": [ + "## Creating resources" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import awswrangler as wr\n", + "import pandas as pd\n", + "from datetime import datetime\n", + "\n", + "wr.timestream.create_database(\"sampleDB\")\n", + "wr.timestream.create_table(\"sampleDB\", \"sampleTable\", memory_retention_hours=1, magnetic_retention_days=1);" ] }, { @@ -18,22 +39,18 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 11, "metadata": {}, "outputs": [ { - "name": "stdout", "output_type": "stream", + "name": "stdout", "text": [ "Number of rejected records: 0\n" ] } ], "source": [ - "import awswrangler as wr\n", - "import pandas as pd\n", - "from datetime import datetime\n", - "\n", "df = pd.DataFrame(\n", " {\n", " \"time\": [datetime.now(), datetime.now(), datetime.now()],\n", @@ -64,72 +81,22 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 12, "metadata": {}, "outputs": [ { + "output_type": "execute_result", "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
timemeasure_value::doubledim0dim1
02020-12-06 21:00:38.5821.0foo1
12020-12-06 21:00:38.5821.1boo2
22020-12-06 21:00:38.5821.2bar3
\n", - "
" - ], "text/plain": [ " time measure_value::double dim0 dim1\n", - "0 2020-12-06 21:00:38.582 1.0 foo 1\n", - "1 2020-12-06 21:00:38.582 1.1 boo 2\n", - "2 2020-12-06 21:00:38.582 1.2 bar 3" - ] + "0 2020-12-08 19:15:32.468 1.0 foo 1\n", + "1 2020-12-08 19:15:32.468 1.2 bar 3\n", + "2 2020-12-08 19:15:32.468 1.1 boo 2" + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
timemeasure_value::doubledim0dim1
02020-12-08 19:15:32.4681.0foo1
12020-12-08 19:15:32.4681.2bar3
22020-12-08 19:15:32.4681.1boo2
\n
" }, - "execution_count": 2, "metadata": {}, - "output_type": "execute_result" + "execution_count": 12 } ], "source": [ @@ -137,13 +104,34 @@ " 'SELECT time, measure_value::double, dim0, dim1 FROM \"sampleDB\".\"sampleTable\" ORDER BY time DESC LIMIT 3'\n", ")" ] + }, + { + "source": [ + "## Deleting resources" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "wr.timestream.delete_table(\"sampleDB\", \"sampleTable\")\n", + "wr.timestream.delete_database(\"sampleDB\")" + ] } ], "metadata": { "kernelspec": { - "display_name": "conda_python3", - "language": "python", - "name": "conda_python3" + "name": "python3", + "display_name": "Python 3.6.12 64-bit ('.venv': venv)", + "metadata": { + "interpreter": { + "hash": "d615d0bdd35e5a4938de94437a0942aa54f5a0cb2fbaf63c7f5fc7fc7d82b041" + } + } }, "language_info": { "codemirror_mode": { @@ -155,7 +143,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.10" + "version": "3.6.12-final" }, "pycharm": { "stem_cell": { diff --git a/tutorials/027 - Amazon Timestream 2.ipynb b/tutorials/027 - Amazon Timestream 2.ipynb index fba0db1e0..59c33c980 100644 --- a/tutorials/027 - Amazon Timestream 2.ipynb +++ b/tutorials/027 - Amazon Timestream 2.ipynb @@ -22,151 +22,8 @@ "metadata": {}, "outputs": [ { + "output_type": "execute_result", "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
indexregionazhostnamemeasure_kindmeasuretime
00us-east-1us-east-1ahost-fj2hxcpu_utilization21.3943632020-12-06 21:21:17.846716
11us-east-1us-east-1ahost-fj2hxmemory_utilization68.5634202020-12-06 21:21:17.846716
22us-east-1us-east-1ahost-6kMPEcpu_utilization17.1445792020-12-06 21:21:17.846716
33us-east-1us-east-1ahost-6kMPEmemory_utilization73.5078702020-12-06 21:21:17.846716
44us-east-1us-east-1ahost-sxj7Xcpu_utilization26.5848652020-12-06 21:21:17.846716
........................
125995125995eu-north-1eu-north-1chost-De8RBmemory_utilization68.0634682020-12-06 21:21:17.846716
125996125996eu-north-1eu-north-1chost-2z8tnmemory_utilization72.2036802020-12-06 21:21:17.846716
125997125997eu-north-1eu-north-1chost-2z8tncpu_utilization29.2122192020-12-06 21:21:17.846716
125998125998eu-north-1eu-north-1chost-9FczWmemory_utilization71.7461342020-12-06 21:21:17.846716
125999125999eu-north-1eu-north-1chost-9FczWcpu_utilization1.6777932020-12-06 21:21:17.846716
\n", - "

126000 rows × 7 columns

\n", - "
" - ], "text/plain": [ " index region az hostname measure_kind \\\n", "0 0 us-east-1 us-east-1a host-fj2hx cpu_utilization \n", @@ -182,24 +39,24 @@ "125999 125999 eu-north-1 eu-north-1c host-9FczW cpu_utilization \n", "\n", " measure time \n", - "0 21.394363 2020-12-06 21:21:17.846716 \n", - "1 68.563420 2020-12-06 21:21:17.846716 \n", - "2 17.144579 2020-12-06 21:21:17.846716 \n", - "3 73.507870 2020-12-06 21:21:17.846716 \n", - "4 26.584865 2020-12-06 21:21:17.846716 \n", + "0 21.394363 2020-12-08 16:18:47.599597 \n", + "1 68.563420 2020-12-08 16:18:47.599597 \n", + "2 17.144579 2020-12-08 16:18:47.599597 \n", + "3 73.507870 2020-12-08 16:18:47.599597 \n", + "4 26.584865 2020-12-08 16:18:47.599597 \n", "... ... ... \n", - "125995 68.063468 2020-12-06 21:21:17.846716 \n", - "125996 72.203680 2020-12-06 21:21:17.846716 \n", - "125997 29.212219 2020-12-06 21:21:17.846716 \n", - "125998 71.746134 2020-12-06 21:21:17.846716 \n", - "125999 1.677793 2020-12-06 21:21:17.846716 \n", + "125995 68.063468 2020-12-08 16:18:47.599597 \n", + "125996 72.203680 2020-12-08 16:18:47.599597 \n", + "125997 29.212219 2020-12-08 16:18:47.599597 \n", + "125998 71.746134 2020-12-08 16:18:47.599597 \n", + "125999 1.677793 2020-12-08 16:18:47.599597 \n", "\n", "[126000 rows x 7 columns]" - ] + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
indexregionazhostnamemeasure_kindmeasuretime
00us-east-1us-east-1ahost-fj2hxcpu_utilization21.3943632020-12-08 16:18:47.599597
11us-east-1us-east-1ahost-fj2hxmemory_utilization68.5634202020-12-08 16:18:47.599597
22us-east-1us-east-1ahost-6kMPEcpu_utilization17.1445792020-12-08 16:18:47.599597
33us-east-1us-east-1ahost-6kMPEmemory_utilization73.5078702020-12-08 16:18:47.599597
44us-east-1us-east-1ahost-sxj7Xcpu_utilization26.5848652020-12-08 16:18:47.599597
........................
125995125995eu-north-1eu-north-1chost-De8RBmemory_utilization68.0634682020-12-08 16:18:47.599597
125996125996eu-north-1eu-north-1chost-2z8tnmemory_utilization72.2036802020-12-08 16:18:47.599597
125997125997eu-north-1eu-north-1chost-2z8tncpu_utilization29.2122192020-12-08 16:18:47.599597
125998125998eu-north-1eu-north-1chost-9FczWmemory_utilization71.7461342020-12-08 16:18:47.599597
125999125999eu-north-1eu-north-1chost-9FczWcpu_utilization1.6777932020-12-08 16:18:47.599597
\n

126000 rows × 7 columns

\n
" }, - "execution_count": 1, "metadata": {}, - "output_type": "execute_result" + "execution_count": 1 } ], "source": [ @@ -230,6 +87,23 @@ "df" ] }, + { + "source": [ + "## Creating resources" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "wr.timestream.create_database(\"sampleDB\")\n", + "wr.timestream.create_table(\"sampleDB\", \"sampleTable\", memory_retention_hours=1, magnetic_retention_days=1);" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -239,13 +113,59 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " index region az hostname measure_kind \\\n", + "0 0 us-east-1 us-east-1a host-fj2hx cpu_utilization \n", + "2 2 us-east-1 us-east-1a host-6kMPE cpu_utilization \n", + "4 4 us-east-1 us-east-1a host-sxj7X cpu_utilization \n", + "6 6 us-east-1 us-east-1a host-ExOui cpu_utilization \n", + "8 8 us-east-1 us-east-1a host-Bwb3j cpu_utilization \n", + "... ... ... ... ... ... \n", + "125990 125990 eu-north-1 eu-north-1c host-aPtc6 cpu_utilization \n", + "125992 125992 eu-north-1 eu-north-1c host-7ZF9L cpu_utilization \n", + "125994 125994 eu-north-1 eu-north-1c host-De8RB cpu_utilization \n", + "125997 125997 eu-north-1 eu-north-1c host-2z8tn cpu_utilization \n", + "125999 125999 eu-north-1 eu-north-1c host-9FczW cpu_utilization \n", + "\n", + " cpu_utilization time \n", + "0 21.394363 2020-12-08 16:18:47.599597 \n", + "2 17.144579 2020-12-08 16:18:47.599597 \n", + "4 26.584865 2020-12-08 16:18:47.599597 \n", + "6 52.930970 2020-12-08 16:18:47.599597 \n", + "8 99.134110 2020-12-08 16:18:47.599597 \n", + "... ... ... \n", + "125990 89.566125 2020-12-08 16:18:47.599597 \n", + "125992 75.510598 2020-12-08 16:18:47.599597 \n", + "125994 2.771261 2020-12-08 16:18:47.599597 \n", + "125997 29.212219 2020-12-08 16:18:47.599597 \n", + "125999 1.677793 2020-12-08 16:18:47.599597 \n", + "\n", + "[63000 rows x 7 columns]" + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
indexregionazhostnamemeasure_kindcpu_utilizationtime
00us-east-1us-east-1ahost-fj2hxcpu_utilization21.3943632020-12-08 16:18:47.599597
22us-east-1us-east-1ahost-6kMPEcpu_utilization17.1445792020-12-08 16:18:47.599597
44us-east-1us-east-1ahost-sxj7Xcpu_utilization26.5848652020-12-08 16:18:47.599597
66us-east-1us-east-1ahost-ExOuicpu_utilization52.9309702020-12-08 16:18:47.599597
88us-east-1us-east-1ahost-Bwb3jcpu_utilization99.1341102020-12-08 16:18:47.599597
........................
125990125990eu-north-1eu-north-1chost-aPtc6cpu_utilization89.5661252020-12-08 16:18:47.599597
125992125992eu-north-1eu-north-1chost-7ZF9Lcpu_utilization75.5105982020-12-08 16:18:47.599597
125994125994eu-north-1eu-north-1chost-De8RBcpu_utilization2.7712612020-12-08 16:18:47.599597
125997125997eu-north-1eu-north-1chost-2z8tncpu_utilization29.2122192020-12-08 16:18:47.599597
125999125999eu-north-1eu-north-1chost-9FczWcpu_utilization1.6777932020-12-08 16:18:47.599597
\n

63000 rows × 7 columns

\n
" + }, + "metadata": {}, + "execution_count": 3 + } + ], "source": [ "df_cpu = df[df.measure_kind == \"cpu_utilization\"].copy()\n", "df_cpu.rename(columns={\"measure\": \"cpu_utilization\"}, inplace=True)\n", - "\n", + "df_cpu" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ "rejected_records = wr.timestream.write(\n", " df=df_cpu,\n", " database=\"sampleDB\",\n", @@ -267,13 +187,60 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " index region az hostname measure_kind \\\n", + "1 1 us-east-1 us-east-1a host-fj2hx memory_utilization \n", + "3 3 us-east-1 us-east-1a host-6kMPE memory_utilization \n", + "5 5 us-east-1 us-east-1a host-sxj7X memory_utilization \n", + "7 7 us-east-1 us-east-1a host-ExOui memory_utilization \n", + "9 9 us-east-1 us-east-1a host-Bwb3j memory_utilization \n", + "... ... ... ... ... ... \n", + "125991 125991 eu-north-1 eu-north-1c host-aPtc6 memory_utilization \n", + "125993 125993 eu-north-1 eu-north-1c host-7ZF9L memory_utilization \n", + "125995 125995 eu-north-1 eu-north-1c host-De8RB memory_utilization \n", + "125996 125996 eu-north-1 eu-north-1c host-2z8tn memory_utilization \n", + "125998 125998 eu-north-1 eu-north-1c host-9FczW memory_utilization \n", + "\n", + " memory_utilization time \n", + "1 68.563420 2020-12-08 16:18:47.599597 \n", + "3 73.507870 2020-12-08 16:18:47.599597 \n", + "5 22.401424 2020-12-08 16:18:47.599597 \n", + "7 45.440135 2020-12-08 16:18:47.599597 \n", + "9 15.042701 2020-12-08 16:18:47.599597 \n", + "... ... ... \n", + "125991 75.686739 2020-12-08 16:18:47.599597 \n", + "125993 18.386152 2020-12-08 16:18:47.599597 \n", + "125995 68.063468 2020-12-08 16:18:47.599597 \n", + "125996 72.203680 2020-12-08 16:18:47.599597 \n", + "125998 71.746134 2020-12-08 16:18:47.599597 \n", + "\n", + "[63000 rows x 7 columns]" + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
indexregionazhostnamemeasure_kindmemory_utilizationtime
11us-east-1us-east-1ahost-fj2hxmemory_utilization68.5634202020-12-08 16:18:47.599597
33us-east-1us-east-1ahost-6kMPEmemory_utilization73.5078702020-12-08 16:18:47.599597
55us-east-1us-east-1ahost-sxj7Xmemory_utilization22.4014242020-12-08 16:18:47.599597
77us-east-1us-east-1ahost-ExOuimemory_utilization45.4401352020-12-08 16:18:47.599597
99us-east-1us-east-1ahost-Bwb3jmemory_utilization15.0427012020-12-08 16:18:47.599597
........................
125991125991eu-north-1eu-north-1chost-aPtc6memory_utilization75.6867392020-12-08 16:18:47.599597
125993125993eu-north-1eu-north-1chost-7ZF9Lmemory_utilization18.3861522020-12-08 16:18:47.599597
125995125995eu-north-1eu-north-1chost-De8RBmemory_utilization68.0634682020-12-08 16:18:47.599597
125996125996eu-north-1eu-north-1chost-2z8tnmemory_utilization72.2036802020-12-08 16:18:47.599597
125998125998eu-north-1eu-north-1chost-9FczWmemory_utilization71.7461342020-12-08 16:18:47.599597
\n

63000 rows × 7 columns

\n
" + }, + "metadata": {}, + "execution_count": 5 + } + ], "source": [ "df_memory = df[df.measure_kind == \"memory_utilization\"].copy()\n", "df_memory.rename(columns={\"measure\": \"memory_utilization\"}, inplace=True)\n", "\n", + "df_memory" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ "rejected_records = wr.timestream.write(\n", " df=df_memory,\n", " database=\"sampleDB\",\n", @@ -295,162 +262,41 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "metadata": {}, "outputs": [ { + "output_type": "execute_result", "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
hostnameregionazmeasure_namemeasure_value::doubletime
0host-dCRm0us-west-1us-west-1acpu_utilization85.7938092020-12-06 21:21:17.847
1host-JI1Tcus-east-2us-east-2acpu_utilization61.5847962020-12-06 21:21:17.847
2host-YL3W0eu-central-1eu-central-1acpu_utilization63.9213832020-12-06 21:21:17.847
3host-Axf0gus-west-2us-west-2acpu_utilization36.9624732020-12-06 21:21:17.847
4host-Bwb3jus-east-1us-east-1acpu_utilization45.0828672020-12-06 21:21:17.847
5host-39Hemap-east-1ap-east-1ccpu_utilization86.0574932020-12-06 21:21:17.847
6host-93VEWus-east-2us-east-2ccpu_utilization20.6918062020-12-06 21:21:17.847
7host-lPyRjeu-central-1eu-central-1bcpu_utilization97.4246922020-12-06 21:21:17.847
8host-c45ybus-west-2us-west-2acpu_utilization87.2054142020-12-06 21:21:17.847
9host-bSljVus-east-2us-east-2acpu_utilization27.8257042020-12-06 21:21:17.847
\n", - "
" - ], "text/plain": [ - " hostname region az measure_name \\\n", - "0 host-dCRm0 us-west-1 us-west-1a cpu_utilization \n", - "1 host-JI1Tc us-east-2 us-east-2a cpu_utilization \n", - "2 host-YL3W0 eu-central-1 eu-central-1a cpu_utilization \n", - "3 host-Axf0g us-west-2 us-west-2a cpu_utilization \n", - "4 host-Bwb3j us-east-1 us-east-1a cpu_utilization \n", - "5 host-39Hem ap-east-1 ap-east-1c cpu_utilization \n", - "6 host-93VEW us-east-2 us-east-2c cpu_utilization \n", - "7 host-lPyRj eu-central-1 eu-central-1b cpu_utilization \n", - "8 host-c45yb us-west-2 us-west-2a cpu_utilization \n", - "9 host-bSljV us-east-2 us-east-2a cpu_utilization \n", + " hostname region az measure_name \\\n", + "0 host-OgvFx us-west-1 us-west-1a cpu_utilization \n", + "1 host-rZUNx eu-north-1 eu-north-1a cpu_utilization \n", + "2 host-t1kAB us-east-2 us-east-2b cpu_utilization \n", + "3 host-RdQRf us-east-1 us-east-1c cpu_utilization \n", + "4 host-4Llhu us-east-1 us-east-1c cpu_utilization \n", + "5 host-2plqa us-west-1 us-west-1a cpu_utilization \n", + "6 host-J3Q4z us-east-1 us-east-1b cpu_utilization \n", + "7 host-VIR5T ap-east-1 ap-east-1a cpu_utilization \n", + "8 host-G042D us-east-1 us-east-1c cpu_utilization \n", + "9 host-8EBHm us-west-2 us-west-2c cpu_utilization \n", "\n", " measure_value::double time \n", - "0 85.793809 2020-12-06 21:21:17.847 \n", - "1 61.584796 2020-12-06 21:21:17.847 \n", - "2 63.921383 2020-12-06 21:21:17.847 \n", - "3 36.962473 2020-12-06 21:21:17.847 \n", - "4 45.082867 2020-12-06 21:21:17.847 \n", - "5 86.057493 2020-12-06 21:21:17.847 \n", - "6 20.691806 2020-12-06 21:21:17.847 \n", - "7 97.424692 2020-12-06 21:21:17.847 \n", - "8 87.205414 2020-12-06 21:21:17.847 \n", - "9 27.825704 2020-12-06 21:21:17.847 " - ] + "0 39.617911 2020-12-08 19:18:47.600 \n", + "1 30.793332 2020-12-08 19:18:47.600 \n", + "2 74.453239 2020-12-08 19:18:47.600 \n", + "3 76.984448 2020-12-08 19:18:47.600 \n", + "4 41.862733 2020-12-08 19:18:47.600 \n", + "5 34.864762 2020-12-08 19:18:47.600 \n", + "6 71.574266 2020-12-08 19:18:47.600 \n", + "7 14.017491 2020-12-08 19:18:47.600 \n", + "8 60.199068 2020-12-08 19:18:47.600 \n", + "9 96.631624 2020-12-08 19:18:47.600 " + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
hostnameregionazmeasure_namemeasure_value::doubletime
0host-OgvFxus-west-1us-west-1acpu_utilization39.6179112020-12-08 19:18:47.600
1host-rZUNxeu-north-1eu-north-1acpu_utilization30.7933322020-12-08 19:18:47.600
2host-t1kABus-east-2us-east-2bcpu_utilization74.4532392020-12-08 19:18:47.600
3host-RdQRfus-east-1us-east-1ccpu_utilization76.9844482020-12-08 19:18:47.600
4host-4Llhuus-east-1us-east-1ccpu_utilization41.8627332020-12-08 19:18:47.600
5host-2plqaus-west-1us-west-1acpu_utilization34.8647622020-12-08 19:18:47.600
6host-J3Q4zus-east-1us-east-1bcpu_utilization71.5742662020-12-08 19:18:47.600
7host-VIR5Tap-east-1ap-east-1acpu_utilization14.0174912020-12-08 19:18:47.600
8host-G042Dus-east-1us-east-1ccpu_utilization60.1990682020-12-08 19:18:47.600
9host-8EBHmus-west-2us-west-2ccpu_utilization96.6316242020-12-08 19:18:47.600
\n
" }, - "execution_count": 4, "metadata": {}, - "output_type": "execute_result" + "execution_count": 7 } ], "source": [ @@ -473,162 +319,41 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { + "output_type": "execute_result", "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
hostnameregionazmeasure_namemeasure_value::doubletime
0host-AyzHSeu-central-1eu-central-1bmemory_utilization83.2098702020-12-06 21:21:17.847
1host-EREJkap-east-1ap-east-1amemory_utilization36.5290792020-12-06 21:21:17.847
2host-YL3W0eu-central-1eu-central-1amemory_utilization32.0101172020-12-06 21:21:17.847
3host-BM6B6eu-north-1eu-north-1amemory_utilization80.9813022020-12-06 21:21:17.847
4host-HE2Gmeu-north-1eu-north-1amemory_utilization22.5558522020-12-06 21:21:17.847
5host-RdQRfus-east-1us-east-1cmemory_utilization47.3567742020-12-06 21:21:17.847
6host-24Gjuus-east-1us-east-1bmemory_utilization56.8680002020-12-06 21:21:17.847
7host-WgAuLus-east-2us-east-2amemory_utilization57.5623152020-12-06 21:21:17.847
8host-t4XZjeu-north-1eu-north-1bmemory_utilization66.3075382020-12-06 21:21:17.847
9host-wWlQ3eu-north-1eu-north-1amemory_utilization18.8070512020-12-06 21:21:17.847
\n", - "
" - ], "text/plain": [ - " hostname region az measure_name \\\n", - "0 host-AyzHS eu-central-1 eu-central-1b memory_utilization \n", - "1 host-EREJk ap-east-1 ap-east-1a memory_utilization \n", - "2 host-YL3W0 eu-central-1 eu-central-1a memory_utilization \n", - "3 host-BM6B6 eu-north-1 eu-north-1a memory_utilization \n", - "4 host-HE2Gm eu-north-1 eu-north-1a memory_utilization \n", - "5 host-RdQRf us-east-1 us-east-1c memory_utilization \n", - "6 host-24Gju us-east-1 us-east-1b memory_utilization \n", - "7 host-WgAuL us-east-2 us-east-2a memory_utilization \n", - "8 host-t4XZj eu-north-1 eu-north-1b memory_utilization \n", - "9 host-wWlQ3 eu-north-1 eu-north-1a memory_utilization \n", + " hostname region az measure_name \\\n", + "0 host-7c897 us-west-2 us-west-2b memory_utilization \n", + "1 host-2z8tn eu-north-1 eu-north-1c memory_utilization \n", + "2 host-J3Q4z us-east-1 us-east-1b memory_utilization \n", + "3 host-mjrQb us-east-1 us-east-1b memory_utilization \n", + "4 host-AyWSI us-east-1 us-east-1c memory_utilization \n", + "5 host-Axf0g us-west-2 us-west-2a memory_utilization \n", + "6 host-ilMBa us-east-2 us-east-2b memory_utilization \n", + "7 host-CWdXX us-west-2 us-west-2c memory_utilization \n", + "8 host-8EBHm us-west-2 us-west-2c memory_utilization \n", + "9 host-dRIJj us-east-1 us-east-1c memory_utilization \n", "\n", " measure_value::double time \n", - "0 83.209870 2020-12-06 21:21:17.847 \n", - "1 36.529079 2020-12-06 21:21:17.847 \n", - "2 32.010117 2020-12-06 21:21:17.847 \n", - "3 80.981302 2020-12-06 21:21:17.847 \n", - "4 22.555852 2020-12-06 21:21:17.847 \n", - "5 47.356774 2020-12-06 21:21:17.847 \n", - "6 56.868000 2020-12-06 21:21:17.847 \n", - "7 57.562315 2020-12-06 21:21:17.847 \n", - "8 66.307538 2020-12-06 21:21:17.847 \n", - "9 18.807051 2020-12-06 21:21:17.847 " - ] + "0 63.427726 2020-12-08 19:18:47.600 \n", + "1 41.071368 2020-12-08 19:18:47.600 \n", + "2 23.944388 2020-12-08 19:18:47.600 \n", + "3 69.173431 2020-12-08 19:18:47.600 \n", + "4 75.591467 2020-12-08 19:18:47.600 \n", + "5 29.720739 2020-12-08 19:18:47.600 \n", + "6 71.544134 2020-12-08 19:18:47.600 \n", + "7 79.792799 2020-12-08 19:18:47.600 \n", + "8 66.082554 2020-12-08 19:18:47.600 \n", + "9 86.748960 2020-12-08 19:18:47.600 " + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
hostnameregionazmeasure_namemeasure_value::doubletime
0host-7c897us-west-2us-west-2bmemory_utilization63.4277262020-12-08 19:18:47.600
1host-2z8tneu-north-1eu-north-1cmemory_utilization41.0713682020-12-08 19:18:47.600
2host-J3Q4zus-east-1us-east-1bmemory_utilization23.9443882020-12-08 19:18:47.600
3host-mjrQbus-east-1us-east-1bmemory_utilization69.1734312020-12-08 19:18:47.600
4host-AyWSIus-east-1us-east-1cmemory_utilization75.5914672020-12-08 19:18:47.600
5host-Axf0gus-west-2us-west-2amemory_utilization29.7207392020-12-08 19:18:47.600
6host-ilMBaus-east-2us-east-2bmemory_utilization71.5441342020-12-08 19:18:47.600
7host-CWdXXus-west-2us-west-2cmemory_utilization79.7927992020-12-08 19:18:47.600
8host-8EBHmus-west-2us-west-2cmemory_utilization66.0825542020-12-08 19:18:47.600
9host-dRIJjus-east-1us-east-1cmemory_utilization86.7489602020-12-08 19:18:47.600
\n
" }, - "execution_count": 5, "metadata": {}, - "output_type": "execute_result" + "execution_count": 8 } ], "source": [ @@ -641,13 +366,34 @@ " LIMIT 10\n", "\"\"\")" ] + }, + { + "source": [ + "## Deleting resources" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "wr.timestream.delete_table(\"sampleDB\", \"sampleTable\")\n", + "wr.timestream.delete_database(\"sampleDB\")" + ] } ], "metadata": { "kernelspec": { - "display_name": "conda_python3", - "language": "python", - "name": "conda_python3" + "name": "python3", + "display_name": "Python 3.6.12 64-bit ('.venv': venv)", + "metadata": { + "interpreter": { + "hash": "d615d0bdd35e5a4938de94437a0942aa54f5a0cb2fbaf63c7f5fc7fc7d82b041" + } + } }, "language_info": { "codemirror_mode": { @@ -659,7 +405,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.10" + "version": "3.6.12-final" }, "pycharm": { "stem_cell": {