From 62623bf0806c0341c1dbb6026cae5f9e8ba80eab Mon Sep 17 00:00:00 2001 From: larkee Date: Tue, 23 Feb 2021 12:56:27 +1100 Subject: [PATCH 01/10] feat: add sample for commit stats --- samples/samples/snippets.py | 44 ++++++++++++++++++++++++++++++++ samples/samples/snippets_test.py | 7 +++++ 2 files changed, 51 insertions(+) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index f0379c0210..a1e608f3b5 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -24,6 +24,7 @@ import base64 import datetime import decimal +import logging from google.cloud import spanner from google.cloud.spanner_v1 import param_types @@ -969,6 +970,46 @@ def insert_singers(transaction): # [END spanner_dml_standard_insert] +# [START spanner_get_commit_stats] +def log_commit_stats(instance_id, database_id): + """Inserts sample data using DML and displays the commit statistics. """ + # By default, commit statistics are logged via stdout at level Info. + # This sample uses a custom logger to access the commit statistics. + class CommitStatsSampleLogger(logging.Logger): + + def __init__(self): + self._last_commit_stats = None + + def info(self, msg, *args, **kwargs): + if kwargs['extras'] and 'commitStats' in kwargs['extras']: + self._last_commit_stats = kwargs['extras']['commitStats'] + super().self.info(msg) + + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id, logger=CommitStatsSampleLogger()) + database.log_commit_stats = True + + def insert_singers(transaction): + row_ct = transaction.execute_update( + "INSERT Singers (SingerId, FirstName, LastName) " + " VALUES (110, 'Virginia', 'Watson')" + ) + + print("{} record(s) inserted.".format(row_ct)) + + database.run_in_transaction(insert_singers) + commit_stats = database.logger._last_commit_stats + print( + "{} mutation(s) in transaction.".format( + commit_stats.mutation_count + ) + ) + + +# [END spanner_get_commit_stats] + + def update_data_with_dml(instance_id, database_id): """Updates sample data from the database using a DML statement. """ # [START spanner_dml_standard_update] @@ -1710,6 +1751,7 @@ def create_client_with_query_options(instance_id, database_id): "query_nested_struct_field", help=query_nested_struct_field.__doc__ ) subparsers.add_parser("insert_data_with_dml", help=insert_data_with_dml.__doc__) + subparsers.add_parser("log_commit_stats", help=log_commit_stats.__doc__) subparsers.add_parser("update_data_with_dml", help=update_data_with_dml.__doc__) subparsers.add_parser("delete_data_with_dml", help=delete_data_with_dml.__doc__) subparsers.add_parser( @@ -1820,6 +1862,8 @@ def create_client_with_query_options(instance_id, database_id): query_nested_struct_field(args.instance_id, args.database_id) elif args.command == "insert_data_with_dml": insert_data_with_dml(args.instance_id, args.database_id) + elif args.command == "log_commit_stats": + log_commit_stats(args.instance_id, args.database_id) elif args.command == "update_data_with_dml": update_data_with_dml(args.instance_id, args.database_id) elif args.command == "delete_data_with_dml": diff --git a/samples/samples/snippets_test.py b/samples/samples/snippets_test.py index 237389c8b1..07031afccd 100644 --- a/samples/samples/snippets_test.py +++ b/samples/samples/snippets_test.py @@ -236,6 +236,13 @@ def test_insert_data_with_dml(capsys): assert "1 record(s) inserted." in out +def test_log_commit_stats(capsys): + snippets.log_commit_stats(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert "1 record(s) inserted." in out + assert "3 mutations(s) in transaction." in out + + def test_update_data_with_dml(capsys): snippets.update_data_with_dml(INSTANCE_ID, DATABASE_ID) out, _ = capsys.readouterr() From a57f69d657fb92331a60ea1ea7efdfd90c46970b Mon Sep 17 00:00:00 2001 From: larkee Date: Tue, 23 Feb 2021 21:50:16 +1100 Subject: [PATCH 02/10] fix: use correct kwarg --- samples/samples/snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index a1e608f3b5..dba8d6e00c 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -981,8 +981,8 @@ def __init__(self): self._last_commit_stats = None def info(self, msg, *args, **kwargs): - if kwargs['extras'] and 'commitStats' in kwargs['extras']: - self._last_commit_stats = kwargs['extras']['commitStats'] + if kwargs['extra'] and 'commitStats' in kwargs['extra']: + self._last_commit_stats = kwargs['extra']['commitStats'] super().self.info(msg) spanner_client = spanner.Client() From fdc51a8ccac948243481474d920a96b859dbbb8a Mon Sep 17 00:00:00 2001 From: larkee Date: Tue, 23 Feb 2021 22:13:21 +1100 Subject: [PATCH 03/10] fix: correct super call --- samples/samples/snippets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index dba8d6e00c..170dd7e33d 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -983,7 +983,7 @@ def __init__(self): def info(self, msg, *args, **kwargs): if kwargs['extra'] and 'commitStats' in kwargs['extra']: self._last_commit_stats = kwargs['extra']['commitStats'] - super().self.info(msg) + super().info(msg) spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) From bcaab001e606ca0af5aa6c1e25abf9975f490444 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 24 Feb 2021 13:21:10 +1100 Subject: [PATCH 04/10] fix: add missing super init call --- samples/samples/snippets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index 170dd7e33d..c8b9797ee2 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -979,6 +979,7 @@ class CommitStatsSampleLogger(logging.Logger): def __init__(self): self._last_commit_stats = None + super().__init__() def info(self, msg, *args, **kwargs): if kwargs['extra'] and 'commitStats' in kwargs['extra']: From 96f7bc246d9b0536b9a6f76aa27caaf607a41dcc Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 24 Feb 2021 14:48:06 +1100 Subject: [PATCH 05/10] fix: update super init call --- samples/samples/snippets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index c8b9797ee2..15c6efa04a 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -979,7 +979,7 @@ class CommitStatsSampleLogger(logging.Logger): def __init__(self): self._last_commit_stats = None - super().__init__() + super().__init__("commit_stats_sample") def info(self, msg, *args, **kwargs): if kwargs['extra'] and 'commitStats' in kwargs['extra']: From c1dedb6343caf15e911eacbe1c04bf910f5ebf02 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 24 Feb 2021 19:01:20 +1100 Subject: [PATCH 06/10] fix: use correct key --- samples/samples/snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index 15c6efa04a..fe82076fde 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -982,8 +982,8 @@ def __init__(self): super().__init__("commit_stats_sample") def info(self, msg, *args, **kwargs): - if kwargs['extra'] and 'commitStats' in kwargs['extra']: - self._last_commit_stats = kwargs['extra']['commitStats'] + if kwargs['extra'] and 'commit_stats' in kwargs['extra']: + self._last_commit_stats = kwargs['extra']['commit_stats'] super().info(msg) spanner_client = spanner.Client() From c449089c6eccdd840a968bd4207c0b721918d53e Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 24 Feb 2021 19:01:49 +1100 Subject: [PATCH 07/10] refactor: remove testing file --- test.py | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 7888bbd090..0000000000 --- a/test.py +++ /dev/null @@ -1,53 +0,0 @@ -import base64 -import time -from google.cloud import spanner -from google.auth.credentials import AnonymousCredentials - -instance_id = 'test-instance' -database_id = 'test-db' - -spanner_client = spanner.Client( - project='test-project', - client_options={"api_endpoint": 'localhost:9010'}, - credentials=AnonymousCredentials() -) - -instance = spanner_client.instance(instance_id) -op = instance.create() -op.result() - -database = instance.database(database_id, ddl_statements=[ - "CREATE TABLE Test (id STRING(36) NOT NULL, megafield BYTES(MAX)) PRIMARY KEY (id)" -]) -op = database.create() -op.result() - -# This must be large enough that the SDK will split the megafield payload across two query chunks -# and try to recombine them, causing the error: -data = base64.standard_b64encode(("a" * 1000000).encode("utf8")) - -try: - with database.batch() as batch: - batch.insert( - table="Test", - columns=("id", "megafield"), - values=[ - (1, data), - ], - ) - - with database.snapshot() as snapshot: - toc = time.time() - results = snapshot.execute_sql( - "SELECT * FROM Test" - ) - tic = time.time() - - print("TIME: ", tic - toc) - - for row in results: - print("Id: ", row[0]) - print("Megafield: ", row[1][:100]) -finally: - database.drop() - instance.delete() \ No newline at end of file From edb201b1484ddcea7eae20c4192bff252e5bc485 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 24 Feb 2021 19:35:09 +1100 Subject: [PATCH 08/10] test: fix typo --- samples/samples/snippets_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/samples/snippets_test.py b/samples/samples/snippets_test.py index 07031afccd..ee8c6ebe23 100644 --- a/samples/samples/snippets_test.py +++ b/samples/samples/snippets_test.py @@ -240,7 +240,7 @@ def test_log_commit_stats(capsys): snippets.log_commit_stats(INSTANCE_ID, DATABASE_ID) out, _ = capsys.readouterr() assert "1 record(s) inserted." in out - assert "3 mutations(s) in transaction." in out + assert "3 mutation(s) in transaction." in out def test_update_data_with_dml(capsys): From a738f62718048e0b40090776bc319356dbe11038 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Thu, 25 Feb 2021 12:27:53 +1100 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: skuruppu --- samples/samples/snippets.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index fe82076fde..fed8499535 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -976,14 +976,13 @@ def log_commit_stats(instance_id, database_id): # By default, commit statistics are logged via stdout at level Info. # This sample uses a custom logger to access the commit statistics. class CommitStatsSampleLogger(logging.Logger): - def __init__(self): self._last_commit_stats = None super().__init__("commit_stats_sample") def info(self, msg, *args, **kwargs): - if kwargs['extra'] and 'commit_stats' in kwargs['extra']: - self._last_commit_stats = kwargs['extra']['commit_stats'] + if kwargs["extra"] and "commit_stats" in kwargs["extra"]: + self._last_commit_stats = kwargs["extra"]["commit_stats"] super().info(msg) spanner_client = spanner.Client() @@ -1006,8 +1005,6 @@ def insert_singers(transaction): commit_stats.mutation_count ) ) - - # [END spanner_get_commit_stats] From 13d1309f9bc7bca7d42e9addaec497e11a4aab13 Mon Sep 17 00:00:00 2001 From: larkee Date: Thu, 25 Feb 2021 14:39:27 +1100 Subject: [PATCH 10/10] refactor: make last_commit_stats public --- samples/samples/snippets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index fed8499535..9a94e85a9b 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -977,12 +977,12 @@ def log_commit_stats(instance_id, database_id): # This sample uses a custom logger to access the commit statistics. class CommitStatsSampleLogger(logging.Logger): def __init__(self): - self._last_commit_stats = None + self.last_commit_stats = None super().__init__("commit_stats_sample") def info(self, msg, *args, **kwargs): if kwargs["extra"] and "commit_stats" in kwargs["extra"]: - self._last_commit_stats = kwargs["extra"]["commit_stats"] + self.last_commit_stats = kwargs["extra"]["commit_stats"] super().info(msg) spanner_client = spanner.Client() @@ -999,7 +999,7 @@ def insert_singers(transaction): print("{} record(s) inserted.".format(row_ct)) database.run_in_transaction(insert_singers) - commit_stats = database.logger._last_commit_stats + commit_stats = database.logger.last_commit_stats print( "{} mutation(s) in transaction.".format( commit_stats.mutation_count