From c56da25e91e42faf02840e7a4a48051ea20a78d2 Mon Sep 17 00:00:00 2001 From: Alex Amies Date: Tue, 17 Sep 2019 11:33:45 -0700 Subject: [PATCH] Initial commit --- CONTRIBUTING.md | 28 ++++ LICENSE | 169 +++++++++++++++++++++++ README.md | 236 ++++++++++++++++++++++++++++++++ applog/applog.go | 102 ++++++++++++++ go.mod | 11 ++ go.sum | 134 ++++++++++++++++++ query/query.go | 156 +++++++++++++++++++++ setup.env | 15 ++ spannerlab.go | 229 +++++++++++++++++++++++++++++++ testdata/testdata.go | 105 ++++++++++++++ update/update.go | 319 +++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 1504 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 applog/applog.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 query/query.go create mode 100644 setup.env create mode 100644 spannerlab.go create mode 100644 testdata/testdata.go create mode 100644 update/update.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ebbb59e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..624f02a --- /dev/null +++ b/LICENSE @@ -0,0 +1,169 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + 1. Definitions. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + END OF TERMS AND CONDITIONS + APPENDIX: How to apply the Apache License to your work. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + Copyright [yyyy] [name of copyright owner] + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5617eb0 --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# Spanner Latency Troubleshooting +This application combines a variety of read and write workloads on Spanner to +simulate the problems that can be encountered in a real-world application at +scale and then demonstrates how to debug them. Running the application for a +few hundred thousand iterations in 'simulation' mode will be sufficient to +surface latency problems. The causes for the latency problems include large +payloads, complex transactions, and queries with full table scans. +Instrumentation for metrics and trace collection using OpenCensus and export to +Stackdriver. The application randomly executes a read and write transaction +using one of a variety of query and transaction strategies on each iteration. + +After running the application for a period, view the data collected +in the Stackdriver Monitoring and Stackdriver Trace user interfaces to check +the latency of the requests to find the reasons for the differences +in performance. + +The example application assumes that you are familiar with Go programming, +Google Cloud Platform, [Spanner](https://cloud.google.com/spanner/) basics, +[OpenCensus](https://opencensus.io/), and +[Stackdriver](https://cloud.google.com/stackdriver/). + +## Prerequisites +The steps described here can be run on a Linux or Mac OS command line or the +GCP Cloud Shell. + +- Select or create a GCP project. Go to the + [Project Selector page](https://console.cloud.google.com/projectselector2/home/dashboard) +- Make sure that billing is enabled for your Google Cloud Platform project. + [Learn how to enable billing](https://cloud.google.com/billing/docs/how-to/modify-project). +- Download and install the + [Google Cloud SDK](https://cloud.google.com/sdk/docs/). + +### Project Setup +In the Cloud Shell, clone the GitHub project + +```shell +git clone https://github.com/GoogleCloudPlatform/opencensus-spanner-demo.git +cd opencensus-spanner-demo +``` + +Edit the variables in setup.env and import them into your development +environment: + +```shell +source ./setup.env +``` + +Enable the Stackdriver and Spanner APIs: + +```shell +gcloud services enable stackdriver.googleapis.com \ + cloudtrace.googleapis.com \ + spanner.googleapis.com \ + logging.googleapis.com \ + compute.googleapis.com +``` + +### Setup Spanner +Create a Spanner instance + +```shell +gcloud spanner instances create $SPANNER_INSTANCE \ + --config=regional-us-central1 \ + --description="Test Instance" \ + --nodes=1 +``` + +Create a database + +```shell +gcloud spanner databases create $DATABASE --instance=$SPANNER_INSTANCE +``` + +Create some tables for the test application with the same schema as +[Getting started with Cloud Spanner in Go](https://cloud.google.com/spanner/docs/getting-started/go/). +Following the +[Data Manipulation Language syntax](https://cloud.google.com/spanner/docs/dml-syntax), +in the Cloud Console, navigate to the +[Spanner database](https://console.cloud.google.com/spanner/instances/test-instance/databases/test/createtable). +Check Edit as text, enter the following text into the text area + +```sql +CREATE TABLE Singers ( + SingerId INT64 NOT NULL, + FirstName STRING(1024), + LastName STRING(1024), + BirthDate DATE, + LastUpdated TIMESTAMP, +) PRIMARY KEY(SingerId); +``` + +and click the Create button to create the table Singers. + +Click on the Create index link and check Edit as text. Enter the following text +into the text area. + +```sql +CREATE INDEX SingersByLastName ON Singers(LastName) +``` + +and click the Create button to add an index for last name. + +Go back to Database details and click the Create table link. Check Edit as text +and enter the following text into the text area. + +```sql +CREATE TABLE Albums ( + SingerId INT64 NOT NULL, + AlbumId INT64 NOT NULL, + AlbumTitle STRING(MAX), + MarketingBudget INT64, +) PRIMARY KEY(SingerId, AlbumId), + INTERLEAVE IN PARENT Singers ON DELETE CASCADE; +``` + +Click Create to create the table Albums. + +### Setup an GCE Instance +Back in the Cloud Shell, create a GCE instance to run the test application from + +```shell +gcloud compute instances create $CLIENT_INSTANCE \ + --zone=$ZONE \ + --scopes=https://www.googleapis.com/auth/cloud-platform \ + --boot-disk-size=200GB +``` + +Grant the GCE instance service account the predefined role +[roles/spanner.databaseUser](https://cloud.google.com/spanner/docs/iam#roles) +following these steps. First, find the name of the service +account associated with the instance: + +```shell +gcloud compute instances describe $CLIENT_INSTANCE \ + --zone=$ZONE \ + --format="value(serviceAccounts.email)" +``` + +Make a note of the service account ID to grant role roles/spanner.databaseUser +to the instance service account + +```shell +SA_ACCOUNT=[service account id from command above] +gcloud projects add-iam-policy-binding $GOOGLE_CLOUD_PROJECT \ + --member serviceAccount:$SA_ACCOUNT \ + --role roles/spanner.databaseUser +``` + +SSH to the instance + +```shell +gcloud compute ssh --zone $ZONE $CLIENT_INSTANCE +``` + +Install git + +```shell +sudo apt-get update +sudo apt-get install -y git +``` + +Install [Go](https://golang.org/doc/install) and get the dependent libraries, as +above. + +## Run the test app +Clone the code from the git repo. + +```shell +git clone https://github.com/GoogleCloudPlatform/opencensus-spanner-demo +cd opencensus-spanner-demo +``` + +Edit the file setup.env and initialize the environment + +```shell +source ./setup.env +``` + +Build the code + +```shell +go build +``` + +If you have trouble building the application make sure that you have Go modules +enabled by setting the GO111MODULE environment variable: + +```shell +export GO111MODULE=on +``` + +Set the project + +```shell +export GOOGLE_CLOUD_PROJECT=[your project] +``` + +Run the test application: + +```shell +nohup ./oc-spannerlab --project=$GOOGLE_CLOUD_PROJECT \ + --instance=$SPANNER_INSTANCE \ + --database=$DATABASE \ + --command=simulation \ + --iterations=100000 & +``` +This runs 100,000 iterations of the test application in simulation mode, which +will execute a random combination of queries and updates. It will take several +minutes to run. Check that there are no errors in the command output: + +```shell +tail -f nohup.log +``` + +## View the data +You can view these in the Google Cloud Logging +[Log Viewer](https://console.cloud.google.com/logs/viewer?expandAll=false&resource=gce_instance) +under GCE VM instances. + +Go to the [trace list](https://console.cloud.google.com/traces/traces) to see +the trace data. Notice the payload size in the Trace timeline and how higher +latency tends to be correlated with larger payload size. To view the payload +size, click on a trace in the Trace list and in the Trace timeline click +Show events. Notice the bytes received in the timeline. + +To view log-trace correlation, click on a trace in the Trace list and in the +Trace timeline click on Show logs. Notice the log entry in the trace timeline +and in the trace detail. + +Also, check the aggregate metrics in +[Stackdriver Monitoring](https://console.cloud.google.com/monitoring). +In the Resource menu click Metrics Explorer. In the Metric textfield type in +the prefix 'spanner-oc-test' and select from the metrics displayed. The +metric 'completed_rpcs' is a good metric to view the overall status of the +test. From the Metrics Explorer click Save chart to save the chart into a +new dashboard. diff --git a/applog/applog.go b/applog/applog.go new file mode 100644 index 0000000..f272a5d --- /dev/null +++ b/applog/applog.go @@ -0,0 +1,102 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package applog + +/** + Implment trace-log correlation with Stackdriver Logging and OpenCensus. + **/ + +import ( + "context" + "fmt" + "log" + + "cloud.google.com/go/logging" + "go.opencensus.io/trace" +) + +const LOGNAME string = "oc-spannerlab" + +var ( + client *logging.Client + projectId string +) + +// Close and flush the logging client +func Close() { + err := client.Close() + if err != nil { + fmt.Printf("Failed to close logging client: %v", err) + } +} + +// Log an error with the given context, may include trace and span +func Errorf(ctx context.Context, format string, v ...interface{}) { + printf(ctx, logging.Error, format, v...) +} + +// Log a fatal error with the given context, may include trace and span +func Fatalf(ctx context.Context, format string, v ...interface{}) { + printf(ctx, logging.Critical, format, v...) + log.Fatalf(format, v...) +} + +// Initialize the Cloud Logging client +func Initialize(projId string) { + projectId = projId + ctx := context.Background() + var err error + client, err = logging.NewClient(ctx, projId) + if err != nil { + fmt.Printf("Failed to create logging client: %v", err) + return + } + fmt.Printf("Stackdriver Logging initialized with project id %s, see Cloud " + + " Console under GCE VM instance > all instance_id\n", projectId) +} + +// Send to Cloud Logging service including reference to current span +func Printf(ctx context.Context, format string, v ...interface{}) { + printf(ctx, logging.Info, format, v...) +} + +// Send to Cloud Logging service including reference to current span +// [START spannerlab_trace_correlation] +func printf(ctx context.Context, severity logging.Severity, format string, + v ...interface{}) { + span := trace.FromContext(ctx) + if client == nil { + log.Printf(format, v...) + } else if span == nil { + lg := client.Logger(LOGNAME) + lg.Log(logging.Entry{ + Severity: severity, + Payload: fmt.Sprintf(format, v...), + }) + } else { + sCtx := span.SpanContext() + tr := sCtx.TraceID.String() + lg := client.Logger(LOGNAME) + trace := fmt.Sprintf("projects/%s/traces/%s", projectId, tr) + lg.Log(logging.Entry{ + Severity: severity, + Payload: fmt.Sprintf(format, v...), + Trace: trace, + SpanID: sCtx.SpanID.String(), + }) + } +} + +// [END spannerlab_trace_correlation] diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e2c9364 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module github.com/GoogleCloudPlatform/oc-spannerlab + +go 1.11 + +require ( + cloud.google.com/go v0.43.0 + cloud.google.com/go/logging v1.0.0 + contrib.go.opencensus.io/exporter/stackdriver v0.12.4 + go.opencensus.io v0.22.0 + google.golang.org/api v0.7.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0180b3e --- /dev/null +++ b/go.sum @@ -0,0 +1,134 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0 h1:banaiRPAM8kUVYneOSkhgcDsLzEvL25FinuiSZaH/2w= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= +cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8= +cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= +contrib.go.opencensus.io/exporter/stackdriver v0.12.4 h1:e1itpYdd++w6+DPhvyKqT7uazcc4NwyToI8UJ0tMGCs= +contrib.go.opencensus.io/exporter/stackdriver v0.12.4/go.mod h1:fmn/xkyUfUhd1iD7Ic+HSN8y11KhSK5oe8CWfSjKa7M= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-sdk-go v1.21.0 h1:dRGzi4XZe5GFSJssHkRNUf/hRD/HL8bdqTYa9hpAO8c= +github.com/aws/aws-sdk-go v1.21.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0WCIICM8RkWvDw03lvB77hrhJnpxfU= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/query/query.go b/query/query.go new file mode 100644 index 0000000..a2eadf6 --- /dev/null +++ b/query/query.go @@ -0,0 +1,156 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Test application to demonstrate identification of latency problems. +package query + +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/spanner" + "go.opencensus.io/trace" + "google.golang.org/api/iterator" + + log "github.com/GoogleCloudPlatform/oc-spannerlab/applog" +) + +// Queries albums and singers with a join +func JoinSingerAlbum(ctx context.Context, client *spanner.Client, + w io.Writer) { + ctx, span := trace.StartSpan(ctx, "join-singer-album") + defer span.End() + q := `SELECT s.SingerId, s.FirstName, a.AlbumTitle + FROM Singers AS s + JOIN Albums AS a ON s.SingerId = a.SingerId;` + err := querySingers(span, ctx, client, w, q) + if err != nil { + log.Errorf(ctx, "JoinSingerAlbum Error %v", err) + } +} + +// Queries albums in the Spanner database +func QueryAlbums(ctx context.Context, client *spanner.Client, w io.Writer) { + // [START spannerlab_query_albums_span] + ctx, span := trace.StartSpan(ctx, "query-albums") + defer span.End() + // [END spannerlab_query_albums_span] + q := `SELECT SingerId, AlbumId, AlbumTitle FROM Albums` + err := queryAlbums(ctx, client, w, q) + if err != nil { + log.Errorf(ctx, "Error querying albums %v for query %s", err, q) + } +} + +// Queries albums in the Spanner database with a limit +func QueryAlbumsLimit(ctx context.Context, client *spanner.Client, + w io.Writer) { + ctx, span := trace.StartSpan(ctx, "query-limit") + defer span.End() + q := `SELECT SingerId, AlbumId, AlbumTitle FROM Albums LIMIT 10` + err := queryAlbums(ctx, client, w, q) + if err != nil { + log.Printf(ctx, "QueryLimit Error %v", err) + } +} + +// Execute a query with no parameters +func queryAlbums(ctx context.Context, client *spanner.Client, w io.Writer, + q string) error { + // [START querylbums_ReadOnlyTransaction] + ro := client.ReadOnlyTransaction() + defer ro.Close() + // [END querylbums_ReadOnlyTransaction] + stmt := spanner.Statement{SQL: q} + iter := ro.Query(ctx, stmt) + defer iter.Stop() + counter := 0 + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + var singerID int64 + var albumID int64 + var albumTitle string + if err := row.Columns(&singerID, &albumID, &albumTitle); err != nil { + return err + } + counter++ + fmt.Fprintf(w, "%d %d %s", singerID, albumID, albumTitle) + } + log.Printf(ctx, "queryAlbums: %d results for query: %s", counter, q) + return nil +} + +// Queries singers by first name (has an index) +func QuerySingersFirstName(ctx context.Context, client *spanner.Client, + w io.Writer) { + ctx, span := trace.StartSpan(ctx, "query-singers-first") + defer span.End() + q := `SELECT SingerId, FirstName, LastName FROM Singers + WHERE FirstName = 'Captain'` + err := querySingers(span, ctx, client, w, q) + if err != nil { + log.Printf(ctx, "QuerySingersFirstName Error %v", err) + } +} + +// Queries singers by last name (has an index) +func QuerySingersLastName(ctx context.Context, client *spanner.Client, + w io.Writer) { + ctx, span := trace.StartSpan(ctx, "query-singers-last") + defer span.End() + q := `SELECT SingerId, FirstName, LastName + FROM Singers@{FORCE_INDEX=SingersByLastName} + WHERE LastName = 'Zero'` + err := querySingers(span, ctx, client, w, q) + if err != nil { + log.Printf(ctx, "QuerySingersLastName Error %v", err) + } +} + +// Execute a query with no parameters +func querySingers(span *trace.Span, ctx context.Context, + client *spanner.Client, w io.Writer, q string) error { + ro := client.ReadOnlyTransaction() + defer ro.Close() + stmt := spanner.Statement{SQL: q} + iter := ro.Query(ctx, stmt) + defer iter.Stop() + counter := 0 + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + var singerID int64 + var firstName string + var lastName string + if err := row.Columns(&singerID, &firstName, &lastName); err != nil { + return err + } + counter++ + fmt.Fprintf(w, "%d %s %s", singerID, firstName, lastName) + } + log.Printf(ctx, "querySingers # results: %d for query: %s", counter, q) + return nil +} diff --git a/setup.env b/setup.env new file mode 100644 index 0000000..e1c5d68 --- /dev/null +++ b/setup.env @@ -0,0 +1,15 @@ +#Environment variables for running the solution +# Optionally change these if you want to run in a different zone or with a +# different name of virtual machine, Spanner instance, or database. + +# GPC Zone +ZONE=us-central1-c + +# GCE client instance name +export CLIENT_INSTANCE=spanner-client + +# Spanner instance name +export SPANNER_INSTANCE=test-instance + +# Spanner database +export DATABASE=test diff --git a/spannerlab.go b/spannerlab.go new file mode 100644 index 0000000..718ca9a --- /dev/null +++ b/spannerlab.go @@ -0,0 +1,229 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Test application to demonstrate identification of latency problems. +package main + +// [START spannerlab_imports] +import ( + "bytes" + "context" + "flag" + "fmt" + "os" + + "cloud.google.com/go/spanner" + + "contrib.go.opencensus.io/exporter/stackdriver" + "go.opencensus.io/plugin/ocgrpc" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + + log "github.com/GoogleCloudPlatform/oc-spannerlab/applog" + "github.com/GoogleCloudPlatform/oc-spannerlab/query" + "github.com/GoogleCloudPlatform/oc-spannerlab/testdata" + "github.com/GoogleCloudPlatform/oc-spannerlab/update" +) + +// [END spannerlab_imports] + +// Initialize OpenCensus +// [START spannerlab_initoc] +func initOC(project string) *stackdriver.Exporter { + se, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: project, + MetricPrefix: "spanner-oc-test", + }) + if err != nil { + ctx := context.Background() + log.Fatalf(ctx, "Failed to create exporter: %v", err) + } + trace.RegisterExporter(se) + view.RegisterExporter(se) + if err := view.Register(ocgrpc.DefaultClientViews...); err != nil { + ctx := context.Background() + log.Fatalf(ctx, "Failed to register gRPC default client views: %v", err) + } + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + return se +} + +// [END spannerlab_initoc] + +// Run the query tests +func runQueryTest(client *spanner.Client) { + ctx := context.Background() + buf := bytes.NewBufferString("") + query.QueryAlbums(ctx, client, buf) +} + +// Run a simulation with a mix of queries and adds +func runSimulation(client *spanner.Client, iterations int) { + fmt.Printf("Running simulation with %d iterations\n", iterations) + ctx := context.Background() + for i := 0; i < iterations; i++ { + if i%10 == 0 { + fmt.Printf("Iteration %d\n", i) + } + action := testdata.NextUserAction() + log.Printf(ctx, "Next user action is %d.\n", action) + buf := bytes.NewBufferString("") + switch action { + case testdata.ACTION_QUERY_ALBUMS: + query.QueryAlbums(ctx, client, buf) + case testdata.ACTION_QUERY_LIMIT: + query.QueryAlbumsLimit(ctx, client, buf) + case testdata.ACTION_QUERY_SINGERS_FIRST: + query.QuerySingersFirstName(ctx, client, buf) + case testdata.ACTION_QUERY_SINGERS_LAST: + query.QuerySingersLastName(ctx, client, buf) + case testdata.ACTION_JOIN_SINGER_ALBUM: + query.JoinSingerAlbum(ctx, client, buf) + case testdata.ACTION_ADD_ALL_TXN: + data := testdata.RandomData() + ctx, span := trace.StartSpan(ctx, "add-album-single-txns") + _, err := update.AddAllNoTxn(ctx, client, data.FirstName, data.LastName, + data.AlbumTitle) + if err != nil { + log.Printf(ctx, "Error adding singer %v", err) + } + span.End() + case testdata.ACTION_ADD_SINGLE_TXNS: + data := testdata.RandomData() + ctx, span := trace.StartSpan(ctx, "add-album-all-one-txn") + _, err := update.AddAllTxn(ctx, client, data.FirstName, + data.LastName, data.AlbumTitle) + if err != nil { + log.Printf(ctx, "Error adding singer in transaction %v", err) + } + span.End() + } + } +} + +// Run the update tests +func runUpdateSmallTxns(client *spanner.Client) { + ctx := context.Background() + data := testdata.RandomData() + ctx, span := trace.StartSpan(ctx, "add-album-single-txns") + albumId, err := update.AddAllNoTxn(ctx, client, data.FirstName, + data.LastName, data.AlbumTitle) + if err != nil { + log.Errorf(ctx, "Error adding singer %v", err) + } else { + log.Printf(ctx, "runTest not in transaction %d", albumId) + } + sNum, err := update.CountRows(client, "SingerId", "Singers") + if err != nil { + log.Errorf(ctx, "Error querying singers %v", err) + } else { + log.Printf(ctx, "Total number of singers %d", sNum) + } + aNum, err := update.CountRows(client, "AlbumId", "Albums") + if err != nil { + log.Errorf(ctx, "Error querying singers %v", err) + } else { + log.Printf(ctx, "Total number of albums %d", aNum) + } + span.End() +} + +// Run the update tests +func runUpdateBigTxn(client *spanner.Client) { + ctx := context.Background() + data := testdata.RandomData() + ctx, span := trace.StartSpan(ctx, "add-album-all-one-txn") + albumId, err := update.AddAllTxn(ctx, client, data.FirstName, + data.LastName, data.AlbumTitle) + if err != nil { + log.Printf(ctx, "Error adding singer in transaction %v", err) + } else { + log.Printf(ctx, "runTest in transaction %d", albumId) + } + sNum, err := update.CountRows(client, "SingerId", "Singers") + if err != nil { + log.Printf(ctx, "Error querying singers %v", err) + } else { + log.Printf(ctx, "Total number of singers %d", sNum) + } + aNum, err := update.CountRows(client, "AlbumId", "Albums") + if err != nil { + log.Printf(ctx, "Error querying singers %v", err) + } else { + log.Printf(ctx, "Total number of albums %d", aNum) + } + span.End() +} + +// Entry point for the application +func main() { + project := os.Getenv("GOOGLE_CLOUD_PROJECT") + var projPtr = flag.String("project", project, "The project id") + var instance = flag.String("instance", "test-instance", + "The Spanner instance") + var db = flag.String("database", "test", "The Spanner database name") + var command = flag.String("command", "simulation", + "One of [update_big_txn | update_small_txns | query_test | simulation]") + var iterations = flag.Int("iterations", 100, + "Number of iterations to run for the 'simulation' command") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, + `Usage: oc-spannerlab --project=$GOOGLE_CLOUD_PROJECT \ + --instance=$SPANNER_INSTANCE \ + --database=$DATABASE \ + --command=COMMAND \ + [--iterations=iterations] +`) + } + flag.Parse() + if *projPtr == "" { + fmt.Println("project flag must have a value") + flag.Usage() + os.Exit(2) + } + + log.Initialize(project) + defer log.Close() + + databaseName := fmt.Sprintf("projects/%s/instances/%s/databases/%s", *projPtr, + *instance, *db) + + // Initialize OpenCensus + se := initOC(project) + defer se.Flush() + + // Initialize Spanner client + ctx := context.Background() + client, err := spanner.NewClient(ctx, databaseName) + if err != nil { + fmt.Printf("Failed to create Spanner client %v", err) + os.Exit(1) + } + defer client.Close() + + if *command == "update_big_txn" { + runUpdateBigTxn(client) + } else if *command == "update_small_txns" { + runUpdateSmallTxns(client) + } else if *command == "query_test" { + runQueryTest(client) + } else if *command == "simulation" { + runSimulation(client, *iterations) + } else { + fmt.Printf("Command %s not understood", command) + flag.Usage() + os.Exit(2) + } +} diff --git a/testdata/testdata.go b/testdata/testdata.go new file mode 100644 index 0000000..8729210 --- /dev/null +++ b/testdata/testdata.go @@ -0,0 +1,105 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package to generate test data + +package testdata + +import ( + "fmt" + "math/rand" + "time" +) + +const ( + ACTION_QUERY_ALBUMS Action = iota + 1 + ACTION_QUERY_LIMIT + ACTION_QUERY_SINGERS_FIRST + ACTION_QUERY_SINGERS_LAST + ACTION_JOIN_SINGER_ALBUM + ACTION_ADD_ALL_TXN + ACTION_ADD_SINGLE_TXNS +) + +var ACTIONS = [...]Action{ + ACTION_QUERY_ALBUMS, + ACTION_QUERY_LIMIT, + ACTION_QUERY_SINGERS_FIRST, + ACTION_QUERY_SINGERS_LAST, + ACTION_JOIN_SINGER_ALBUM, + ACTION_ADD_ALL_TXN, + ACTION_ADD_SINGLE_TXNS} + +type Action int + +type SingerAlbum struct { + FirstName, LastName, AlbumTitle string +} + +func init() { + rand.Seed(int64(time.Now().UnixNano())) +} + +func (a Action) String() string { + names := map[Action]string{ + ACTION_QUERY_ALBUMS: "QueryAlbums", + ACTION_QUERY_LIMIT: "QueryLimit", + ACTION_QUERY_SINGERS_FIRST: "QuerySingersFirstName", + ACTION_QUERY_SINGERS_LAST: "QuerySingersLastName", + ACTION_JOIN_SINGER_ALBUM: "JoinSingerAlbum", + ACTION_ADD_ALL_TXN: "AddAllInBigTransaction", + ACTION_ADD_SINGLE_TXNS: "AddEachInSingleTransactions", + } + if name, ok := names[a]; ok { + return name + } else { + return "unknown" + } +} + +func NextUserAction() Action { + r := rand.Intn(len(ACTIONS)) + return ACTIONS[r] +} + +// Generate a random name for a singer +func RandomData() SingerAlbum { + rank := []string{"Private", "Brigadier", "Sergeant", "Captain", "Commander", + "Chief", "Lieutenant", "Officer", "First Officer", "Major", + "General", "Five Star General", "Admiral", "Rear Admiral", + "Vice General"} + initial := []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", + "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", + "Y", "Z"} + surname := []string{"Ryan", "General", "Major", "Zero", "Supreme", + "Petty officer", "Governor", "In Charge", "Blunder", "Chaos"} + generation := []string{"Junior", "II", "III", "IV", "V", "VI", "VII", + "VIII", "IX", "X", "XI", "XII", "XIII", "XIV", "XV", "XVI", + "XVII", "XVIII", "XIX", "XX"} + part1 := []string{"Smoke", "Mist", "Rain", "Fog", "Thunder", "Lightening", + "Frost", "Dew", "Snow", "Shadows", "Water", "Grass", "Trees", + "Dust", "Wind", "Breeze", "Trash", "Graffiti", "Writing"} + part2 := []string{"Water", "River", "Plains", "Road", "Mountain", "Hills", + "Sea", "Bay", "Forest", "Highway", "Wall", "Blackboard"} + m := rand.Int63n(int64(len(rank))) + n := rand.Int63n(int64(len(initial))) + p := rand.Int63n(int64(len(surname))) + q := rand.Int63n(int64(len(generation))) + r := rand.Int63n(int64(len(part1))) + s := rand.Int63n(int64(len(part2))) + firstName := fmt.Sprintf("%s %s", rank[m], initial[n]) + lastName := fmt.Sprintf("%s %s", surname[p], generation[q]) + albumTitle := fmt.Sprintf("%s on the %s", part1[r], part2[s]) + return SingerAlbum{firstName, lastName, albumTitle} +} diff --git a/update/update.go b/update/update.go new file mode 100644 index 0000000..115da3f --- /dev/null +++ b/update/update.go @@ -0,0 +1,319 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Simulate updates by adding test data to a Spanner database +package update + +import ( + "context" + "errors" + "fmt" + "math/rand" + + "cloud.google.com/go/spanner" + "google.golang.org/api/iterator" + + log "github.com/GoogleCloudPlatform/oc-spannerlab/applog" +) + +const ( + NOT_FOUND = -1 + SPANNER_ERROR = -2 +) + +type AppError struct { + Message string + Code int +} + +func (e AppError) Error() string { + return e.Message +} + +func albumNotFound(albumId int64, albumTitle string) *AppError { + msg := fmt.Sprintf("Singer-Album %d %s not found", albumId, albumTitle) + return &AppError{msg, NOT_FOUND} +} + +// Adds a singer-album with a random album id, not checking for existence +// Returns: The id of the newly created album +func addAlbum(ctx context.Context, client *spanner.Client, singerId int64, + albumTitle string) (*int64, error) { + albumId := rand.Int63() + _, err := client.ReadWriteTransaction(ctx, func(ctx context.Context, + txn *spanner.ReadWriteTransaction) error { + stmt := spanner.Statement{ + SQL: `INSERT Albums (SingerId, AlbumId, AlbumTitle) VALUES + (@SingerId, @AlbumId, @AlbumTitle)`, + Params: map[string]interface{}{ + "SingerId": singerId, + "AlbumId": albumId, + "AlbumTitle": albumTitle, + }, + } + rowCount, err := txn.Update(ctx, stmt) + if err != nil { + return err + } + log.Printf(ctx, "%d record(s) inserted.\n", rowCount) + return nil + }) + return &albumId, err +} + +// Adds a singer and album first checking for the existence of the singer but +// not in the same transaction. +// Returns: The id of the singer, either existing or newly created +func AddAllNoTxn(ctx context.Context, client *spanner.Client, + firstName, lastName, albumTitle string) (*int64, error) { + singerId, e := getSingerId(ctx, client, nil, firstName, lastName) + if e != nil && e.Code != NOT_FOUND { + log.Printf(ctx, "Error looking up singer") + return nil, errors.New(e.Message) + } + if e != nil && e.Code == NOT_FOUND { + var err error + singerId, err = addSinger(ctx, client, firstName, lastName) + if err != nil { + log.Printf(ctx, "Could not add singer") + return nil, err + } + } + var albumId *int64 + albumId, e = getAlbumId(ctx, client, nil, singerId, albumTitle) + if e != nil && e.Code != NOT_FOUND { + log.Printf(ctx, "Error looking up album") + return nil, errors.New(e.Message) + } + if e != nil && e.Code == NOT_FOUND { + var err error + albumId, err = addAlbum(ctx, client, singerId, albumTitle) + if err != nil { + log.Printf(ctx, "Could not add album") + return nil, err + } + } + return albumId, nil +} + +// Adds a singer and album first checking for existence within the transaction +// Return the album id created or that already existed +func AddAllTxn(ctx context.Context, client *spanner.Client, + firstName, lastName, albumTitle string) (*int64, error) { + var albumId *int64 + _, err := client.ReadWriteTransaction(ctx, func(ctx context.Context, + txn *spanner.ReadWriteTransaction) error { + // adds the singer with given singerId and name + addAlbum := func(singerId int64, albumTitle string) (*int64, error) { + albumId := rand.Int63() + stmt := spanner.Statement{ + SQL: `INSERT Albums (SingerId, AlbumId, AlbumTitle) VALUES + (@SingerId, @AlbumId, @AlbumTitle)`, + Params: map[string]interface{}{ + "SingerId": singerId, + "AlbumId": albumId, + "AlbumTitle": albumTitle, + }, + } + _, err := txn.Update(ctx, stmt) + return &albumId, err + } + + addSinger := func(firstName, lastName string) (int64, error) { + singerId := rand.Int63() + stmt := spanner.Statement{ + SQL: `INSERT Singers (SingerId, FirstName, LastName) VALUES + (@SingerId, @FirstName, @LastName)`, + Params: map[string]interface{}{ + "SingerId": singerId, + "FirstName": firstName, + "LastName": lastName, + }, + } + _, err := txn.Update(ctx, stmt) + return singerId, err + } + + singerId, e := getSingerId(ctx, client, txn, firstName, lastName) + if e != nil && e.Code != NOT_FOUND { + return errors.New(e.Message) + } + if e == nil { + return nil + } + // The singer will be added only if they do not exist already + var err error + singerId, err = addSinger(firstName, lastName) + if err != nil { + return err + } + log.Printf(ctx, "Added singer %s %s in transaction", firstName, lastName) + + // Add album + albumId, e = getAlbumId(ctx, client, txn, singerId, albumTitle) + if e != nil && e.Code != NOT_FOUND { + log.Printf(ctx, "Error looking up album") + return errors.New(e.Message) + } + if e != nil && e.Code == NOT_FOUND { + var err error + albumId, err = addAlbum(singerId, albumTitle) + if err != nil { + log.Printf(ctx, "Could not add album") + return err + } + } + return nil + }) + return albumId, err +} + +// Adds a singer with a random id, not checking for the existence of the singer. +// Returns: The id of the newly created singer +func addSinger(ctx context.Context, client *spanner.Client, + firstName, lastName string) (int64, error) { + singerId := rand.Int63() + _, err := client.ReadWriteTransaction(ctx, func(ctx context.Context, + txn *spanner.ReadWriteTransaction) error { + stmt := spanner.Statement{ + SQL: `INSERT Singers (SingerId, FirstName, LastName) VALUES + (@SingerId, @FirstName, @LastName)`, + Params: map[string]interface{}{ + "SingerId": singerId, + "FirstName": firstName, + "LastName": lastName, + }, + } + rowCount, err := txn.Update(ctx, stmt) + if err != nil { + return err + } + log.Printf(ctx, "%d record(s) inserted.\n", rowCount) + return nil + }) + return singerId, err +} + +// Count the singers with a select query +func CountRows(client *spanner.Client, + fieldName, tableName string) (int64, error) { + ctx := context.Background() + selectCount := fmt.Sprintf("SELECT COUNT(%s) FROM %s", fieldName, tableName) + stmt := spanner.Statement{ + SQL: selectCount, + } + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() + for { + row, err := iter.Next() + if err == iterator.Done { + return -1, errors.New("No results") + } + if err != nil { + return -1, err + } + var count int64 + err = row.Columns(&count) + if err != nil { + return -1, err + } + return count, nil + } + return -1, errors.New("No results") +} + +// Return the id, if the album with given singer and title is in the database +func getAlbumId(ctx context.Context, client *spanner.Client, + txn *spanner.ReadWriteTransaction, singerId int64, + albumTitle string) (*int64, *AppError) { + stmt := spanner.Statement{ + SQL: `SELECT + SingerId, AlbumId + FROM Albums + WHERE + SingerId = @SingerId AND AlbumTitle = @AlbumTitle`, + Params: map[string]interface{}{ + "SingerId": singerId, + "AlbumTitle": albumTitle, + }, + } + // Reuse transaction if not nil + var iter *spanner.RowIterator + if txn != nil { + iter = txn.Query(ctx, stmt) + } else { + iter = client.Single().Query(ctx, stmt) + } + defer iter.Stop() + for { + row, err := iter.Next() + if err == iterator.Done { + return nil, albumNotFound(singerId, albumTitle) + } + if err != nil { + return nil, &AppError{err.Error(), SPANNER_ERROR} + } + var singerID int64 + if row.Columns(&singerID) != nil { + log.Printf(ctx, "Failed to parse row") + } + var albumId int64 + if row.Columns(&albumId) != nil { + log.Printf(ctx, "Failed to parse row") + } + return &albumId, nil + } + return nil, albumNotFound(singerId, albumTitle) +} + +// If the singer is in the database then return the id. +// If a transaction is supplied then use it. Otherwise, create a new, single +// query transaction. +func getSingerId(ctx context.Context, client *spanner.Client, + txn *spanner.ReadWriteTransaction, + firstName, lastName string) (int64, *AppError) { + stmt := spanner.Statement{ + SQL: `SELECT SingerId FROM Singers + WHERE FirstName = @FirstName AND LastName = @LastName`, + Params: map[string]interface{}{ + "FirstName": firstName, + "LastName": lastName, + }, + } + // Reuse transaction if not nil + var iter *spanner.RowIterator + if txn != nil { + iter = txn.Query(ctx, stmt) + } else { + iter = client.Single().Query(ctx, stmt) + } + defer iter.Stop() + for { + row, err := iter.Next() + if err == iterator.Done { + msg := fmt.Sprintf("Singer not found", firstName, lastName) + return -1, &AppError{msg, NOT_FOUND} + } + if err != nil { + return -1, &AppError{err.Error(), SPANNER_ERROR} + } + var singerID int64 + if row.Columns(&singerID) != nil { + log.Printf(ctx, "Failed to parse row") + } + return singerID, nil + } + msg := fmt.Sprintf("Singer %s %s not found", firstName, lastName) + return -1, &AppError{msg, NOT_FOUND} +}