Skip to content
Permalink
Browse files

Rename 'mako_examples' to 'examples'

PiperOrigin-RevId: 264206172
Change-Id: I1c67c3d6395db38684cf79044490b93eb9d575f5
  • Loading branch information...
timford authored and andrewstewart committed Aug 19, 2019
1 parent ceed288 commit 8028f963e8160b40dd243b059d74d987522accb4
@@ -42,7 +42,7 @@ If you’re writing a new performance test that will use a Mako client to store
results in https://mako.dev and to guard against performance regressions, please
read [GUIDE.md](docs/GUIDE.md).

There are example performance tests in the [`mako_examples/`](./mako_examples)
There are example performance tests in the [`examples/`](./examples)
folder.

## Accessing Mako data with the storage libaries
@@ -339,7 +339,7 @@ The typical structure of a Quickstore run is:
aggregate data.
7. Call the `Store` method to process the data and upload it to https://mako.dev.

The examples in [`mako_examples/`](../mako_examples/) illustrate these steps.
The examples in [`examples/`](../examples/) illustrate these steps.

#### Add Regression Detection

@@ -350,13 +350,13 @@ your system’s performance, then you should strongly consider integrating
analyzers.

In
[`mako_examples/go_quickstore/example_test.go`](../mako_examples/go_quickstore/example_test.go)
[`examples/go_quickstore/example_test.go`](../examples/go_quickstore/example_test.go)
we configure a threshold analyzer. This is the simplest analyzer conceptually
and the easiest to configure. Most tests should start with a threshold analyzer
and expand from there.

Note that in
[`mako_examples/go_quickstore/example_test.go`](../mako_examples/go_quickstore/example_test.go)
[`examples/go_quickstore/example_test.go`](../examples/go_quickstore/example_test.go)
we fail the test when Quickstore reports an analyzer failure. This allows us to
treat the performance test like a correctness/functional test regarding how
failures are reported.
@@ -40,5 +40,5 @@ proto message allows you to specify tags to filter on when graphing the
historical context.

## Implementation/Examples
* [C++](../mako_examples/cxx_quickstore/example_test.cc)
* [Go](../mako_examples/go_quickstore/example_test.go)
* [C++](../examples/cxx_quickstore/example_test.cc)
* [Go](../examples/go_quickstore/example_test.go)
@@ -0,0 +1,35 @@
licenses(["notice"])

cc_test(
name = "example_test",
srcs = ["example_test.cc"],
tags = [
"manual",
"notap",
],
deps = [
":perf_data_cc_proto",
"//clients/cxx/analyzers:threshold_analyzer",
"//clients/proto/analyzers:threshold_analyzer_cc_proto",
"//helpers/cxx/quickstore",
"//helpers/cxx/status",
"//helpers/cxx/status:status_matchers",
"//helpers/cxx/status:statusor",
"//quickstore:quickstore_cc_proto",
"//spec/proto:mako_cc_proto",
"@com_google_absl//absl/flags:parse",
"@com_google_glog//:glog",
"@com_google_googletest//:gtest_main",
"@com_google_protobuf//:protobuf",
],
)

proto_library(
name = "perf_data_proto",
srcs = ["perf_data.proto"],
)

cc_proto_library(
name = "perf_data_cc_proto",
deps = [":perf_data_proto"],
)
@@ -0,0 +1,4 @@
# C++ Quickstore Example

This example illustrates the use of the Mako Quickstore C++ library to add
performance test results to http://mako.dev.
@@ -0,0 +1,147 @@
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// see the license for the specific language governing permissions and
// limitations under the license.
#include "glog/logging.h"
#include "src/google/protobuf/text_format.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "clients/proto/analyzers/threshold_analyzer.pb.h"
#include "absl/flags/parse.h"
#include "clients/cxx/analyzers/threshold_analyzer.h"
#include "helpers/cxx/status/canonical_errors.h"
#include "helpers/cxx/status/status_matchers.h"
#include "helpers/cxx/status/statusor.h"
#include "examples/cxx_quickstore/perf_data.pb.h"
#include "quickstore/cxx/quickstore.h"
#include "quickstore/quickstore.pb.h"
#include "spec/proto/mako.pb.h"

namespace {

constexpr char kBenchmarkKey[] = "6500986839367680";

mako::helpers::StatusOr<cxx_quickstore::PerfData> ReadData() {
const std::string perf_data_string = R"proto(
counters: {
throughput: 4312.84
branch_miss_percentage: 1.048
page_faults: 42383
}
samples: { timestamp: 1316713628128 write_latency: 258 cpu_load: 0.13 }
samples: { timestamp: 1316713628386 read_latency: 737 cpu_load: 0.19 }
samples: { timestamp: 1316713629123 write_latency: 1256 cpu_load: 0.28 }
samples: { timestamp: 1316713870379 read_latency: 455 cpu_load: 0.34 }
samples: { timestamp: 1316713870834 write_latency: 279 cpu_load: 0.38 }
samples: { timestamp: 1316713871834 read_latency: 383 cpu_load: 0.51 }
metadata: {
git_hash: "43bf39d0a2c36583aed1c346a2e5b958ab379718"
timestamp_ms: 1316713628128
run_duration_ms: 2438
tags: "environment=continuous_integration"
tags: "branch=master"
}
)proto";
cxx_quickstore::PerfData perf_data;
if (google::protobuf::TextFormat::ParseFromString(perf_data_string, &perf_data)) {
return perf_data;
}
return mako::helpers::InvalidArgumentError(
"Failed parsing PerfData proto.");
}

TEST(MakoExample, PerformanceTest) {
// STEP 1: Collect some performance data. Here we read some data from a
// serialized format.
//
// Read more about the Mako data format at
// http://github.com/google/mako/blob/master/docs/GUIDE.md#preparing-your-performance-test-data.
auto status_or_data = ReadData();
ASSERT_OK(status_or_data);
cxx_quickstore::PerfData data = std::move(status_or_data).value();

// STEP 2: Configure run metadata in QuickstoreInput.
//
// Read about the run metadata you can set in QuickstoreInput at
// http://github.com/google/mako/blob/master/docs/GUIDE.md#run-metadata.
mako::quickstore::QuickstoreInput quickstore_input;
quickstore_input.set_benchmark_key(kBenchmarkKey);
quickstore_input.set_duration_time_ms(data.metadata().run_duration_ms());
quickstore_input.set_timestamp_ms(data.metadata().timestamp_ms());
quickstore_input.set_hover_text(data.metadata().git_hash());
quickstore_input.mutable_tags()->CopyFrom(data.metadata().tags());

// STEP 3: Configure an Analyzer
//
// Read more about analyzers at
// https://github.com/google/mako/blob/master/docs/ANALYZERS.md
mako::analyzers::threshold_analyzer::ThresholdAnalyzerInput*
analyzer_input = quickstore_input.add_threshold_inputs();

// Threshold on a metric aggregate (median of WriteLatency)
mako::analyzers::threshold_analyzer::ThresholdConfig* config =
analyzer_input->add_configs();
config->set_config_name("writes_lt_900");
config->set_max(900);
mako::DataFilter* data_filter = config->mutable_data_filter();
data_filter->set_data_type(mako::DataFilter::METRIC_AGGREGATE_MEDIAN);
data_filter->set_value_key("wl");

// Threshold on a custom aggregate (Throughput).
config = analyzer_input->add_configs();
config->set_config_name("throughput_gt_400");
config->set_min(4000);
data_filter = config->mutable_data_filter();
data_filter->set_data_type(mako::DataFilter::CUSTOM_AGGREGATE);
data_filter->set_value_key("tp");

// STEP 4: Create a Quickstore instance which reports to the Mako server
//
// Read about setting up authentication at
// http://github.com/google/mako/blob/master/docs/GUIDE.md#setting-up-authentication
mako::quickstore::Quickstore quickstore(quickstore_input);

// STEP 5: Feed your sample point data to the Mako Quickstore client.
for (const auto& sample : data.samples()) {
std::map<std::string, double> metrics = {{"cpu", sample.cpu_load()}};
if (sample.has_read_latency()) {
metrics["rl"] = sample.read_latency();
}
if (sample.has_write_latency()) {
metrics["wl"] = sample.write_latency();
}
quickstore.AddSamplePoint(sample.timestamp(), metrics);
}

// STEP 6: Feed your custom aggregate data to the Mako Quickstore client.
quickstore.AddRunAggregate("tp", data.counters().throughput());
quickstore.AddRunAggregate("bm", data.counters().branch_miss_percentage());
quickstore.AddRunAggregate("pf", data.counters().page_faults());

// STEP 7: Call Store() to instruct Mako to process the data and upload it to
// http://mako.dev.
mako::quickstore::QuickstoreOutput output = quickstore.Store();
switch (output.status()) {
case mako::quickstore::QuickstoreOutput::SUCCESS:
LOG(INFO) << " Done! Run can be found at: " << output.run_chart_link();
break;
case mako::quickstore::QuickstoreOutput::ERROR:
FAIL() << "Quickstore Store error: " << output.summary_output();
break;
case mako::quickstore::QuickstoreOutput::ANALYSIS_FAIL:
FAIL() << "Quickstore Analysis Failure: " << output.summary_output()
<< "\nRun can be found at: " << output.run_chart_link();
break;
}
}

} // namespace
@@ -0,0 +1,42 @@
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// see the license for the specific language governing permissions and
// limitations under the license.
syntax = "proto2";

package cxx_quickstore;

message Counters {
optional double throughput = 1;
optional double branch_miss_percentage = 2;
optional int64 page_faults = 3;
}

message Sample {
optional int64 timestamp = 1;
optional int64 write_latency = 2;
optional int64 read_latency = 3;
optional double cpu_load = 4;
}

message Metadata {
optional double run_duration_ms = 1;
optional string git_hash = 2;
repeated string tags = 3;
optional double timestamp_ms = 4;
}

message PerfData {
optional Counters counters = 1;
repeated Sample samples = 2;
optional Metadata metadata = 3;
}
@@ -0,0 +1,5 @@
# Go Quickstore Example

This example illustrates the use of the Mako Quickstore Go library to add
performance test results to http://mako.dev and perform regression detection.

0 comments on commit 8028f96

Please sign in to comment.
You can’t perform that action at this time.