resource_group_client: statistic sql cpu cost

client/go.mod

@@ -3,6 +3,7 @@ module github.com/tikv/pd/client
 go 1.20
 
 require (
+	github.com/elastic/gosigar v0.14.2
 	github.com/gogo/protobuf v1.3.2
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
@@ -23,6 +24,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect

client/go.sum

@@ -17,6 +17,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
+github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -169,6 +171,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

client/resource_group/controller/config.go

@@ -81,8 +81,8 @@ type RequestUnitConfig struct {
 	CPUMsCost float64 `toml:"read-cpu-ms-cost" json:"read-cpu-ms-cost"`
 }
 
-// defaultRequestUnitConfig returns the default request unit configuration.
-func defaultRequestUnitConfig() *RequestUnitConfig {
+// DefaultRequestUnitConfig returns the default request unit configuration.
+func DefaultRequestUnitConfig() *RequestUnitConfig {
 	return &RequestUnitConfig{
 		ReadBaseCost:     defaultReadBaseCost,
 		ReadCostPerByte:  defaultReadCostPerByte,
@@ -102,12 +102,14 @@ type Config struct {
 	WriteBaseCost  RequestUnit
 	WriteBytesCost RequestUnit
 	CPUMsCost      RequestUnit
+	// The CPU statistics need to distinguish between different environments.
+	isSingleGroupByKeyspace bool
 }
 
 // DefaultConfig returns the default configuration.
 func DefaultConfig() *Config {
 	return GenerateConfig(
-		defaultRequestUnitConfig(),
+		DefaultRequestUnitConfig(),
 	)
 }
 

client/resource_group/controller/controller.go

@@ -56,6 +56,16 @@ type ResourceGroupProvider interface {
 	LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]pd.GlobalConfigItem, int64, error)
 }
 
+// ResourceControlCreateOption create a ResourceGroupsController with the optional settings.
+type ResourceControlCreateOption func(controller *ResourceGroupsController)
+
+// EnableSingleGroupByKeyspace is the option to enable single group by keyspace feature.
+func EnableSingleGroupByKeyspace() ResourceControlCreateOption {
+	return func(controller *ResourceGroupsController) {
+		controller.config.isSingleGroupByKeyspace = true
+	}
+}
+
 var _ ResourceGroupKVInterceptor = (*ResourceGroupsController)(nil)
 
 // ResourceGroupsController impls ResourceGroupKVInterceptor.
@@ -82,7 +92,7 @@ type ResourceGroupsController struct {
 		lastRequestTime time.Time
 
 		// requestInProgress is true if we are in the process of sending a request.
-		// It gets set to false when we receives the response in the main loop,
+		// It gets set to false when we receive the response in the main loop,
 		// even in error cases.
 		requestInProgress bool
 
@@ -100,6 +110,7 @@ func NewResourceGroupController(
 	clientUniqueID uint64,
 	provider ResourceGroupProvider,
 	requestUnitConfig *RequestUnitConfig,
+	opts ...ResourceControlCreateOption,
 ) (*ResourceGroupsController, error) {
 	if requestUnitConfig == nil {
 		var err error
@@ -109,15 +120,19 @@ func NewResourceGroupController(
 		}
 	}
 	config := GenerateConfig(requestUnitConfig)
-	return &ResourceGroupsController{
+	controller := &ResourceGroupsController{
 		clientUniqueID:        clientUniqueID,
 		provider:              provider,
 		config:                config,
 		lowTokenNotifyChan:    make(chan struct{}, 1),
 		tokenResponseChan:     make(chan []*rmpb.TokenBucketResponse, 1),
 		tokenBucketUpdateChan: make(chan *groupCostController, maxNotificationChanLen),
-		calculators:           []ResourceCalculator{newKVCalculator(config), newSQLCalculator(config)},
-	}, nil
+	}
+	for _, opt := range opts {
+		opt(controller)
+	}
+	controller.calculators = []ResourceCalculator{newKVCalculator(controller.config), newSQLCalculator(controller.config)}
+	return controller, nil
 }
 
 func loadRequestUnitConfig(ctx context.Context, provider ResourceGroupProvider) (*RequestUnitConfig, error) {
@@ -158,7 +173,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
 			case resp := <-c.tokenResponseChan:
 				c.run.requestInProgress = false
 				if resp != nil {
-					c.updateRunState(c.loopCtx)
+					c.updateRunState()
 					c.handleTokenBucketResponse(resp)
 				} else {
 					// A nil response indicates a failure (which would have been logged).
@@ -169,14 +184,14 @@ func (c *ResourceGroupsController) Start(ctx context.Context) {
 					log.Error("[resource group controller] clean up resource groups failed", zap.Error(err))
 				}
 			case <-stateUpdateTicker.C:
-				c.updateRunState(c.loopCtx)
+				c.updateRunState()
 				c.updateAvgRequestResourcePerSec()
 				if c.run.requestNeedsRetry || c.shouldReportConsumption() {
 					c.run.requestNeedsRetry = false
 					c.collectTokenBucketRequests(c.loopCtx, "report", false /* select all */)
 				}
 			case <-c.lowTokenNotifyChan:
-				c.updateRunState(c.loopCtx)
+				c.updateRunState()
 				c.updateAvgRequestResourcePerSec()
 				if !c.run.requestInProgress {
 					c.collectTokenBucketRequests(c.loopCtx, "low_ru", true /* only select low tokens resource group */)
@@ -256,11 +271,11 @@ func (c *ResourceGroupsController) initRunState() {
 	c.run.lastRequestTime = now
 }
 
-func (c *ResourceGroupsController) updateRunState(ctx context.Context) {
+func (c *ResourceGroupsController) updateRunState() {
 	c.run.now = time.Now()
 	c.groupsController.Range(func(name, value any) bool {
 		gc := value.(*groupCostController)
-		gc.updateRunState(ctx)
+		gc.updateRunState()
 		return true
 	})
 }
@@ -477,7 +492,7 @@ func (gc *groupCostController) initRunState() {
 	gc.run.now = now
 	gc.run.targetPeriod = defaultTargetPeriod
 	gc.run.consumption = &rmpb.Consumption{}
-	gc.run.lastRequestConsumption = &rmpb.Consumption{}
+	gc.run.lastRequestConsumption = &rmpb.Consumption{SqlLayerCpuTimeMs: getSQLProcessCPUTime(gc.mainCfg.isSingleGroupByKeyspace)}
 
 	cfgFunc := func(tb *rmpb.TokenBucket) tokenBucketReconfigureArgs {
 		cfg := tokenBucketReconfigureArgs{
@@ -522,31 +537,14 @@ func (gc *groupCostController) initRunState() {
 	}
 }
 
-func (gc *groupCostController) updateRunState(ctx context.Context) {
+func (gc *groupCostController) updateRunState() {
 	newTime := time.Now()
-	deltaConsumption := &rmpb.Consumption{}
+	gc.mu.Lock()
 	for _, calc := range gc.calculators {
-		calc.Trickle(ctx, deltaConsumption)
+		calc.Trickle(gc.mu.consumption)
 	}
-	gc.mu.Lock()
-	add(gc.mu.consumption, deltaConsumption)
 	*gc.run.consumption = *gc.mu.consumption
 	gc.mu.Unlock()
-	// remove tokens
-	switch gc.mode {
-	case rmpb.GroupMode_RUMode:
-		for typ, counter := range gc.run.requestUnitTokens {
-			if v := getRUValueFromConsumption(deltaConsumption, typ); v > 0 {
-				counter.limiter.RemoveTokens(newTime, v)
-			}
-		}
-	case rmpb.GroupMode_RawMode:
-		for typ, counter := range gc.run.resourceTokens {
-			if v := getRawResourceValueFromConsumption(deltaConsumption, typ); v > 0 {
-				counter.limiter.RemoveTokens(newTime, v)
-			}
-		}
-	}
 	log.Debug("[resource group controller] update run state", zap.Any("request unit consumption", gc.run.consumption))
 	gc.run.now = newTime
 }

client/resource_group/controller/model.go

@@ -15,7 +15,11 @@
 package controller
 
 import (
-	"context"
+	"os"
+
+	"github.com/elastic/gosigar"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
 
 	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
 )
@@ -26,14 +30,14 @@ import (
 type RequestUnit float64
 
 // RequestInfo is the interface of the request information provider. A request should be
-// able tell whether it's a write request and if so, the written bytes would also be provided.
+// able to tell whether it's a write request and if so, the written bytes would also be provided.
 type RequestInfo interface {
 	IsWrite() bool
 	WriteBytes() uint64
 }
 
 // ResponseInfo is the interface of the response information provider. A response should be
-// able tell how many bytes it read and KV CPU cost in milliseconds.
+// able to tell how many bytes it read and KV CPU cost in milliseconds.
 type ResponseInfo interface {
 	ReadBytes() uint64
 	KVCPUMs() uint64
@@ -46,7 +50,8 @@ type ResponseInfo interface {
 type ResourceCalculator interface {
 	// Trickle is used to calculate the resource consumption periodically rather than on the request path.
 	// It's mainly used to calculate like the SQL CPU cost.
-	Trickle(context.Context, *rmpb.Consumption)
+	// Need to check if it is a serverless environment
+	Trickle(*rmpb.Consumption)
 	// BeforeKVRequest is used to calculate the resource consumption before the KV request.
 	// It's mainly used to calculate the base and write request cost.
 	BeforeKVRequest(*rmpb.Consumption, RequestInfo)
@@ -67,7 +72,7 @@ func newKVCalculator(cfg *Config) *KVCalculator {
 }
 
 // Trickle ...
-func (kc *KVCalculator) Trickle(ctx context.Context, consumption *rmpb.Consumption) {
+func (kc *KVCalculator) Trickle(*rmpb.Consumption) {
 }
 
 // BeforeKVRequest ...
@@ -133,9 +138,11 @@ func newSQLCalculator(cfg *Config) *SQLCalculator {
 	return &SQLCalculator{Config: cfg}
 }
 
-// Trickle ...
-// TODO: calculate the SQL CPU cost and related resource consumption.
-func (dsc *SQLCalculator) Trickle(ctx context.Context, consumption *rmpb.Consumption) {
+// Trickle update sql layer CPU consumption.
+func (dsc *SQLCalculator) Trickle(consumption *rmpb.Consumption) {
+	delta := getSQLProcessCPUTime(dsc.isSingleGroupByKeyspace) - consumption.SqlLayerCpuTimeMs
+	consumption.TotalCpuTimeMs += delta
+	consumption.SqlLayerCpuTimeMs += delta
 }
 
 // BeforeKVRequest ...
@@ -205,3 +212,26 @@ func sub(custom1 *rmpb.Consumption, custom2 *rmpb.Consumption) {
 	custom1.KvReadRpcCount -= custom2.KvReadRpcCount
 	custom1.KvWriteRpcCount -= custom2.KvWriteRpcCount
 }
+
+// getSQLProcessCPUTime returns the cumulative user+system time (in ms) since the process start.
+func getSQLProcessCPUTime(isSingleGroupByKeyspace bool) float64 {
+	if isSingleGroupByKeyspace {
+		return getSysProcessCPUTime()
+	}
+	return getGroupProcessCPUTime()
+}
+
+func getSysProcessCPUTime() float64 {
+	pid := os.Getpid()
+	cpuTime := gosigar.ProcTime{}
+	if err := cpuTime.Get(pid); err != nil {
+		log.Error("getCPUTime get pid failed", zap.Error(err))
+	}
+
+	return float64(cpuTime.User + cpuTime.Sys)
+}
+
+// TODO: Need a way to calculate in the case of multiple groups.
+func getGroupProcessCPUTime() float64 {
+	return 0
+}

pkg/mcs/resource_manager/server/manager.go

@@ -264,6 +264,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 				name                     = consumptionInfo.resourceGroupName
 				rruMetrics               = readRequestUnitCost.WithLabelValues(name)
 				wruMetrics               = writeRequestUnitCost.WithLabelValues(name)
+				sqlLayerRuMetrics        = sqlLayerRequestUnitCost.WithLabelValues(name)
 				readByteMetrics          = readByteCost.WithLabelValues(name)
 				writeByteMetrics         = writeByteCost.WithLabelValues(name)
 				kvCPUMetrics             = kvCPUCost.WithLabelValues(name)
@@ -288,6 +289,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 			// CPU time info.
 			if consumption.TotalCpuTimeMs > 0 {
 				if consumption.SqlLayerCpuTimeMs > 0 {
+					sqlLayerRuMetrics.Add(consumption.SqlLayerCpuTimeMs * m.ruConfig.CPUMsCost)
 					sqlCPUMetrics.Observe(consumption.SqlLayerCpuTimeMs)
 				}
 				kvCPUMetrics.Observe(consumption.TotalCpuTimeMs - consumption.SqlLayerCpuTimeMs)
@@ -308,6 +310,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 				if time.Since(lastTime) > metricsCleanupTimeout {
 					readRequestUnitCost.DeleteLabelValues(name)
 					writeRequestUnitCost.DeleteLabelValues(name)
+					sqlLayerRequestUnitCost.DeleteLabelValues(name)
 					readByteCost.DeleteLabelValues(name)
 					writeByteCost.DeleteLabelValues(name)
 					kvCPUCost.DeleteLabelValues(name)

pkg/mcs/resource_manager/server/metrics.go

@@ -46,6 +46,13 @@ var (
 			Help:      "Bucketed histogram of the write request unit cost for all resource groups.",
 			Buckets:   prometheus.ExponentialBuckets(3, 10, 5), // 3 ~ 300000
 		}, []string{resourceGroupNameLabel})
+	sqlLayerRequestUnitCost = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: ruSubsystem,
+			Name:      "sql_layer_request_unit",
+			Help:      "The number of the sql layer request unit cost for all resource groups.",
+		}, []string{resourceGroupNameLabel})
 
 	// Resource cost metrics.
 	readByteCost = prometheus.NewHistogramVec(
@@ -92,6 +99,7 @@ var (
 func init() {
 	prometheus.MustRegister(readRequestUnitCost)
 	prometheus.MustRegister(writeRequestUnitCost)
+	prometheus.MustRegister(sqlLayerRequestUnitCost)
 	prometheus.MustRegister(readByteCost)
 	prometheus.MustRegister(writeByteCost)
 	prometheus.MustRegister(kvCPUCost)

tests/mcs/go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/elliotchance/pie/v2 v2.1.0 // indirect
 	github.com/fogleman/gg v1.3.0 // indirect
 	github.com/gin-contrib/cors v1.4.0 // indirect

tests/mcs/go.sum

@@ -88,6 +88,8 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05w
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
+github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4=
 github.com/elliotchance/pie/v2 v2.1.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -630,6 +632,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

tests/mcs/resource_manager/resource_manager_test.go

@@ -301,7 +301,7 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() {
 		CPUMsCost:        1,
 	}
 
-	controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg)
+	controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg, controller.EnableSingleGroupByKeyspace())
 	controller.Start(suite.ctx)
 
 	testCases := []struct {

tools/pd-tso-bench/go.sum

@@ -412,6 +412,7 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -775,6 +776,7 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=