config: add config file support. (#4509)

config/config.go

@@ -13,38 +13,140 @@
 
 package config
 
-import "sync"
+import (
+	"github.com/BurntSushi/toml"
+	"github.com/juju/errors"
+	"github.com/pingcap/pd/pkg/logutil"
+)
 
 // Config contains configuration options.
 type Config struct {
-	Addr           string `json:"addr" toml:"addr"`
-	LogLevel       string `json:"log_level" toml:"log_level"`
-	SkipAuth       bool   `json:"skip_auth" toml:"skip_auth"`
-	StatusAddr     string `json:"status_addr" toml:"status_addr"`
-	Socket         string `json:"socket" toml:"socket"`
-	ReportStatus   bool   `json:"report_status" toml:"report_status"`
-	StorePath      string `json:"store_path" toml:"store_path"`
-	Store          string `json:"store" toml:"store"`
-	SlowThreshold  int    `json:"slow_threshold" toml:"slow_threshold"`
-	QueryLogMaxlen int    `json:"query_log_max_len" toml:"query_log_max_len"`
-	TCPKeepAlive   bool   `json:"tcp_keep_alive" toml:"tcp_keep_alive"`
-	SSLCAPath      string `json:"ssl_ca_path" toml:"ssl_ca_path"`
-	SSLCertPath    string `json:"ssl_cert_path" toml:"ssl_cert_path"`
-	SSLKeyPath     string `json:"ssl_key_path" toml:"ssl_key_path"`
-}
-
-var cfg *Config
-var once sync.Once
+	Host         string `toml:"host" json:"host"`
+	Port         int    `toml:"port" json:"port"`
+	Store        string `toml:"store" json:"store"`
+	Path         string `toml:"path" json:"path"`
+	Socket       string `toml:"socket" json:"socket"`
+	BinlogSocket string `toml:"binlog-socket" json:"binlog-socket"`
+	Lease        string `toml:"lease" json:"lease"`
+	RunDDL       bool   `toml:"run-ddl" json:"run-ddl"`
+
+	Log         Log         `toml:"log" json:"log"`
+	Security    Security    `toml:"security" json:"security"`
+	Status      Status      `toml:"status" json:"status"`
+	Performance Performance `toml:"performance" json:"performance"`
+	XProtocol   XProtocol   `toml:"xprotocol" json:"xprotocol"`
+}
+
+// Log is the log section of config.
+type Log struct {
+	// Log level.
+	Level string `toml:"level" json:"level"`
+	// Log format. one of json, text, or console.
+	Format string `toml:"format" json:"format"`
+	// Disable automatic timestamps in output.
+	DisableTimestamp bool `toml:"disable-timestamp" json:"disable-timestamp"`
+	// File log config.
+	File logutil.FileLogConfig `toml:"file" json:"file"`
+
+	SlowThreshold int `toml:"slow-threshold" json:"slow-threshold"`
+
+	QueryLogMaxLen int `toml:"query-log-max-len" json:"query-log-max-len"`
+}
+
+// Security is the security section of the config.
+type Security struct {
+	SkipGrantTable bool   `toml:"skip-grant-table" json:"skip-grant-table"`
+	SSLCA          string `toml:"ssl-ca" json:"ssl-ca"`
+	SSLCert        string `toml:"ssl-cert" json:"ssl-cert"`
+	SSLKey         string `toml:"ssl-key" json:"ssl-key"`
+}
+
+// Status is the status section of the config.
+type Status struct {
+	ReportStatus    bool   `toml:"report-status" json:"report-status"`
+	StatusPort      int    `toml:"status-port" json:"status-port"`
+	MetricsAddr     string `toml:"metrics-addr" json:"metrics-addr"`
+	MetricsInterval int    `toml:"metrics-interval" json:"metrics-interval"`
+}
+
+// Performance is the performance section of the config.
+type Performance struct {
+	TCPKeepAlive    bool   `toml:"tcp-keep-alive" json:"tcp-keep-alive"`
+	RetryLimit      int    `toml:"retry-limit" json:"retry-limit"`
+	JoinConcurrency int    `toml:"join-concurrency" json:"join-concurrency"`
+	CrossJoin       bool   `toml:"cross-join" json:"cross-join"`
+	StatsLease      string `toml:"stats-lease" json:"stats-lease"`
+}
+
+// XProtocol is the XProtocol section of the config.
+type XProtocol struct {
+	XServer bool   `toml:"xserver" json:"xserver"`
+	XHost   string `toml:"xhost" json:"xhost"`
+	XPort   int    `toml:"xport" json:"xport"`
+	XSocket string `toml:"xsocket" json:"xsocket"`
+}
+
+var defaultConf = Config{
+	Host:   "0.0.0.0",
+	Port:   4000,
+	Store:  "mocktikv",
+	Path:   "/tmp/tidb",
+	RunDDL: true,
+	Lease:  "10s",
+	Log: Log{
+		Level:  "info",
+		Format: "text",
+		File: logutil.FileLogConfig{
+			LogRotate: true,
+		},
+		SlowThreshold:  300,
+		QueryLogMaxLen: 2048,
+	},
+	Status: Status{
+		ReportStatus:    true,
+		StatusPort:      10080,
+		MetricsInterval: 15,
+	},
+	Performance: Performance{
+		TCPKeepAlive:    true,
+		RetryLimit:      10,
+		JoinConcurrency: 5,
+		CrossJoin:       true,
+		StatsLease:      "3s",
+	},
+	XProtocol: XProtocol{
+		XHost: "0.0.0.0",
+		XPort: 14000,
+	},
+}
+
+var globalConf = defaultConf
+
+// NewConfig creates a new config instance with default value.
+func NewConfig() *Config {
+	conf := defaultConf
+	return &conf
+}
 
 // GetGlobalConfig returns the global configuration for this server.
 // It should store configuration from command line and configuration file.
 // Other parts of the system can read the global configuration use this function.
 func GetGlobalConfig() *Config {
-	once.Do(func() {
-		cfg = &Config{
-			SlowThreshold:  300,
-			QueryLogMaxlen: 2048,
-		}
-	})
-	return cfg
+	return &globalConf
+}
+
+// Load loads config options from a toml file.
+func (c *Config) Load(confFile string) error {
+	_, err := toml.DecodeFile(confFile, c)
+	return errors.Trace(err)
+}
+
+// ToLogConfig converts *Log to *logutil.LogConfig.
+func (l *Log) ToLogConfig() *logutil.LogConfig {
+	return &logutil.LogConfig{
+		Level:            l.Level,
+		Format:           l.Format,
+		DisableTimestamp: l.DisableTimestamp,
+		File:             l.File,
+	}
 }

config/config.toml.example

@@ -0,0 +1,113 @@
+# TiDB Configuration.
+
+# TiDB server host.
+host = "0.0.0.0"
+
+# TiDB server port.
+port = 4000
+
+# Registered store name, [memory, goleveldb, boltdb, tikv, mocktikv]
+store = "mocktikv"
+
+# TiDB storage path.
+path = "/tmp/tidb"
+
+# The socket file to use for connection.
+#socket = ""
+
+# Socket file to write binlog.
+#binlog-socket = ""
+
+# Run ddl worker on this tidb-server.
+run-ddl = true
+
+# Schema lease duration, very dangerous to change only if you know what you do.
+lease = "10s"
+
+[log]
+# Log level: info, debug, warn, error, fatal.
+level = "info"
+
+# Log format, one of json, text, console.
+format = "text"
+
+# Disable automatic timestamps in output
+disable-timestamp = false
+
+# Queries with execution time greater than this value will be logged. (Milliseconds)
+slow-threshold = 300
+
+# Maximum query length recorded in log.
+query-log-max-len = 2048
+
+# File logging.
+[log.file]
+# Log file name.
+filename = ""
+
+# Max log file size in MB.
+#max-size = 300
+
+# Max log file keep days.
+#max-days = 28
+
+# Maximum number of old log files to retain.
+#max-backups = 7
+
+# Rotate log by day
+log-rotate = true
+
+[security]
+# This option causes the server to start without using the privilege system at all.
+skip-grant-table = false
+
+# Path of file that contains list of trusted SSL CAs.
+ssl-ca = ""
+
+# Path of file that contains X509 certificate in PEM format.
+ssl-cert = ""
+
+# Path of file that contains X509 key in PEM format.
+ssl-key = ""
+
+[status]
+# If enable status report HTTP service.
+report-status = true
+
+# TiDB status port.
+status-port = 10080
+
+# Prometheus pushgateway address, leaves it empty will disable prometheus push.
+metrics-addr = ""
+
+# Prometheus client push interval in second, set \"0\" to disable prometheus push.
+metrics-interval = 15
+
+[performance]
+# Set keep alive option for tcp connection.
+tcp-keep-alive = true
+
+# The maximum number of retries when commit a transaction.
+retry-limit = 10
+
+# The number of goroutines that participate joining.
+join-concurrency = 5
+
+# Whether support cartesian product.
+cross-join = true
+
+# Stats lease duration, which inflences the time of analyze and stats load.
+stats-lease = "3s"
+
+[xprotocol]
+# Start TiDB x server.
+xserver = false
+
+# TiDB x protocol server host.
+xhost = "0.0.0.0"
+
+# TiDB x protocol server port.
+xport = 14000
+
+# The socket file to use for x protocol connection.
+xsocket = ""

config/config_test.go

@@ -0,0 +1,54 @@
+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"path"
+	"runtime"
+	"testing"
+
+	. "github.com/pingcap/check"
+)
+
+var _ = Suite(&testConfigSuite{})
+
+type testConfigSuite struct{}
+
+func TestT(t *testing.T) {
+	CustomVerboseFlag = true
+	TestingT(t)
+}
+
+func (s *testConfigSuite) TestConfig(c *C) {
+	conf := new(Config)
+	conf.BinlogSocket = "/tmp/socket"
+	conf.Performance.RetryLimit = 20
+
+	_, filename, _, _ := runtime.Caller(0)
+	configFile := path.Join(path.Dir(filename), "config.toml.example")
+	err := conf.Load(configFile)
+	c.Assert(err, IsNil)
+
+	// Test that the original value will not be clear by load the config file that does not contain the option.
+	c.Assert(conf.BinlogSocket, Equals, "/tmp/socket")
+
+	// Test that the value will be overwritten by the config file.
+	c.Assert(conf.Performance.RetryLimit, Equals, 10)
+
+	// Reset
+	conf.BinlogSocket = ""
+
+	// Make sure the example config is the same as default config.
+	c.Assert(conf, DeepEquals, GetGlobalConfig())
+}

executor/adapter.go

@@ -259,11 +259,11 @@ func (a *statement) logSlowQuery() {
 	cfg := config.GetGlobalConfig()
 	costTime := time.Since(a.startTime)
 	sql := a.text
-	if len(sql) > cfg.QueryLogMaxlen {
-		sql = sql[:cfg.QueryLogMaxlen] + fmt.Sprintf("(len:%d)", len(sql))
+	if len(sql) > cfg.Log.QueryLogMaxLen {
+		sql = sql[:cfg.Log.QueryLogMaxLen] + fmt.Sprintf("(len:%d)", len(sql))
 	}
 	connID := a.ctx.GetSessionVars().ConnectionID
-	if costTime < time.Duration(cfg.SlowThreshold)*time.Millisecond {
+	if costTime < time.Duration(cfg.Log.SlowThreshold)*time.Millisecond {
 		log.Debugf("[%d][TIME_QUERY] %v %s", connID, costTime, sql)
 	} else {
 		log.Warnf("[%d][TIME_QUERY] %v %s", connID, costTime, sql)

server/http_status.go

@@ -15,6 +15,7 @@ package server
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"sync"
 
@@ -50,8 +51,8 @@ func (s *Server) startHTTPServer() {
 		router.Handle("/mvcc/txn/{startTS}/{db}/{table}", mvccTxnHandler{tikvHandler, opMvccGetByTxn})
 		router.Handle("/mvcc/txn/{startTS}", mvccTxnHandler{tikvHandler, opMvccGetByTxn})
 	}
-	addr := s.cfg.StatusAddr
-	if len(addr) == 0 {
+	addr := fmt.Sprintf(":%d", s.cfg.Status.StatusPort)
+	if s.cfg.Status.StatusPort == 0 {
 		addr = defaultStatusAddr
 	}
 	log.Infof("Listening on %v for status and metrics report.", addr)

server/region_handler_test.go

@@ -178,18 +178,17 @@ func (ts *TidbRegionHandlerTestSuite) startServer(c *C) {
 	tidbdrv := NewTiDBDriver(store)
 
 	cfg := &config.Config{
-		Addr:         ":4001",
-		LogLevel:     "debug",
-		StatusAddr:   ":10090",
-		ReportStatus: true,
-		Store:        "tikv",
+		Port:  4001,
+		Store: "tikv",
 	}
+	cfg.Status.StatusPort = 10090
+	cfg.Status.ReportStatus = true
 
 	server, err := NewServer(cfg, tidbdrv)
 	c.Assert(err, IsNil)
 	ts.server = server
 	go server.Run()
-	waitUntilServerOnline(cfg.StatusAddr)
+	waitUntilServerOnline(cfg.Status.StatusPort)
 }
 
 func (ts *TidbRegionHandlerTestSuite) stopServer(c *C) {