extension/serverless: introduce serverless audit log (pingcap#870)

* init * audit: init first version of audit log * audit: support log global rotate * audit: update log keys * audit: more common notify implement * audit: add record id * audit: rename some names * audit: default use normal log path * audit: add server ip info * audit: do some refine * audit: Add some log keys * audit: update some item format * audit: add stmt demo * audit: support redact * audit: fix bug * audit: update filter * audit: update filter * audit: check user * audit: update * update * audit: update * audit: fix bug * aduit: update * audit: update * update * update * audit: add filter unit test * fmt * Add tests for `tidb_audit_enabled` and `tidb_audit_log` * Add tests for `tidb_audit_log_max_size` and `tidb_audit_log_max_lifetime` * Add tests for `tidb_audit_log_reserved_*` * TODO: TestAuditLogRedact * Fininsh `TestAuditLogRedact` * Update (pingcap#1) * fix typo (pingcap#2) * Add `TRANSACTION` * rename sysvar * audit_log_create_filter, audit_log_remove_filter * finish function call and table test * test privilege * finish test for sysvar * TODO: TestConnectionEvenClass * update * audit: fix lint for audit log (pingcap#1) * audit: fix UT failure caused by the change of redact log (pingcap#3) * audit: use `t.TempDir()` to make test stable (pingcap#4) * audit: fix test failed for 7.1 * Add `OWNERS` file (pingcap#35) Co-authored-by: Chao Wang <cclcwangchao@hotmail.com> * audit: fix panic when logging sometime (pingcap#26) (pingcap#31) * test: fix unstable test TestAuditLogReservedDays (pingcap#8) * Format sysvar_test.go * audit: use `StatementContext` to generate redacted SQL (pingcap#9) * audit: fix panic when logging sometime (pingcap#26) * update * add owner * Update OWNERS --------- Co-authored-by: CbcWestwolf <1004626265@qq.com> Co-authored-by: wuhuizuo <wuhuizuo@126.com> * audit: fix panic sometimes when `create user` without password (pingcap#37) (pingcap#39) * This is an automated cherry-pick of pingcap#37 Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> * fix conflict --------- Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> Co-authored-by: 王超 <cclcwangchao@hotmail.com> * extension/audit: introduce serverless audit log Signed-off-by: Wen Jiazhi <jiazhi.wen@pingcap.com> * update bazel config Signed-off-by: Wen Jiazhi <jiazhi.wen@pingcap.com> * support enable audit log when activate Signed-off-by: Wen Jiazhi <jiazhi.wen@pingcap.com> * add gwconnid to audit log Signed-off-by: Wen Jiazhi <jiazhi.wen@pingcap.com> * comment unstable test * update bazel * add log about activate request * diff audit log enabled * audit: fix memory leak for executeSQL (pingcap#44) (pingcap#45) * This is an automated cherry-pick of pingcap#44 Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> * Update util.go --------- Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> Co-authored-by: 王超 <cclcwangchao@hotmail.com> * Update extension/serverless/OWNERS --------- Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> Signed-off-by: Wen Jiazhi <jiazhi.wen@pingcap.com> Co-authored-by: Chao Wang <cclcwangchao@hotmail.com> Co-authored-by: cbcwestwolf <1004626265@qq.com> Co-authored-by: Ti Chi Robot <ti-community-prow-bot@tidb.io> Co-authored-by: wuhuizuo <wuhuizuo@126.com> Co-authored-by: zzm <zhouzemin@pingcap.com> Co-authored-by: Yuqing Bai <baiyuq@gmail.com>
rleungx · Feb 21, 2024 · aef4fc1 · aef4fc1
1 parent eef5296
commit aef4fc1
Show file tree

Hide file tree

Showing 23 changed files with 4,579 additions and 2 deletions.
diff --git a/config/config.go b/config/config.go
@@ -322,6 +322,8 @@ type Config struct {
 
 	TiDBWorker TiDBWorker `toml:"tidb-worker" json:"tidb-worker"`
 
+	AuditLog AuditLog `toml:"audit-log" json:"audit-log"`
+
 	// The following items are deprecated. We need to keep them here temporarily
 	// to support the upgrade process. They can be removed in future.
 
@@ -1188,6 +1190,7 @@ var defaultConf = Config{
 	},
 	RewriteCollations: make(map[string]map[string]string),
 	TiDBWorker:        defaultTiDBWorker(),
+	AuditLog:          defaultAuditLog(),
 	ExportID:          "",
 }
 

diff --git a/config/serverless.go b/config/serverless.go
@@ -147,3 +147,40 @@ func (w *TiDBWorker) Valid(c *Config) error {
 	}
 	return nil
 }
+
+const (
+	// LogFormatText is stardard log format surrounded with []
+	LogFormatText = "TEXT"
+	// LogFormatJSON is json format
+	LogFormatJSON = "JSON"
+)
+
+// AuditLog is the config for serverless audit log
+type AuditLog struct {
+	// Enable indicates whether audit log will be recorded
+	Enable bool `toml:"enable" json:"enable"`
+	// Path is the audit log output path
+	Path string `toml:"path" json:"path"`
+	// Format is the output format of audit log, both text and json are supported
+	Format string `toml:"format" json:"format"`
+	// MaxFilesize is the maximum file size before the log file be rotated, unit is MB
+	MaxFilesize int64 `toml:"max-filesize" json:"max-filesize"`
+	// MaxLifetime is the maximum time before the log file be rotated, unit is second
+	MaxLifetime int64 `toml:"max-lifetime" json:"max-lifetime"`
+	// Redacted indicates whether audit log redaction is enabled. If it set to true, user data will be replaced with `?`
+	Redacted bool `toml:"redacted" json:"redacted"`
+	// EncryptKey is used to encrypt sensitive informations in audit log. This key should be 32 bytes, we use AES-256 for encryption
+	EncryptKey string `toml:"encrypt-key" json:"encrypt-key"`
+}
+
+func defaultAuditLog() AuditLog {
+	return AuditLog{
+		Enable:      false,
+		Path:        "tidb-audit.log",
+		Format:      LogFormatText,
+		MaxFilesize: 10,
+		MaxLifetime: 24 * 60 * 60,
+		Redacted:    true,
+		EncryptKey:  "",
+	}
+}
diff --git a/extension/serverless/OWNERS b/extension/serverless/OWNERS
@@ -0,0 +1,4 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+approvers:
+reviewers: []
+
diff --git a/extension/serverless/README.md b/extension/serverless/README.md
@@ -0,0 +1 @@
+This repo maintains the tidb enterprise extensions code.
diff --git a/extension/serverless/audit/BUILD.bazel b/extension/serverless/audit/BUILD.bazel
@@ -0,0 +1,72 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "audit",
+    srcs = [
+        "entry.go",
+        "filter.go",
+        "logger.go",
+        "manager.go",
+        "register.go",
+        "util.go",
+    ],
+    importpath = "github.com/pingcap/tidb/extension/serverless/audit",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//config",
+        "//extension",
+        "//keyspace",
+        "//kv",
+        "//metrics",
+        "//parser/ast",
+        "//parser/auth",
+        "//parser/mysql",
+        "//parser/terror",
+        "//sessionctx/stmtctx",
+        "//sessionctx/variable",
+        "//types",
+        "//util/chunk",
+        "//util/encrypt",
+        "//util/logutil",
+        "//util/sqlexec",
+        "//util/stringutil",
+        "//util/table-filter",
+        "@com_github_google_uuid//:uuid",
+        "@com_github_ngaut_pools//:pools",
+        "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_log//:log",
+        "@in_gopkg_natefinch_lumberjack_v2//:lumberjack_v2",
+        "@io_etcd_go_etcd_api_v3//mvccpb",
+        "@io_etcd_go_etcd_client_v3//:client",
+        "@org_uber_go_atomic//:atomic",
+        "@org_uber_go_zap//:zap",
+        "@org_uber_go_zap//zapcore",
+    ],
+)
+
+go_test(
+    name = "audit_test",
+    timeout = "short",
+    srcs = [
+        "entry_test.go",
+        "filter_test.go",
+        "function_test.go",
+        "sysvar_test.go",
+    ],
+    embed = [":audit"],
+    flaky = True,
+    shard_count = 20,
+    deps = [
+        "//config",
+        "//errno",
+        "//parser",
+        "//parser/auth",
+        "//server",
+        "//sessionctx/stmtctx",
+        "//testkit",
+        "//util/sem",
+        "@com_github_pingcap_errors//:errors",
+        "@com_github_stretchr_testify//require",
+        "@org_uber_go_zap//:zap",
+    ],
+)