pingcap · ti-chi-bot · Apr 12, 2023
diff --git a/statistics/handle/BUILD.bazel b/statistics/handle/BUILD.bazel
@@ -0,0 +1,96 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "handle",
+    srcs = [
+        "bootstrap.go",
+        "ddl.go",
+        "dump.go",
+        "gc.go",
+        "handle.go",
+        "handle_hist.go",
+        "historical_stats_handler.go",
+        "lru_cache.go",
+        "statscache.go",
+        "update.go",
+    ],
+    importpath = "github.com/pingcap/tidb/statistics/handle",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//config",
+        "//ddl/util",
+        "//infoschema",
+        "//kv",
+        "//metrics",
+        "//parser/ast",
+        "//parser/model",
+        "//parser/mysql",
+        "//parser/terror",
+        "//sessionctx",
+        "//sessionctx/stmtctx",
+        "//sessionctx/variable",
+        "//sessiontxn",
+        "//statistics",
+        "//statistics/handle/metrics",
+        "//table",
+        "//types",
+        "//util",
+        "//util/chunk",
+        "//util/codec",
+        "//util/collate",
+        "//util/logutil",
+        "//util/mathutil",
+        "//util/memory",
+        "//util/ranger",
+        "//util/sqlexec",
+        "//util/syncutil",
+        "//util/timeutil",
+        "@com_github_ngaut_pools//:pools",
+        "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_pingcap_log//:log",
+        "@com_github_pingcap_tipb//go-tipb",
+        "@com_github_tikv_client_go_v2//oracle",
+        "@org_golang_x_exp//slices",
+        "@org_uber_go_atomic//:atomic",
+        "@org_uber_go_zap//:zap",
+    ],
+)
+
+go_test(
+    name = "handle_test",
+    timeout = "short",
+    srcs = [
+        "ddl_test.go",
+        "dump_test.go",
+        "gc_test.go",
+        "handle_hist_test.go",
+        "lru_cache_test.go",
+        "main_test.go",
+        "update_list_test.go",
+    ],
+    embed = [":handle"],
+    flaky = True,
+    race = "on",
+    shard_count = 34,
+    deps = [
+        "//config",
+        "//domain",
+        "//parser/model",
+        "//parser/mysql",
+        "//sessionctx/stmtctx",
+        "//sessionctx/variable",
+        "//statistics",
+        "//statistics/handle/internal",
+        "//testkit",
+        "//testkit/testsetup",
+        "//types",
+        "//util",
+        "//util/mathutil",
+        "//util/mock",
+        "//util/sqlexec",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_stretchr_testify//require",
+        "@org_uber_go_goleak//:goleak",
+    ],
+)
diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go
@@ -272,11 +272,13 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			hist := statistics.HistogramFromProto(jsonIdx.Histogram)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.Correlation = idxInfo.ID, jsonIdx.NullCount, jsonIdx.LastUpdateVersion, jsonIdx.Correlation
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonIdx.CMSketch)
-			// If the statistics is loaded from a JSON without stats version,
-			// we set it to 1.
-			statsVer := int64(statistics.Version1)
+			statsVer := int64(statistics.Version0)
 			if jsonIdx.StatsVer != nil {
 				statsVer = *jsonIdx.StatsVer
+			} else if jsonIdx.Histogram.Ndv > 0 || jsonIdx.NullCount > 0 {
+				// If the statistics are collected without setting stats version(which happens in v4.0 and earlier versions),
+				// we set it to 1.
+				statsVer = int64(statistics.Version1)
 			}
 			idx := &statistics.Index{
 				Histogram: *hist,
@@ -303,11 +305,13 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch)
 			fms := statistics.FMSketchFromProto(jsonCol.FMSketch)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation
-			// If the statistics is loaded from a JSON without stats version,
-			// we set it to 1.
-			statsVer := int64(statistics.Version1)
+			statsVer := int64(statistics.Version0)
 			if jsonCol.StatsVer != nil {
 				statsVer = *jsonCol.StatsVer
+			} else if jsonCol.Histogram.Ndv > 0 || jsonCol.NullCount > 0 {
+				// If the statistics are collected without setting stats version(which happens in v4.0 and earlier versions),
+				// we set it to 1.
+				statsVer = int64(statistics.Version1)
 			}
 			col := &statistics.Column{
 				PhysicalID: physicalID,

diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go
@@ -384,3 +384,159 @@ func TestDumpVer2Stats(t *testing.T) {
 	// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
 	requireTableEqual(t, statsCacheTbl, loadTbl)
 }
+<<<<<<< HEAD
+=======
+
+func TestLoadStatsForNewCollation(t *testing.T) {
+	// This test is almost the same as TestDumpVer2Stats, except that: b varchar(10) => b varchar(3) collate utf8mb4_unicode_ci
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b varchar(3) collate utf8mb4_unicode_ci)")
+	tk.MustExec("insert into t value(1, 'aaa'), (1, 'aaa'), (3, 'aab'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc'), (7, 'Ste')")
+	// mark column stats as needed
+	tk.MustExec("select * from t where a = 3")
+	tk.MustExec("select * from t where b = 'bbb'")
+	tk.MustExec("alter table t add index single(a)")
+	tk.MustExec("alter table t add index multi(a, b)")
+	tk.MustExec("analyze table t with 2 topn")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+
+	storageTbl, err := h.TableStatsFromStorage(tableInfo.Meta(), tableInfo.Meta().ID, false, 0)
+	require.NoError(t, err)
+
+	dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true)
+	require.NoError(t, err)
+
+	jsonBytes, err := json.MarshalIndent(dumpJSONTable, "", " ")
+	require.NoError(t, err)
+
+	loadJSONTable := &handle.JSONTable{}
+	err = json.Unmarshal(jsonBytes, loadJSONTable)
+	require.NoError(t, err)
+
+	loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, loadJSONTable)
+	require.NoError(t, err)
+
+	// assert that a statistics.Table from storage dumped into JSON text and then unmarshalled into a statistics.Table keeps unchanged
+	requireTableEqual(t, loadTbl, storageTbl)
+
+	// assert that this statistics.Table is the same as the one in stats cache
+	statsCacheTbl := h.GetTableStats(tableInfo.Meta())
+	requireTableEqual(t, loadTbl, statsCacheTbl)
+
+	err = h.LoadStatsFromJSON(is, loadJSONTable)
+	require.NoError(t, err)
+	require.Nil(t, h.Update(is))
+	statsCacheTbl = h.GetTableStats(tableInfo.Meta())
+	// assert that after the JSONTable above loaded into storage then updated into the stats cache,
+	// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
+	requireTableEqual(t, statsCacheTbl, loadTbl)
+}
+
+func TestJSONTableToBlocks(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b varchar(10))")
+	tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
+	// mark column stats as needed
+	tk.MustExec("select * from t where a = 3")
+	tk.MustExec("select * from t where b = 'bbb'")
+	tk.MustExec("alter table t add index single(a)")
+	tk.MustExec("alter table t add index multi(a, b)")
+	tk.MustExec("analyze table t with 2 topn")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+
+	dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true)
+	require.NoError(t, err)
+	jsOrigin, _ := json.Marshal(dumpJSONTable)
+
+	blockSize := 30
+	js, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true)
+	require.NoError(t, err)
+	dumpJSONBlocks, err := handle.JSONTableToBlocks(js, blockSize)
+	require.NoError(t, err)
+	jsConverted, err := handle.BlocksToJSONTable(dumpJSONBlocks)
+	require.NoError(t, err)
+	jsonStr, err := json.Marshal(jsConverted)
+	require.NoError(t, err)
+	require.JSONEq(t, string(jsOrigin), string(jsonStr))
+}
+
+func TestLoadStatsFromOldVersion(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, index idx(b))")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	require.NoError(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+	require.NoError(t, h.Update(is))
+
+	statsJSONFromOldVersion := `{
+ "database_name": "test",
+ "table_name": "t",
+ "columns": {
+  "a": {
+   "histogram": {
+    "ndv": 0
+   },
+   "cm_sketch": null,
+   "null_count": 0,
+   "tot_col_size": 256,
+   "last_update_version": 440735055846047747,
+   "correlation": 0
+  },
+  "b": {
+   "histogram": {
+    "ndv": 0
+   },
+   "cm_sketch": null,
+   "null_count": 0,
+   "tot_col_size": 256,
+   "last_update_version": 440735055846047747,
+   "correlation": 0
+  }
+ },
+ "indices": {
+  "idx": {
+   "histogram": {
+    "ndv": 0
+   },
+   "cm_sketch": null,
+   "null_count": 0,
+   "tot_col_size": 0,
+   "last_update_version": 440735055846047747,
+   "correlation": 0
+  }
+ },
+ "count": 256,
+ "modify_count": 256,
+ "partitions": null
+}`
+	jsonTbl := &handle.JSONTable{}
+	require.NoError(t, json.Unmarshal([]byte(statsJSONFromOldVersion), jsonTbl))
+	require.NoError(t, h.LoadStatsFromJSON(is, jsonTbl))
+	tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+	statsTbl := h.GetTableStats(tbl.Meta())
+	for _, col := range statsTbl.Columns {
+		require.False(t, col.IsStatsInitialized())
+	}
+	for _, idx := range statsTbl.Indices {
+		require.False(t, idx.IsStatsInitialized())
+	}
+}
+>>>>>>> db68a1222e9 (staistics: fix load stats from old version json (#42967))