From ec51a8fbe03ebb9d9bbc94ff504d0c632bc3147e Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 16 Jan 2024 15:59:50 +0800 Subject: [PATCH 01/10] plugin interface support start line --- sqle/driver/plugin_adapter_v2.go | 3 +- sqle/driver/v2/driver_interface.go | 3 + sqle/driver/v2/proto/driver_v2.pb.go | 247 ++++++++++++++------------- sqle/driver/v2/proto/driver_v2.proto | 1 + 4 files changed, 134 insertions(+), 120 deletions(-) diff --git a/sqle/driver/plugin_adapter_v2.go b/sqle/driver/plugin_adapter_v2.go index 702e5125e3..c8af4ffa50 100644 --- a/sqle/driver/plugin_adapter_v2.go +++ b/sqle/driver/plugin_adapter_v2.go @@ -3,9 +3,9 @@ package driver import ( "context" sqlDriver "database/sql/driver" + "errors" "fmt" "sync" - "errors" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" protoV2 "github.com/actiontech/sqle/sqle/driver/v2/proto" @@ -211,6 +211,7 @@ func (s *PluginImplV2) Parse(ctx context.Context, sqlText string) ([]driverV2.No Type: node.Type, Text: node.Text, Fingerprint: node.Fingerprint, + StartLine: node.StartLine, } } return nodes, nil diff --git a/sqle/driver/v2/driver_interface.go b/sqle/driver/v2/driver_interface.go index 0b6279f475..178e665de0 100644 --- a/sqle/driver/v2/driver_interface.go +++ b/sqle/driver/v2/driver_interface.go @@ -99,6 +99,9 @@ type Node struct { // Fingerprint is fingerprint of Node's raw SQL. Fingerprint string + + // StartLine is the starting row number of the Node's raw SQL. + StartLine uint64 } type RuleLevel string diff --git a/sqle/driver/v2/proto/driver_v2.pb.go b/sqle/driver/v2/proto/driver_v2.pb.go index 42371b6609..c2fac7d644 100644 --- a/sqle/driver/v2/proto/driver_v2.pb.go +++ b/sqle/driver/v2/proto/driver_v2.pb.go @@ -504,6 +504,7 @@ type Node struct { Text string `protobuf:"bytes,1,opt,name=text" json:"text,omitempty"` Type string `protobuf:"bytes,2,opt,name=type" json:"type,omitempty"` Fingerprint string `protobuf:"bytes,3,opt,name=fingerprint" json:"fingerprint,omitempty"` + StartLine uint64 `protobuf:"varint,4,opt,name=startLine" json:"startLine,omitempty"` } func (m *Node) Reset() { *m = Node{} } @@ -532,6 +533,13 @@ func (m *Node) GetFingerprint() string { return "" } +func (m *Node) GetStartLine() uint64 { + if m != nil { + return m.StartLine + } + return 0 +} + type ParseResponse struct { Nodes []*Node `protobuf:"bytes,1,rep,name=nodes" json:"nodes,omitempty"` } @@ -2121,123 +2129,124 @@ var _Driver_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("driver_v2.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1875 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x18, 0xdb, 0x72, 0x14, 0xc7, - 0x35, 0x7b, 0x95, 0xf6, 0xec, 0x45, 0xeb, 0x96, 0x04, 0xcb, 0x02, 0xb2, 0xd2, 0x80, 0xac, 0xc2, - 0x44, 0x60, 0x91, 0xe0, 0xc2, 0x24, 0x29, 0x88, 0xa4, 0x80, 0x8c, 0x51, 0x44, 0x8b, 0x90, 0x2a, - 0x57, 0xb9, 0xec, 0xd6, 0x4e, 0x6b, 0x3d, 0xc5, 0xec, 0xf4, 0x6a, 0xba, 0x17, 0x89, 0x2f, 0xc8, - 0x07, 0xe4, 0x21, 0xbf, 0x91, 0x17, 0x3f, 0xe6, 0x47, 0xf2, 0x35, 0xa9, 0xbe, 0xcc, 0x4c, 0xcf, - 0x65, 0x6d, 0xb3, 0x4f, 0xd3, 0x7d, 0xee, 0x7d, 0xce, 0xe9, 0xd3, 0xe7, 0x0c, 0xac, 0x78, 0x91, - 0xff, 0x9e, 0x45, 0xdf, 0xbf, 0xdf, 0xdd, 0x99, 0x46, 0x5c, 0x72, 0xb4, 0xa4, 0x3f, 0x6f, 0x77, - 0xf1, 0x12, 0x34, 0x0e, 0x26, 0x53, 0xf9, 0x01, 0x5f, 0x83, 0xa5, 0x13, 0x26, 0x84, 0xcf, 0x43, - 0xd4, 0x83, 0xaa, 0xef, 0x0d, 0x2a, 0x9b, 0x95, 0xed, 0x16, 0xa9, 0xfa, 0x1e, 0xfe, 0x07, 0x34, - 0x8e, 0x69, 0x44, 0x27, 0xa8, 0x0f, 0xb5, 0x77, 0xec, 0x83, 0xc5, 0xa8, 0x25, 0x5a, 0x83, 0xc6, - 0x7b, 0x1a, 0xcc, 0xd8, 0xa0, 0xaa, 0x61, 0x66, 0x83, 0x10, 0xd4, 0x3d, 0x26, 0x46, 0x83, 0x9a, - 0x06, 0xea, 0xb5, 0x82, 0xc9, 0x0f, 0x53, 0x36, 0xa8, 0x1b, 0x98, 0x5a, 0xe3, 0x9f, 0x2a, 0x50, - 0xdb, 0x3f, 0x39, 0x52, 0xb8, 0x1f, 0xb9, 0x90, 0x56, 0xb0, 0x5e, 0x2b, 0xd8, 0x94, 0x47, 0xd2, - 0x0a, 0xd6, 0x6b, 0x05, 0x9b, 0x09, 0x16, 0xc5, 0x72, 0xd5, 0x1a, 0x0d, 0x61, 0x79, 0x4a, 0x85, - 0xb8, 0xe0, 0x91, 0x67, 0x65, 0x27, 0x7b, 0x85, 0xf3, 0xa8, 0xa4, 0xa7, 0x54, 0xb0, 0x41, 0xc3, - 0xe0, 0xe2, 0x3d, 0xfa, 0x0a, 0xfa, 0xd4, 0xf3, 0x7c, 0xe9, 0xf3, 0x90, 0x06, 0xfa, 0x78, 0x62, - 0xd0, 0xdc, 0xac, 0x6d, 0xb7, 0x77, 0x7b, 0x3b, 0xd6, 0x39, 0x3b, 0x1a, 0x4c, 0x0a, 0x74, 0xf8, - 0x7f, 0x15, 0xa8, 0x93, 0x59, 0xa0, 0x0f, 0x1a, 0xd2, 0x09, 0x8b, 0x0d, 0x57, 0xeb, 0xe4, 0xf0, - 0x55, 0xe7, 0xf0, 0x6b, 0xd0, 0x08, 0xd8, 0x7b, 0x16, 0x58, 0xcb, 0xcd, 0x46, 0x99, 0x37, 0xa2, - 0x92, 0x8d, 0x79, 0xf4, 0x21, 0x36, 0x3d, 0xde, 0xa3, 0x2d, 0x68, 0x4e, 0x8d, 0x51, 0x8d, 0x52, - 0xa3, 0x2c, 0x16, 0x6d, 0x00, 0xd0, 0x30, 0xe4, 0x92, 0x2a, 0x03, 0x07, 0x4d, 0x2d, 0xc5, 0x81, - 0xa0, 0x07, 0xd0, 0x7a, 0x17, 0xf2, 0x8b, 0x80, 0x79, 0x63, 0x36, 0x58, 0xda, 0xac, 0x6c, 0xb7, - 0x77, 0x51, 0x22, 0xea, 0x65, 0x8c, 0x21, 0x29, 0x11, 0xbe, 0x03, 0xad, 0x04, 0x8e, 0x06, 0xb0, - 0x34, 0xe2, 0xa1, 0x64, 0x61, 0x1c, 0x9c, 0x78, 0x8b, 0x7f, 0xaa, 0x42, 0xf7, 0x15, 0x93, 0x54, - 0x10, 0x26, 0xa6, 0x3c, 0x14, 0x4c, 0x99, 0x32, 0x0d, 0x66, 0x63, 0x3f, 0x3c, 0x4a, 0x5d, 0xe2, - 0x40, 0xd0, 0x03, 0x58, 0x8d, 0xbd, 0xbf, 0xcf, 0xce, 0xe8, 0x2c, 0x90, 0xc7, 0x71, 0x80, 0x6b, - 0xa4, 0x0c, 0x85, 0xbe, 0x86, 0x41, 0x0c, 0x7e, 0x96, 0x8f, 0x55, 0xad, 0xd4, 0x2d, 0x73, 0xe9, - 0xd1, 0x2d, 0x68, 0x44, 0xb3, 0x80, 0x89, 0x41, 0x5d, 0x33, 0x76, 0x13, 0x46, 0x15, 0x48, 0x62, - 0x70, 0xe8, 0x15, 0xac, 0xb3, 0x90, 0x9e, 0x06, 0xcc, 0xfb, 0xdb, 0xd4, 0x70, 0xbf, 0xe2, 0xde, - 0x2c, 0x60, 0x3a, 0x08, 0xbd, 0xdd, 0xab, 0x09, 0x53, 0x16, 0x4d, 0xca, 0xb9, 0x54, 0x2a, 0x04, - 0x7c, 0xcc, 0x75, 0x58, 0x3a, 0x44, 0xaf, 0x31, 0x81, 0xf6, 0x61, 0xe8, 0x4b, 0xc2, 0xce, 0x67, - 0x4c, 0x48, 0xb4, 0x01, 0x35, 0x4f, 0x84, 0xda, 0x5b, 0xed, 0xdd, 0x4e, 0x22, 0x7f, 0xff, 0xe4, - 0x88, 0x28, 0x44, 0x6a, 0x76, 0x75, 0xbe, 0xd9, 0xf8, 0x2b, 0xe8, 0x18, 0x99, 0x36, 0x12, 0x77, - 0x61, 0x49, 0x98, 0xbb, 0x6c, 0x05, 0xf7, 0x13, 0x36, 0x7b, 0xc7, 0x49, 0x4c, 0xa0, 0x78, 0xf7, - 0x02, 0x2e, 0x58, 0x6c, 0xd0, 0xc7, 0xf0, 0x3e, 0x05, 0xf4, 0xd2, 0x0f, 0x82, 0xe3, 0x88, 0x8f, - 0x98, 0x10, 0x8b, 0x48, 0xf8, 0x2d, 0xb4, 0x8e, 0x69, 0x24, 0x98, 0x77, 0xf2, 0xfa, 0x1b, 0x75, - 0x4b, 0xce, 0x67, 0x2c, 0x8a, 0x0b, 0x8c, 0xd9, 0xe0, 0x1f, 0xa0, 0xa3, 0x49, 0x16, 0x10, 0x8f, - 0x6e, 0x43, 0x4d, 0x9c, 0x07, 0x3a, 0xc5, 0xdc, 0xbc, 0x4f, 0x54, 0x12, 0x85, 0xc6, 0xc7, 0x50, - 0x3f, 0xe2, 0x9e, 0x0e, 0x97, 0x64, 0x97, 0x49, 0x19, 0x52, 0xeb, 0xa4, 0x6c, 0x55, 0xd3, 0xb2, - 0x85, 0x36, 0xa1, 0x7d, 0xe6, 0x87, 0x63, 0x16, 0x4d, 0x23, 0x3f, 0x94, 0xf6, 0x4e, 0xbb, 0x20, - 0xfc, 0x7b, 0xe8, 0x5a, 0x9b, 0x6d, 0x44, 0x6e, 0x41, 0x23, 0xe4, 0x1e, 0x13, 0x83, 0x4a, 0x2e, - 0x8c, 0x4a, 0x31, 0x31, 0x38, 0xbc, 0x09, 0xcb, 0xcf, 0x66, 0x9e, 0x2f, 0xe7, 0xfb, 0x82, 0x42, - 0x47, 0x53, 0x2c, 0xe2, 0x8b, 0x3b, 0x50, 0x17, 0xe7, 0x41, 0x9c, 0x48, 0x9f, 0x24, 0x84, 0xb1, - 0x4a, 0xa2, 0xd1, 0xf8, 0x5b, 0x68, 0x5b, 0x15, 0x62, 0x16, 0x48, 0x55, 0x00, 0x26, 0x4c, 0x08, - 0x3a, 0x8e, 0x6f, 0x74, 0xbc, 0x4d, 0x6b, 0x5a, 0xd5, 0xad, 0x69, 0xd7, 0xa1, 0xa5, 0x72, 0xf2, - 0x7b, 0x5d, 0x16, 0x8d, 0x67, 0x96, 0x15, 0x40, 0x55, 0x00, 0xfc, 0xe7, 0xc4, 0x7c, 0x25, 0x5b, - 0xa0, 0x1d, 0x58, 0x8a, 0xcc, 0xd2, 0xfa, 0x65, 0x2d, 0x6b, 0x95, 0xa1, 0x23, 0x31, 0x11, 0xfe, - 0x1a, 0xba, 0x31, 0xdc, 0xb8, 0xf5, 0x31, 0x74, 0xa8, 0x23, 0xd0, 0x4a, 0x59, 0x2f, 0x93, 0x22, - 0x48, 0x86, 0x14, 0x7f, 0x06, 0x2b, 0x47, 0x8c, 0x79, 0x84, 0x07, 0xc1, 0x29, 0x1d, 0xbd, 0x9b, - 0xef, 0x73, 0x0e, 0xeb, 0xcf, 0x59, 0xe8, 0xd0, 0x2d, 0xe2, 0xfc, 0xbb, 0x6e, 0x22, 0x0e, 0xd2, - 0xe8, 0x67, 0x2d, 0x30, 0xe9, 0xf8, 0x27, 0x68, 0xff, 0xa2, 0x55, 0x6e, 0x5c, 0xaa, 0x99, 0xb8, - 0xe0, 0xa7, 0x70, 0x25, 0x6f, 0xaf, 0xf5, 0xd6, 0x96, 0x31, 0xc2, 0x18, 0x9b, 0xba, 0xba, 0x60, - 0xc0, 0x63, 0x68, 0x1f, 0xfb, 0xe1, 0x78, 0x91, 0xfb, 0xfc, 0x29, 0x2c, 0x1d, 0x5c, 0xb2, 0xd1, - 0x7c, 0x6f, 0x7e, 0x07, 0x6d, 0x45, 0xb0, 0x88, 0x0f, 0xb1, 0xeb, 0xc3, 0x94, 0xce, 0xea, 0x33, - 0xa6, 0xff, 0xa7, 0x02, 0x60, 0xe4, 0xeb, 0xec, 0xc5, 0xd0, 0x09, 0xa8, 0x90, 0x87, 0xa1, 0x60, - 0x91, 0x3c, 0x34, 0x3d, 0x4d, 0x8d, 0x64, 0x60, 0xe8, 0x1e, 0x7c, 0xe2, 0xee, 0x0f, 0xa2, 0x88, - 0x47, 0xd6, 0xa7, 0x45, 0x84, 0x92, 0x18, 0xf1, 0x0b, 0xf1, 0xec, 0xec, 0x8c, 0x8d, 0x24, 0xf3, - 0x74, 0x8a, 0xd7, 0x48, 0x06, 0xa6, 0x24, 0xba, 0x7b, 0x23, 0xd1, 0x3c, 0xf0, 0x45, 0x04, 0x7e, - 0x02, 0x1d, 0x6b, 0xb1, 0x89, 0xd2, 0xe7, 0xd0, 0x34, 0xf9, 0x6e, 0x3d, 0xb2, 0x9a, 0x39, 0xa9, - 0xbd, 0x12, 0x96, 0x04, 0x7f, 0x07, 0xad, 0x37, 0x97, 0x8b, 0x55, 0x46, 0xb7, 0x1a, 0x14, 0xbd, - 0x69, 0x8a, 0xc1, 0x13, 0x00, 0x25, 0xde, 0x5a, 0xf6, 0xbb, 0xfc, 0x75, 0x2d, 0x35, 0x2d, 0xb9, - 0xad, 0x9b, 0xb0, 0xfc, 0x5a, 0xc5, 0x7c, 0x7e, 0x32, 0x7c, 0x01, 0x2d, 0x4d, 0xb1, 0xc7, 0xc3, - 0x33, 0x74, 0x1b, 0xba, 0xd2, 0x9f, 0x30, 0x3e, 0x93, 0x27, 0x6c, 0xc4, 0x43, 0x13, 0xac, 0x2e, - 0xc9, 0x02, 0xf1, 0x3f, 0x2b, 0xd0, 0xd1, 0x3c, 0x8b, 0x1c, 0xfa, 0x96, 0x9b, 0x41, 0x69, 0x05, - 0x8c, 0xad, 0xd4, 0x29, 0x84, 0xb6, 0xa0, 0x3e, 0xe2, 0xe1, 0x99, 0x8e, 0xac, 0xfb, 0x68, 0x24, - 0x96, 0x12, 0x8d, 0xc7, 0x1e, 0x74, 0xad, 0x21, 0xc9, 0xf5, 0x6a, 0x8e, 0x78, 0x30, 0x9b, 0x84, - 0xd6, 0x3b, 0x85, 0x96, 0xcd, 0x60, 0xd1, 0xe7, 0x50, 0x57, 0x59, 0x60, 0x5d, 0x7f, 0x35, 0xab, - 0xc0, 0x3a, 0x91, 0x5f, 0x10, 0x4d, 0x84, 0xf7, 0xa0, 0x97, 0x85, 0xa3, 0x2f, 0xa0, 0xa9, 0xbb, - 0xec, 0x38, 0x08, 0xd7, 0xca, 0x04, 0xbc, 0x55, 0x14, 0xc4, 0x12, 0xe2, 0x6d, 0xe8, 0xe7, 0x71, - 0x69, 0xe7, 0x5e, 0x71, 0x3a, 0x77, 0x8c, 0xd5, 0xf5, 0x99, 0x06, 0xd4, 0x0f, 0xe7, 0x47, 0x6d, - 0x04, 0x3d, 0x4b, 0xb3, 0xd8, 0x33, 0xe4, 0xc4, 0xc0, 0x4d, 0xa0, 0x58, 0xab, 0xb9, 0xc8, 0x6f, - 0x61, 0x25, 0x51, 0x62, 0xfd, 0xbb, 0x07, 0xdd, 0x51, 0x40, 0x85, 0xf0, 0x6d, 0xa6, 0x59, 0x5d, - 0x37, 0xf3, 0x32, 0xf6, 0x5c, 0x22, 0x92, 0xe5, 0xc1, 0x4f, 0x61, 0xad, 0x8c, 0x0c, 0x6d, 0x43, - 0x5d, 0xb5, 0x8e, 0x85, 0xe2, 0xf8, 0x86, 0x9e, 0xce, 0x02, 0x1a, 0xed, 0x53, 0x49, 0x89, 0xa6, - 0xc0, 0xcf, 0x60, 0xf5, 0x39, 0x93, 0xfb, 0xb6, 0xcf, 0x5c, 0xa8, 0xeb, 0xd9, 0x80, 0xe5, 0x98, - 0xbf, 0x6c, 0x84, 0xc0, 0xcf, 0x61, 0x2d, 0xab, 0xc2, 0x7a, 0xe0, 0x3e, 0xb4, 0xe2, 0xfe, 0x36, - 0x8e, 0x7e, 0x9a, 0xc5, 0x31, 0x39, 0x49, 0x69, 0xf0, 0x43, 0x68, 0xbc, 0x51, 0x8d, 0x69, 0xe9, - 0xa0, 0x72, 0x05, 0x9a, 0x62, 0xf4, 0x23, 0x9b, 0x50, 0x5b, 0xed, 0xec, 0x0e, 0x8f, 0xf5, 0x01, - 0x35, 0x9f, 0x6a, 0xf0, 0x17, 0xab, 0x2e, 0x0d, 0xa9, 0xf8, 0x6d, 0x98, 0x7b, 0xae, 0x3b, 0x55, - 0xdb, 0xaa, 0x91, 0xf8, 0x85, 0x3e, 0xa6, 0xa3, 0xc8, 0x1e, 0xf3, 0x01, 0xb4, 0x64, 0x0c, 0xb4, - 0xba, 0x50, 0x56, 0x82, 0x26, 0x4f, 0x89, 0xf0, 0x7f, 0x2b, 0xd0, 0x4a, 0x10, 0xe8, 0x11, 0xb4, - 0xcd, 0x55, 0x13, 0x87, 0xe1, 0x19, 0x2f, 0x84, 0x74, 0x2f, 0xc5, 0x11, 0x97, 0x50, 0xf1, 0xf9, - 0xa1, 0xc7, 0x2e, 0x99, 0xe1, 0xab, 0xe6, 0xf8, 0x0e, 0x53, 0x1c, 0x71, 0x09, 0xd1, 0x16, 0xf4, - 0x46, 0x11, 0xa3, 0x92, 0x69, 0x13, 0x4e, 0x5e, 0x7f, 0x63, 0x1b, 0x9f, 0x1c, 0xd4, 0x7d, 0xb3, - 0xeb, 0xd9, 0x37, 0xfb, 0x4b, 0x68, 0x3b, 0x56, 0x7d, 0x44, 0x32, 0x7e, 0xa9, 0xa6, 0x89, 0xd4, - 0x92, 0x5f, 0xcf, 0xf8, 0x18, 0x56, 0x1c, 0xe0, 0x0b, 0x46, 0xbd, 0x5f, 0x3b, 0xcc, 0xaa, 0xce, - 0xc9, 0x95, 0xc7, 0x2f, 0x84, 0x2a, 0x14, 0xbe, 0x64, 0x13, 0x93, 0x94, 0x2d, 0x62, 0x36, 0x98, - 0x43, 0xdb, 0x21, 0x44, 0xbb, 0x6a, 0x96, 0xd4, 0x87, 0xb4, 0xb9, 0x3b, 0x28, 0xb3, 0x4f, 0x99, - 0x42, 0x62, 0x42, 0x74, 0x2f, 0x53, 0x2b, 0x4b, 0x19, 0x94, 0x01, 0xb6, 0x58, 0xde, 0x56, 0x4f, - 0xa9, 0x8c, 0xa8, 0x7a, 0x5c, 0xe7, 0xd7, 0xaf, 0x73, 0x18, 0x5a, 0x2a, 0x1d, 0x99, 0xbf, 0x46, - 0x7c, 0xb2, 0x60, 0x57, 0xf7, 0x99, 0x5b, 0xcb, 0xd6, 0x9d, 0x3a, 0x94, 0xda, 0x60, 0xaa, 0xd9, - 0x01, 0x5c, 0x2f, 0x55, 0x99, 0xbe, 0x1c, 0x3a, 0x97, 0x45, 0xe1, 0xe5, 0x30, 0xf7, 0xc5, 0x62, - 0xf1, 0x1d, 0xe8, 0x9a, 0xde, 0x41, 0x9d, 0x79, 0xfe, 0x01, 0x25, 0xdc, 0x38, 0x10, 0xd2, 0x9f, - 0x50, 0xa9, 0xd2, 0x2e, 0xe5, 0x58, 0xe4, 0x88, 0xdb, 0xee, 0x11, 0xaf, 0xa4, 0x8d, 0xb5, 0x6b, - 0x86, 0x39, 0xe3, 0xdf, 0xe1, 0xe6, 0x1c, 0xad, 0xf6, 0x94, 0x6b, 0xd0, 0x18, 0xf1, 0x99, 0xfd, - 0x93, 0x50, 0x23, 0x66, 0x83, 0x36, 0x00, 0x58, 0x14, 0xbd, 0xca, 0xf4, 0xb2, 0x0e, 0x04, 0xff, - 0x01, 0x56, 0x33, 0x33, 0x66, 0xfa, 0xb3, 0xc1, 0x61, 0xab, 0xe4, 0xd9, 0xee, 0xfe, 0xab, 0x02, - 0xbd, 0xc2, 0x34, 0xde, 0xcb, 0x36, 0xc6, 0xfd, 0xdf, 0xa0, 0x16, 0x34, 0xf4, 0xcb, 0xd8, 0xaf, - 0xa0, 0xb6, 0x6a, 0x5d, 0xf5, 0xcb, 0xd0, 0xaf, 0xa2, 0x3e, 0x74, 0xdc, 0xd2, 0xd4, 0xaf, 0xa1, - 0xab, 0xb0, 0x5a, 0x12, 0xc2, 0x7e, 0x1d, 0x5d, 0x83, 0xf5, 0xd2, 0x73, 0xf7, 0x1b, 0x68, 0x05, - 0xda, 0x8e, 0xed, 0xfd, 0xe6, 0xee, 0xbf, 0x97, 0xa1, 0xb9, 0xaf, 0x7f, 0xc5, 0xa1, 0xfb, 0xd0, - 0xd0, 0xbf, 0x4f, 0x50, 0x1a, 0x6c, 0xfd, 0x23, 0x6e, 0x98, 0x3a, 0x39, 0xfb, 0x7b, 0xe5, 0x21, - 0xd4, 0xd5, 0x90, 0x8f, 0xdc, 0x82, 0x94, 0x4c, 0x82, 0xc3, 0xf5, 0x1c, 0xd4, 0x32, 0xed, 0x40, - 0x43, 0x4f, 0xf7, 0x28, 0xc5, 0xbb, 0xd3, 0xfe, 0x30, 0xa7, 0x1c, 0xbd, 0xc8, 0x58, 0x8c, 0xae, - 0xa7, 0xbf, 0x8a, 0x0a, 0x73, 0xfe, 0xf0, 0x46, 0x39, 0xd2, 0x6a, 0x7e, 0xa4, 0x7f, 0x1a, 0x66, - 0x34, 0xbb, 0x63, 0xbc, 0x73, 0xcc, 0xec, 0xa4, 0xfc, 0x08, 0x1a, 0x7a, 0x6a, 0x43, 0x85, 0x29, - 0x2e, 0xcf, 0x97, 0x1d, 0x05, 0x5f, 0xe7, 0xa3, 0x8b, 0x36, 0x12, 0xca, 0xd2, 0xf9, 0x6d, 0xf8, - 0xe9, 0x5c, 0xbc, 0x15, 0x79, 0x0f, 0xea, 0x6a, 0x0e, 0x72, 0x3c, 0xee, 0x8c, 0x45, 0x05, 0xd7, - 0x3d, 0x84, 0xba, 0xea, 0x82, 0x1d, 0x6a, 0x67, 0xd0, 0x19, 0xae, 0xe7, 0x5b, 0xe5, 0xb8, 0xa5, - 0xae, 0xbe, 0xb9, 0x44, 0xce, 0xeb, 0x16, 0x37, 0xf3, 0xc3, 0xd5, 0x0c, 0x2c, 0x75, 0x8e, 0x4e, - 0x57, 0xc7, 0x39, 0x6e, 0x33, 0xec, 0x38, 0x27, 0xdb, 0x9a, 0xfe, 0x31, 0xc9, 0x6d, 0x74, 0x35, - 0xdf, 0x2e, 0xc5, 0xbc, 0x83, 0x22, 0xc2, 0x72, 0xbf, 0xd4, 0x97, 0x21, 0x69, 0x47, 0xd0, 0x0d, - 0xc7, 0x71, 0x85, 0x46, 0x68, 0x78, 0x73, 0x0e, 0x36, 0x23, 0x2c, 0x7d, 0xac, 0x33, 0xc2, 0xf2, - 0x4d, 0x47, 0x56, 0x58, 0xb1, 0x53, 0xf8, 0xa1, 0xf4, 0x52, 0xa2, 0x5b, 0xf9, 0x52, 0x5c, 0x52, - 0xe8, 0x87, 0xb7, 0x7f, 0x9e, 0xc8, 0x6a, 0x38, 0x9b, 0x73, 0xbb, 0xd1, 0x9d, 0x94, 0xfd, 0x67, - 0x6a, 0xed, 0x70, 0xeb, 0x97, 0xc8, 0x8c, 0x9e, 0xbf, 0x74, 0xbe, 0x85, 0x9d, 0xfb, 0x4f, 0x2c, - 0xed, 0x69, 0x53, 0x2f, 0x1e, 0xfe, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x9b, 0x00, 0x1c, 0xb8, - 0x17, 0x00, 0x00, + // 1893 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xeb, 0x72, 0x14, 0xb9, + 0x15, 0xce, 0x5c, 0xed, 0x39, 0x73, 0xf1, 0xac, 0x6c, 0xc3, 0x30, 0x80, 0xd7, 0x11, 0xe0, 0x75, + 0xb1, 0xc4, 0xb0, 0x26, 0x61, 0x8b, 0x25, 0x49, 0x41, 0x6c, 0x07, 0xbc, 0x80, 0x63, 0x64, 0x42, + 0xaa, 0xb6, 0x6a, 0x6b, 0x57, 0x9e, 0x96, 0x67, 0xbb, 0xe8, 0x69, 0x8d, 0x5b, 0x1a, 0x6c, 0x9e, + 0x20, 0x0f, 0x90, 0x1f, 0x79, 0x8d, 0xfc, 0xd9, 0x9f, 0x79, 0x91, 0x3c, 0x4d, 0x4a, 0x97, 0xee, + 0x56, 0x5f, 0x66, 0x2f, 0xf3, 0xab, 0xa5, 0x73, 0xd3, 0xa7, 0x73, 0x8e, 0x8e, 0x8e, 0x1a, 0x56, + 0xbc, 0xc8, 0xff, 0xc0, 0xa2, 0xef, 0x3e, 0xec, 0xee, 0x4c, 0x23, 0x2e, 0x39, 0x5a, 0xd2, 0x9f, + 0x77, 0xbb, 0x78, 0x09, 0x1a, 0x07, 0x93, 0xa9, 0xfc, 0x88, 0xaf, 0xc1, 0xd2, 0x09, 0x13, 0xc2, + 0xe7, 0x21, 0xea, 0x41, 0xd5, 0xf7, 0x06, 0x95, 0xcd, 0xca, 0x76, 0x8b, 0x54, 0x7d, 0x0f, 0xff, + 0x03, 0x1a, 0xc7, 0x34, 0xa2, 0x13, 0xd4, 0x87, 0xda, 0x7b, 0xf6, 0xd1, 0x72, 0xd4, 0x10, 0xad, + 0x41, 0xe3, 0x03, 0x0d, 0x66, 0x6c, 0x50, 0xd5, 0x34, 0x33, 0x41, 0x08, 0xea, 0x1e, 0x13, 0xa3, + 0x41, 0x4d, 0x13, 0xf5, 0x58, 0xd1, 0xe4, 0xc7, 0x29, 0x1b, 0xd4, 0x0d, 0x4d, 0x8d, 0xf1, 0x8f, + 0x15, 0xa8, 0xed, 0x9f, 0x1c, 0x29, 0xde, 0x0f, 0x5c, 0x48, 0x6b, 0x58, 0x8f, 0x15, 0x6d, 0xca, + 0x23, 0x69, 0x0d, 0xeb, 0xb1, 0xa2, 0xcd, 0x04, 0x8b, 0x62, 0xbb, 0x6a, 0x8c, 0x86, 0xb0, 0x3c, + 0xa5, 0x42, 0x5c, 0xf0, 0xc8, 0xb3, 0xb6, 0x93, 0xb9, 0xe2, 0x79, 0x54, 0xd2, 0x53, 0x2a, 0xd8, + 0xa0, 0x61, 0x78, 0xf1, 0x1c, 0x7d, 0x05, 0x7d, 0xea, 0x79, 0xbe, 0xf4, 0x79, 0x48, 0x03, 0xbd, + 0x3d, 0x31, 0x68, 0x6e, 0xd6, 0xb6, 0xdb, 0xbb, 0xbd, 0x1d, 0xeb, 0x9c, 0x1d, 0x4d, 0x26, 0x05, + 0x39, 0xfc, 0xbf, 0x0a, 0xd4, 0xc9, 0x2c, 0xd0, 0x1b, 0x0d, 0xe9, 0x84, 0xc5, 0xc0, 0xd5, 0x38, + 0xd9, 0x7c, 0xd5, 0xd9, 0xfc, 0x1a, 0x34, 0x02, 0xf6, 0x81, 0x05, 0x16, 0xb9, 0x99, 0x28, 0x78, + 0x23, 0x2a, 0xd9, 0x98, 0x47, 0x1f, 0x63, 0xe8, 0xf1, 0x1c, 0x6d, 0x41, 0x73, 0x6a, 0x40, 0x35, + 0x4a, 0x41, 0x59, 0x2e, 0xda, 0x00, 0xa0, 0x61, 0xc8, 0x25, 0x55, 0x00, 0x07, 0x4d, 0x6d, 0xc5, + 0xa1, 0xa0, 0x07, 0xd0, 0x7a, 0x1f, 0xf2, 0x8b, 0x80, 0x79, 0x63, 0x36, 0x58, 0xda, 0xac, 0x6c, + 0xb7, 0x77, 0x51, 0x62, 0xea, 0x65, 0xcc, 0x21, 0xa9, 0x10, 0xbe, 0x03, 0xad, 0x84, 0x8e, 0x06, + 0xb0, 0x34, 0xe2, 0xa1, 0x64, 0x61, 0x1c, 0x9c, 0x78, 0x8a, 0x7f, 0xac, 0x42, 0xf7, 0x35, 0x93, + 0x54, 0x10, 0x26, 0xa6, 0x3c, 0x14, 0x4c, 0x41, 0x99, 0x06, 0xb3, 0xb1, 0x1f, 0x1e, 0xa5, 0x2e, + 0x71, 0x28, 0xe8, 0x01, 0xac, 0xc6, 0xde, 0xdf, 0x67, 0x67, 0x74, 0x16, 0xc8, 0xe3, 0x38, 0xc0, + 0x35, 0x52, 0xc6, 0x42, 0x5f, 0xc3, 0x20, 0x26, 0x3f, 0xcb, 0xc7, 0xaa, 0x56, 0xea, 0x96, 0xb9, + 0xf2, 0xe8, 0x16, 0x34, 0xa2, 0x59, 0xc0, 0xc4, 0xa0, 0xae, 0x15, 0xbb, 0x89, 0xa2, 0x0a, 0x24, + 0x31, 0x3c, 0xf4, 0x1a, 0xd6, 0x59, 0x48, 0x4f, 0x03, 0xe6, 0xfd, 0x6d, 0x6a, 0xb4, 0x5f, 0x73, + 0x6f, 0x16, 0x30, 0x1d, 0x84, 0xde, 0xee, 0xd5, 0x44, 0x29, 0xcb, 0x26, 0xe5, 0x5a, 0x2a, 0x15, + 0x02, 0x3e, 0xe6, 0x3a, 0x2c, 0x1d, 0xa2, 0xc7, 0x98, 0x40, 0xfb, 0x30, 0xf4, 0x25, 0x61, 0xe7, + 0x33, 0x26, 0x24, 0xda, 0x80, 0x9a, 0x27, 0x42, 0xed, 0xad, 0xf6, 0x6e, 0x27, 0xb1, 0xbf, 0x7f, + 0x72, 0x44, 0x14, 0x23, 0x85, 0x5d, 0x9d, 0x0f, 0x1b, 0x7f, 0x05, 0x1d, 0x63, 0xd3, 0x46, 0xe2, + 0x2e, 0x2c, 0x09, 0x73, 0x96, 0xad, 0xe1, 0x7e, 0xa2, 0x66, 0xcf, 0x38, 0x89, 0x05, 0x94, 0xee, + 0x5e, 0xc0, 0x05, 0x8b, 0x01, 0xfd, 0x1a, 0xdd, 0xa7, 0x80, 0x5e, 0xfa, 0x41, 0x70, 0x1c, 0xf1, + 0x11, 0x13, 0x62, 0x11, 0x0b, 0xbf, 0x85, 0xd6, 0x31, 0x8d, 0x04, 0xf3, 0x4e, 0xde, 0xbc, 0x52, + 0xa7, 0xe4, 0x7c, 0xc6, 0xa2, 0xb8, 0xc0, 0x98, 0x09, 0xfe, 0x1e, 0x3a, 0x5a, 0x64, 0x01, 0xf3, + 0xe8, 0x36, 0xd4, 0xc4, 0x79, 0xa0, 0x53, 0xcc, 0xcd, 0xfb, 0x64, 0x49, 0xa2, 0xd8, 0x38, 0x84, + 0xfa, 0x11, 0xf7, 0x74, 0xb8, 0x24, 0xbb, 0x4c, 0xca, 0x90, 0x1a, 0x27, 0x65, 0xab, 0x9a, 0x96, + 0x2d, 0xb4, 0x09, 0xed, 0x33, 0x3f, 0x1c, 0xb3, 0x68, 0x1a, 0xf9, 0xa1, 0xb4, 0x67, 0xda, 0x25, + 0xa1, 0x1b, 0xd0, 0x12, 0x92, 0x46, 0xf2, 0x95, 0x1f, 0x9a, 0x8a, 0x57, 0x27, 0x29, 0x01, 0xff, + 0x1e, 0xba, 0x76, 0x47, 0x36, 0x5e, 0xb7, 0xa0, 0x11, 0x72, 0x8f, 0x89, 0x41, 0x25, 0x17, 0x64, + 0x05, 0x8b, 0x18, 0x1e, 0xde, 0x84, 0xe5, 0x67, 0x33, 0xcf, 0x97, 0xf3, 0x3d, 0x45, 0xa1, 0xa3, + 0x25, 0x16, 0xf1, 0xd4, 0x1d, 0xa8, 0x8b, 0xf3, 0x20, 0x4e, 0xb3, 0x4f, 0x12, 0xc1, 0x78, 0x49, + 0xa2, 0xd9, 0xf8, 0x1b, 0x68, 0xdb, 0x25, 0xc4, 0x2c, 0x90, 0xaa, 0x3c, 0x4c, 0x98, 0x10, 0x74, + 0x1c, 0x9f, 0xf7, 0x78, 0x9a, 0x56, 0xbc, 0xaa, 0x5b, 0xf1, 0xae, 0x43, 0x4b, 0x65, 0xec, 0x77, + 0xba, 0x68, 0x1a, 0xbf, 0x2d, 0x2b, 0x82, 0xaa, 0x0f, 0xf8, 0xcf, 0x09, 0x7c, 0x65, 0x5b, 0xa0, + 0x1d, 0x58, 0x8a, 0xcc, 0xd0, 0xfa, 0x65, 0x2d, 0x8b, 0xca, 0xc8, 0x91, 0x58, 0x08, 0x7f, 0x0d, + 0xdd, 0x98, 0x6e, 0xdc, 0xfa, 0x18, 0x3a, 0xd4, 0x31, 0x68, 0xad, 0xac, 0x97, 0x59, 0x11, 0x24, + 0x23, 0x8a, 0x3f, 0x83, 0x95, 0x23, 0xc6, 0x3c, 0xc2, 0x83, 0xe0, 0x94, 0x8e, 0xde, 0xcf, 0xf7, + 0x39, 0x87, 0xf5, 0xe7, 0x2c, 0x74, 0xe4, 0x16, 0x71, 0xfe, 0x5d, 0x37, 0x4d, 0x07, 0x69, 0xf4, + 0xb3, 0x08, 0x4c, 0xb2, 0xfe, 0x09, 0xda, 0x3f, 0x8b, 0xca, 0x8d, 0x4b, 0x35, 0x13, 0x17, 0xfc, + 0x14, 0xae, 0xe4, 0xf1, 0x5a, 0x6f, 0x6d, 0x19, 0x10, 0x06, 0x6c, 0xea, 0xea, 0x02, 0x80, 0xc7, + 0xd0, 0x3e, 0xf6, 0xc3, 0xf1, 0x22, 0xa7, 0xfd, 0x53, 0x58, 0x3a, 0xb8, 0x64, 0xa3, 0xf9, 0xde, + 0xfc, 0x16, 0xda, 0x4a, 0x60, 0x11, 0x1f, 0x62, 0xd7, 0x87, 0xa9, 0x9c, 0x5d, 0xcf, 0x40, 0xff, + 0x4f, 0x05, 0xc0, 0xd8, 0xd7, 0xd9, 0x8b, 0xa1, 0x13, 0x50, 0x21, 0x0f, 0x43, 0xc1, 0x22, 0x79, + 0x68, 0x3a, 0x9e, 0x1a, 0xc9, 0xd0, 0xd0, 0x3d, 0xf8, 0xc4, 0x9d, 0x1f, 0x44, 0x11, 0x8f, 0xac, + 0x4f, 0x8b, 0x0c, 0x65, 0x31, 0xe2, 0x17, 0xe2, 0xd9, 0xd9, 0x19, 0x1b, 0x49, 0xe6, 0xe9, 0x14, + 0xaf, 0x91, 0x0c, 0x4d, 0x59, 0x74, 0xe7, 0xc6, 0xa2, 0xb9, 0xfe, 0x8b, 0x0c, 0xfc, 0x04, 0x3a, + 0x16, 0xb1, 0x89, 0xd2, 0xe7, 0xd0, 0x34, 0xf9, 0x6e, 0x3d, 0xb2, 0x9a, 0xd9, 0xa9, 0x3d, 0x12, + 0x56, 0x04, 0x7f, 0x0b, 0xad, 0xb7, 0x97, 0x8b, 0xd5, 0x4d, 0xb7, 0x1a, 0x14, 0xbd, 0x69, 0x8a, + 0xc1, 0x13, 0x00, 0x65, 0xde, 0x22, 0xfb, 0x5d, 0xfe, 0xb8, 0x96, 0x42, 0x4b, 0x4e, 0xeb, 0x26, + 0x2c, 0xbf, 0x51, 0x31, 0x9f, 0x9f, 0x0c, 0x5f, 0x40, 0x4b, 0x4b, 0xec, 0xf1, 0xf0, 0x0c, 0xdd, + 0x86, 0xae, 0xf4, 0x27, 0x8c, 0xcf, 0xe4, 0x09, 0x1b, 0xf1, 0xd0, 0x04, 0xab, 0x4b, 0xb2, 0x44, + 0xfc, 0xcf, 0x0a, 0x74, 0xb4, 0xce, 0x22, 0x9b, 0xbe, 0xe5, 0x66, 0x50, 0x5a, 0x01, 0x63, 0x94, + 0x3a, 0x85, 0xd0, 0x16, 0xd4, 0x47, 0x3c, 0x3c, 0xd3, 0x91, 0x75, 0xaf, 0x94, 0x04, 0x29, 0xd1, + 0x7c, 0xec, 0x41, 0xd7, 0x02, 0x49, 0x8e, 0x57, 0x73, 0xc4, 0x83, 0xd9, 0x24, 0xb4, 0xde, 0x29, + 0x34, 0x74, 0x86, 0x8b, 0x3e, 0x87, 0xba, 0xca, 0x02, 0xeb, 0xfa, 0xab, 0xd9, 0x05, 0xac, 0x13, + 0xf9, 0x05, 0xd1, 0x42, 0x78, 0x0f, 0x7a, 0x59, 0x3a, 0xfa, 0x02, 0x9a, 0xba, 0x07, 0x8f, 0x83, + 0x70, 0xad, 0xcc, 0xc0, 0x3b, 0x25, 0x41, 0xac, 0x20, 0xde, 0x86, 0x7e, 0x9e, 0x97, 0xf6, 0xf5, + 0x15, 0xa7, 0xaf, 0xc7, 0x58, 0x1d, 0x9f, 0x69, 0x40, 0xfd, 0x70, 0x7e, 0xd4, 0x46, 0xd0, 0xb3, + 0x32, 0x8b, 0x5d, 0x43, 0x4e, 0x0c, 0xdc, 0x04, 0x8a, 0x57, 0x35, 0x07, 0xf9, 0x1d, 0xac, 0x24, + 0x8b, 0x58, 0xff, 0xee, 0x41, 0x77, 0x14, 0x50, 0x21, 0x7c, 0x9b, 0x69, 0x76, 0xad, 0x9b, 0x79, + 0x1b, 0x7b, 0xae, 0x10, 0xc9, 0xea, 0xe0, 0xa7, 0xb0, 0x56, 0x26, 0x86, 0xb6, 0xa1, 0xae, 0x1a, + 0xcb, 0x42, 0x71, 0x7c, 0x4b, 0x4f, 0x67, 0x01, 0x8d, 0xf6, 0xa9, 0xa4, 0x44, 0x4b, 0xe0, 0x67, + 0xb0, 0xfa, 0x9c, 0xc9, 0x7d, 0xdb, 0x85, 0x2e, 0xd4, 0x13, 0x6d, 0xc0, 0x72, 0xac, 0x5f, 0xf6, + 0xc0, 0xc0, 0xcf, 0x61, 0x2d, 0xbb, 0x84, 0xf5, 0xc0, 0x7d, 0x68, 0xc5, 0xdd, 0x6f, 0x1c, 0xfd, + 0x34, 0x8b, 0x63, 0x71, 0x92, 0xca, 0xe0, 0x87, 0xd0, 0x78, 0xab, 0xda, 0xd6, 0xd2, 0x67, 0xcc, + 0x15, 0x68, 0x8a, 0xd1, 0x0f, 0x6c, 0x42, 0x6d, 0xb5, 0xb3, 0x33, 0x3c, 0xd6, 0x1b, 0xd4, 0x7a, + 0xaa, 0xfd, 0x5f, 0xac, 0xba, 0x34, 0xa4, 0xd2, 0xb7, 0x61, 0xee, 0xb9, 0xee, 0x54, 0x4d, 0xad, + 0x66, 0xe2, 0x17, 0x7a, 0x9b, 0xce, 0x42, 0x76, 0x9b, 0x0f, 0xa0, 0x25, 0x63, 0xa2, 0x5d, 0x0b, + 0x65, 0x2d, 0x68, 0xf1, 0x54, 0x08, 0xff, 0xb7, 0x02, 0xad, 0x84, 0x81, 0x1e, 0x41, 0xdb, 0x1c, + 0x35, 0x71, 0x18, 0x9e, 0xf1, 0x42, 0x48, 0xf7, 0x52, 0x1e, 0x71, 0x05, 0x95, 0x9e, 0x1f, 0x7a, + 0xec, 0x92, 0x19, 0xbd, 0x6a, 0x4e, 0xef, 0x30, 0xe5, 0x11, 0x57, 0x10, 0x6d, 0x41, 0x6f, 0x14, + 0x31, 0x2a, 0x99, 0x86, 0x70, 0xf2, 0xe6, 0x95, 0x6d, 0x7c, 0x72, 0x54, 0xf7, 0xce, 0xae, 0x67, + 0xef, 0xec, 0x2f, 0xa1, 0xed, 0xa0, 0xfa, 0x15, 0xc9, 0xf8, 0xa5, 0x7a, 0x6b, 0xa4, 0x48, 0x7e, + 0xb9, 0xe2, 0x63, 0x58, 0x71, 0x88, 0x2f, 0x18, 0xf5, 0x7e, 0xe9, 0x53, 0x57, 0x75, 0x4e, 0xae, + 0x3d, 0x7e, 0x21, 0x54, 0xa1, 0xf0, 0x25, 0x9b, 0x98, 0xa4, 0x6c, 0x11, 0x33, 0xc1, 0x1c, 0xda, + 0x8e, 0x20, 0xda, 0x55, 0x2f, 0x4d, 0xbd, 0x49, 0x9b, 0xbb, 0x83, 0x32, 0x7c, 0x0a, 0x0a, 0x89, + 0x05, 0xd1, 0xbd, 0x4c, 0xad, 0x2c, 0x55, 0x50, 0x00, 0x6c, 0xb1, 0xbc, 0xad, 0xae, 0x52, 0x19, + 0x51, 0x75, 0xb9, 0xce, 0xaf, 0x5f, 0xe7, 0x30, 0xb4, 0x52, 0x3a, 0x32, 0x7f, 0x8d, 0xf8, 0x64, + 0xc1, 0xae, 0xee, 0x33, 0xb7, 0x96, 0xad, 0x3b, 0x75, 0x28, 0xc5, 0x60, 0xaa, 0xd9, 0x01, 0x5c, + 0x2f, 0x5d, 0x32, 0xbd, 0x39, 0x74, 0x2e, 0x8b, 0xc2, 0xcd, 0x61, 0xce, 0x8b, 0xe5, 0xe2, 0x3b, + 0xd0, 0x35, 0xbd, 0x83, 0xda, 0xf3, 0xfc, 0x0d, 0x4a, 0xb8, 0x71, 0x20, 0xa4, 0x3f, 0xa1, 0x52, + 0xa5, 0x5d, 0xaa, 0xb1, 0xc8, 0x16, 0xb7, 0xdd, 0x2d, 0x5e, 0x49, 0x1b, 0x6b, 0x17, 0x86, 0xd9, + 0xe3, 0xdf, 0xe1, 0xe6, 0x9c, 0x55, 0xed, 0x2e, 0xd7, 0xa0, 0x31, 0xe2, 0x33, 0xfb, 0x9f, 0xa1, + 0x46, 0xcc, 0x04, 0x6d, 0x00, 0xb0, 0x28, 0x7a, 0x9d, 0xe9, 0x65, 0x1d, 0x0a, 0xfe, 0x03, 0xac, + 0x66, 0x5e, 0xa0, 0xe9, 0xaf, 0x08, 0x47, 0xad, 0x92, 0x57, 0xbb, 0xfb, 0xaf, 0x0a, 0xf4, 0x0a, + 0x6f, 0xf5, 0x5e, 0xb6, 0x31, 0xee, 0xff, 0x06, 0xb5, 0xa0, 0xa1, 0x6f, 0xc6, 0x7e, 0x05, 0xb5, + 0x55, 0xeb, 0xaa, 0x6f, 0x86, 0x7e, 0x15, 0xf5, 0xa1, 0xe3, 0x96, 0xa6, 0x7e, 0x0d, 0x5d, 0x85, + 0xd5, 0x92, 0x10, 0xf6, 0xeb, 0xe8, 0x1a, 0xac, 0x97, 0xee, 0xbb, 0xdf, 0x40, 0x2b, 0xd0, 0x76, + 0xb0, 0xf7, 0x9b, 0xbb, 0xff, 0x5e, 0x86, 0xe6, 0xbe, 0xfe, 0x51, 0x87, 0xee, 0x43, 0x43, 0xff, + 0x5c, 0x41, 0x69, 0xb0, 0xf5, 0x6f, 0xba, 0x61, 0xea, 0xe4, 0xec, 0xcf, 0x97, 0x87, 0x50, 0x3f, + 0x0c, 0x7d, 0x89, 0xdc, 0x82, 0x94, 0xbc, 0x04, 0x87, 0xeb, 0x39, 0xaa, 0x55, 0xda, 0x81, 0x86, + 0x7e, 0xfb, 0xa3, 0x94, 0xef, 0xfe, 0x0b, 0x18, 0xe6, 0x16, 0x47, 0x2f, 0x32, 0x88, 0xd1, 0xf5, + 0xf4, 0x47, 0x52, 0xe1, 0x2f, 0xc0, 0xf0, 0x46, 0x39, 0xd3, 0xae, 0xfc, 0x48, 0xff, 0x52, 0xcc, + 0xac, 0xec, 0x3e, 0xf2, 0x9d, 0x6d, 0x66, 0x5f, 0xca, 0x8f, 0xa0, 0xa1, 0x5f, 0x6d, 0xa8, 0xf0, + 0x8a, 0xcb, 0xeb, 0x65, 0x9f, 0x82, 0x6f, 0xf2, 0xd1, 0x45, 0x1b, 0x89, 0x64, 0xe9, 0xfb, 0x6d, + 0xf8, 0xe9, 0x5c, 0xbe, 0x35, 0x79, 0x0f, 0xea, 0xea, 0x1d, 0xe4, 0x78, 0xdc, 0x79, 0x16, 0x15, + 0x5c, 0xf7, 0x10, 0xea, 0xaa, 0x0b, 0x76, 0xa4, 0x9d, 0x87, 0xce, 0x70, 0x3d, 0xdf, 0x2a, 0xc7, + 0x2d, 0x75, 0xf5, 0xed, 0x25, 0x72, 0x6e, 0xb7, 0xb8, 0x99, 0x1f, 0xae, 0x66, 0x68, 0xa9, 0x73, + 0x74, 0xba, 0x3a, 0xce, 0x71, 0x9b, 0x61, 0xc7, 0x39, 0xd9, 0xd6, 0xf4, 0x8f, 0x49, 0x6e, 0xa3, + 0xab, 0xf9, 0x76, 0x29, 0xd6, 0x1d, 0x14, 0x19, 0x56, 0xfb, 0xa5, 0x3e, 0x0c, 0x49, 0x3b, 0x82, + 0x6e, 0x38, 0x8e, 0x2b, 0x34, 0x42, 0xc3, 0x9b, 0x73, 0xb8, 0x19, 0x63, 0xe9, 0x65, 0x9d, 0x31, + 0x96, 0x6f, 0x3a, 0xb2, 0xc6, 0x8a, 0x9d, 0xc2, 0xf7, 0xa5, 0x87, 0x12, 0xdd, 0xca, 0x97, 0xe2, + 0x92, 0x42, 0x3f, 0xbc, 0xfd, 0xd3, 0x42, 0x76, 0x85, 0xb3, 0x39, 0xa7, 0x1b, 0xdd, 0x49, 0xd5, + 0x7f, 0xa2, 0xd6, 0x0e, 0xb7, 0x7e, 0x4e, 0xcc, 0xac, 0xf3, 0x97, 0xce, 0x37, 0xb0, 0x73, 0xff, + 0x89, 0x95, 0x3d, 0x6d, 0xea, 0xc1, 0xc3, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0xb3, 0x57, 0xbd, + 0x7d, 0xd6, 0x17, 0x00, 0x00, } diff --git a/sqle/driver/v2/proto/driver_v2.proto b/sqle/driver/v2/proto/driver_v2.proto index 52d61966d4..579e00791d 100644 --- a/sqle/driver/v2/proto/driver_v2.proto +++ b/sqle/driver/v2/proto/driver_v2.proto @@ -123,6 +123,7 @@ message Node { string text = 1; string type = 2; string fingerprint = 3; + uint64 startLine = 4; } message ParseResponse { From c24fb93610ce51e783e1d21f127a79668abb32a0 Mon Sep 17 00:00:00 2001 From: luowei Date: Tue, 16 Jan 2024 20:04:41 +0800 Subject: [PATCH 02/10] return start line for display --- sqle/api/controller/v1/sql_audit_record.go | 1 + sqle/api/controller/v2/task.go | 1 + sqle/model/task.go | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sqle/api/controller/v1/sql_audit_record.go b/sqle/api/controller/v1/sql_audit_record.go index 86ab74e3ed..a6b10a9f00 100644 --- a/sqle/api/controller/v1/sql_audit_record.go +++ b/sqle/api/controller/v1/sql_audit_record.go @@ -188,6 +188,7 @@ func addSQLsFromFileToTasks(sqls getSQLFromFileResp, task *model.Task, plugin dr Number: num, Content: node.Text, SourceFile: sqlsFromOneFile.FilePath, + StartLine: node.StartLine, }, }) num++ diff --git a/sqle/api/controller/v2/task.go b/sqle/api/controller/v2/task.go index e6ba4233db..40397f818c 100644 --- a/sqle/api/controller/v2/task.go +++ b/sqle/api/controller/v2/task.go @@ -103,6 +103,7 @@ func GetTaskSQLs(c echo.Context) error { Description: taskSQL.Description, ExecSQL: taskSQL.ExecSQL, SQLSourceFile: taskSQL.SQLSourceFile.String, + SQLStartLine: taskSQL.SQLStartLine, AuditLevel: taskSQL.AuditLevel, AuditStatus: taskSQL.AuditStatus, ExecResult: taskSQL.ExecResult, diff --git a/sqle/model/task.go b/sqle/model/task.go index 84ca78bd59..9b5a2742e3 100644 --- a/sqle/model/task.go +++ b/sqle/model/task.go @@ -115,6 +115,7 @@ type BaseSQL struct { ExecResult string `json:"exec_result" gorm:"type:text"` Schema string `json:"schema"` SourceFile string `json:"source_file"` + StartLine uint64 `json:"start_line"` } func (s *BaseSQL) GetExecStatusDesc() string { @@ -439,6 +440,7 @@ type TaskSQLDetail struct { Description string `json:"description"` ExecSQL string `json:"exec_sql"` SQLSourceFile sql.NullString `json:"sql_source_file"` + SQLStartLine uint64 `json:"sql_start_line"` AuditResults AuditResults `json:"audit_results"` AuditLevel string `json:"audit_level"` AuditStatus string `json:"audit_status"` @@ -455,7 +457,7 @@ func (t *TaskSQLDetail) GetAuditResults() string { return t.AuditResults.String() } -var taskSQLsQueryTpl = `SELECT e_sql.number, e_sql.description, e_sql.content AS exec_sql, e_sql.source_file AS sql_source_file, r_sql.content AS rollback_sql, +var taskSQLsQueryTpl = `SELECT e_sql.number, e_sql.description, e_sql.content AS exec_sql, e_sql.source_file AS sql_source_file, e_sql.start_line AS sql_start_line, r_sql.content AS rollback_sql, e_sql.audit_results, e_sql.audit_level, e_sql.audit_status, e_sql.exec_result, e_sql.exec_status {{- template "body" . -}} From f2f8c18cfb811332d4b50b5b63840e3b4161113b Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 17 Jan 2024 18:08:39 +0800 Subject: [PATCH 03/10] return start line by mysql plugin --- sqle/driver/mysql/mysql.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqle/driver/mysql/mysql.go b/sqle/driver/mysql/mysql.go index e5ca9195cf..e7057d1ed8 100644 --- a/sqle/driver/mysql/mysql.go +++ b/sqle/driver/mysql/mysql.go @@ -261,6 +261,7 @@ func (i *MysqlDriverImpl) Parse(ctx context.Context, sqlText string) ([]driverV2 } n.Fingerprint = fingerprint n.Text = nodes[i].Text() + n.StartLine = uint64(nodes[i].StartLine()) switch nodes[i].(type) { case ast.DMLNode: n.Type = driverV2.SQLTypeDML From 5eff54c69e3d6e9519ae9a82d8bcc4c444ed52f2 Mon Sep 17 00:00:00 2001 From: luowei Date: Wed, 17 Jan 2024 18:21:06 +0800 Subject: [PATCH 04/10] for reusing --- sqle/cmd/scannerd/scanners/common/file.go | 6 +++--- sqle/cmd/scannerd/scanners/mybatis/mybatis.go | 6 ++++-- sqle/cmd/scannerd/scanners/scanner.go | 5 ----- sqle/cmd/scannerd/scanners/sql_file/sqlfile.go | 4 ++-- sqle/utils/const.go | 6 ++++++ 5 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 sqle/utils/const.go diff --git a/sqle/cmd/scannerd/scanners/common/file.go b/sqle/cmd/scannerd/scanners/common/file.go index e9a175d3dc..4d1c3723b4 100644 --- a/sqle/cmd/scannerd/scanners/common/file.go +++ b/sqle/cmd/scannerd/scanners/common/file.go @@ -3,6 +3,7 @@ package common import ( "context" "fmt" + "github.com/actiontech/sqle/sqle/utils" "io/ioutil" "os" "path" @@ -11,7 +12,6 @@ import ( mybatisParser "github.com/actiontech/mybatis-mapper-2-sql" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" - "github.com/actiontech/sqle/sqle/cmd/scannerd/scanners" ) func GetSQLFromPath(pathName string, skipErrorQuery, skipErrorFile bool, fileSuffix string) (allSQL []driverV2.Node, err error) { @@ -55,7 +55,7 @@ func GetSQLFromFile(file string, skipErrorQuery bool, fileSuffix string) (r []dr return nil, err } switch fileSuffix { - case scanners.MybatisFileSuffix: + case utils.MybatisFileSuffix: sqls, err := mybatisParser.ParseXMLQuery(content, skipErrorQuery) if err != nil { return nil, err @@ -67,7 +67,7 @@ func GetSQLFromFile(file string, skipErrorQuery bool, fileSuffix string) (r []dr } r = append(r, n...) } - case scanners.SQLFileSuffix: + case utils.SQLFileSuffix: n, err := Parse(context.TODO(), content) if err != nil { return nil, err diff --git a/sqle/cmd/scannerd/scanners/mybatis/mybatis.go b/sqle/cmd/scannerd/scanners/mybatis/mybatis.go index b4503fa6a1..d36f73abf8 100644 --- a/sqle/cmd/scannerd/scanners/mybatis/mybatis.go +++ b/sqle/cmd/scannerd/scanners/mybatis/mybatis.go @@ -3,10 +3,12 @@ package mybatis import ( "context" + "github.com/actiontech/sqle/sqle/utils" + "github.com/actiontech/sqle/sqle/cmd/scannerd/scanners" + "github.com/actiontech/sqle/sqle/cmd/scannerd/scanners/common" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" "github.com/actiontech/sqle/sqle/pkg/scanner" - "github.com/actiontech/sqle/sqle/cmd/scannerd/scanners/common" "github.com/sirupsen/logrus" ) @@ -49,7 +51,7 @@ func New(params *Params, l *logrus.Entry, c *scanner.Client) (*MyBatis, error) { } func (mb *MyBatis) Run(ctx context.Context) error { - sqls, err := common.GetSQLFromPath(mb.xmlDir, mb.skipErrorQuery, mb.skipErrorXml, scanners.MybatisFileSuffix) + sqls, err := common.GetSQLFromPath(mb.xmlDir, mb.skipErrorQuery, mb.skipErrorXml, utils.MybatisFileSuffix) if err != nil { return err } diff --git a/sqle/cmd/scannerd/scanners/scanner.go b/sqle/cmd/scannerd/scanners/scanner.go index 0a084cab66..34329967ab 100644 --- a/sqle/cmd/scannerd/scanners/scanner.go +++ b/sqle/cmd/scannerd/scanners/scanner.go @@ -5,11 +5,6 @@ import ( "time" ) -const ( - MybatisFileSuffix = "xml" - SQLFileSuffix = "sql" -) - type SQL struct { Fingerprint string RawText string diff --git a/sqle/cmd/scannerd/scanners/sql_file/sqlfile.go b/sqle/cmd/scannerd/scanners/sql_file/sqlfile.go index 32de285897..01fd5b986a 100644 --- a/sqle/cmd/scannerd/scanners/sql_file/sqlfile.go +++ b/sqle/cmd/scannerd/scanners/sql_file/sqlfile.go @@ -2,7 +2,7 @@ package sqlFile import ( "context" - + "github.com/actiontech/sqle/sqle/utils" "github.com/actiontech/sqle/sqle/cmd/scannerd/scanners" "github.com/actiontech/sqle/sqle/cmd/scannerd/scanners/common" @@ -48,7 +48,7 @@ func New(params *Params, l *logrus.Entry, c *scanner.Client) (*SQLFile, error) { } func (sf *SQLFile) Run(ctx context.Context) error { - sqls, err := common.GetSQLFromPath(sf.sqlDir, false, sf.skipErrorSqlFile, scanners.SQLFileSuffix) + sqls, err := common.GetSQLFromPath(sf.sqlDir, false, sf.skipErrorSqlFile, utils.SQLFileSuffix) if err != nil { return err } diff --git a/sqle/utils/const.go b/sqle/utils/const.go new file mode 100644 index 0000000000..9cd8383748 --- /dev/null +++ b/sqle/utils/const.go @@ -0,0 +1,6 @@ +package utils + +const ( + MybatisFileSuffix = "xml" + SQLFileSuffix = "sql" +) From f010d6f2343e963c9a2aebeb0805ef71b6979528 Mon Sep 17 00:00:00 2001 From: luowei Date: Thu, 18 Jan 2024 18:03:51 +0800 Subject: [PATCH 05/10] return start line when SQLs are from SQL file or XML file --- sqle/api/controller/v1/sql_audit_record.go | 147 ++++++++++++--------- sqle/api/controller/v1/task.go | 74 ++++------- 2 files changed, 109 insertions(+), 112 deletions(-) diff --git a/sqle/api/controller/v1/sql_audit_record.go b/sqle/api/controller/v1/sql_audit_record.go index a6b10a9f00..a67612725b 100644 --- a/sqle/api/controller/v1/sql_audit_record.go +++ b/sqle/api/controller/v1/sql_audit_record.go @@ -2,7 +2,6 @@ package v1 import ( "archive/zip" - "bytes" "context" "encoding/json" e "errors" @@ -101,7 +100,10 @@ func CreateSQLAuditRecord(c echo.Context) error { return controller.JSONBaseErrorReq(c, err) } if req.Sqls != "" { - sqls = getSQLFromFileResp{model.TaskSQLSourceFromFormData, []SQLsFromFile{{SQLs: req.Sqls}}} + sqls = getSQLFromFileResp{ + SourceType: model.TaskSQLSourceFromFormData, + SQLsFromFormData: req.Sqls, + } } else { sqls, err = getSQLFromFile(c) if err != nil { @@ -166,8 +168,10 @@ func CreateSQLAuditRecord(c echo.Context) error { } type getSQLFromFileResp struct { - SourceType string - SQLs []SQLsFromFile + SourceType string + SQLsFromFormData string + SQLsFromSQLFiles []SQLsFromFile + SQLsFromXMLs []SQLFromXML } type SQLsFromFile struct { @@ -175,24 +179,58 @@ type SQLsFromFile struct { SQLs string } +type SQLFromXML struct { + FilePath string + StartLine uint64 + SQL string +} + func addSQLsFromFileToTasks(sqls getSQLFromFileResp, task *model.Task, plugin driver.Plugin) error { var num uint = 1 - for _, sqlsFromOneFile := range sqls.SQLs { - nodes, err := plugin.Parse(context.TODO(), sqlsFromOneFile.SQLs) + + fileTask := func(sqlsText, filePath string, defaultStartLine uint64) error { + nodes, err := plugin.Parse(context.TODO(), sqlsText) if err != nil { return fmt.Errorf("parse sqls failed: %v", err) } for _, node := range nodes { + startLine := defaultStartLine + if startLine == 0 { + startLine = node.StartLine + } task.ExecuteSQLs = append(task.ExecuteSQLs, &model.ExecuteSQL{ BaseSQL: model.BaseSQL{ Number: num, Content: node.Text, - SourceFile: sqlsFromOneFile.FilePath, - StartLine: node.StartLine, + SourceFile: filePath, + StartLine: startLine, }, }) num++ } + return nil + } + + // 处理从页面输入的SQL + err := fileTask(sqls.SQLsFromFormData, "", 0) + if err != nil { + return fmt.Errorf("parse sqls failed: %v", err) + } + + for _, sqlsFromOneFile := range sqls.SQLsFromSQLFiles { + // SQL文件和Java文件里的SQL都在这里处理 + err := fileTask(sqlsFromOneFile.SQLs, sqlsFromOneFile.FilePath, 0) + if err != nil { + return fmt.Errorf("parse sqls failed: %v", err) + } + } + + for _, sqlsFromOneFile := range sqls.SQLsFromXMLs { + // 这里遍历的一个元素里只有一条SQL + err := fileTask(sqlsFromOneFile.SQL, sqlsFromOneFile.FilePath, sqlsFromOneFile.StartLine) + if err != nil { + return fmt.Errorf("parse sqls failed: %v", err) + } } return nil } @@ -267,31 +305,31 @@ func buildOfflineTaskForAudit(userId uint64, dbType string, sqls getSQLFromFileR return task, nil } -func getSqlsFromZip(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) { +func getSqlsFromZip(c echo.Context) (sqlsFromSQLFile []SQLsFromFile, sqlsFromXML []SQLFromXML, exist bool, err error) { file, err := c.FormFile(InputZipFileName) if err == http.ErrMissingFile { - return nil, false, nil + return nil, nil, false, nil } if err != nil { - return nil, false, err + return nil, nil, false, err } f, err := file.Open() if err != nil { - return nil, false, err + return nil, nil, false, err } defer f.Close() currentPos, err := f.Seek(0, io.SeekEnd) // get size of zip file if err != nil { - return nil, false, err + return nil, nil, false, err } size := currentPos + 1 if size > maxZipFileSize { - return nil, false, fmt.Errorf("file can't be bigger than %vM", maxZipFileSize/1024/1024) + return nil, nil, false, fmt.Errorf("file can't be bigger than %vM", maxZipFileSize/1024/1024) } r, err := zip.NewReader(f, size) if err != nil { - return nil, false, err + return nil, nil, false, err } var xmlContents []xmlParser.XmlFile @@ -306,11 +344,11 @@ func getSqlsFromZip(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) r, err := srcFile.Open() if err != nil { - return nil, false, fmt.Errorf("open src file failed: %v", err) + return nil, nil, false, fmt.Errorf("open src file failed: %v", err) } content, err := io.ReadAll(r) if err != nil { - return nil, false, fmt.Errorf("read src file failed: %v", err) + return nil, nil, false, fmt.Errorf("read src file failed: %v", err) } if strings.HasSuffix(srcFile.Name, ".xml") { @@ -319,7 +357,7 @@ func getSqlsFromZip(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) Content: string(content), }) } else if strings.HasSuffix(srcFile.Name, ".sql") { - sqls = append(sqls, SQLsFromFile{ + sqlsFromSQLFile = append(sqlsFromSQLFile, SQLsFromFile{ FilePath: srcFile.Name, SQLs: string(content), }) @@ -331,12 +369,12 @@ func getSqlsFromZip(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) { sqlsFromXmls, err := parseXMLsWithFilePath(xmlContents) if err != nil { - return nil, false, err + return nil, nil, false, err } - sqls = append(sqls, sqlsFromXmls...) + sqlsFromXML = append(sqlsFromXML, sqlsFromXmls...) } - return sqls, true, nil + return sqlsFromSQLFile, sqlsFromXML, true, nil } func getSQLsByFilePath(filePath string, stmtsInfo []xmlAst.StmtInfo) []string { @@ -350,50 +388,34 @@ func getSQLsByFilePath(filePath string, stmtsInfo []xmlAst.StmtInfo) []string { return sqls } -func parseXMLsWithFilePath(xmlContents []xmlParser.XmlFile) ([]SQLsFromFile, error) { +func parseXMLsWithFilePath(xmlContents []xmlParser.XmlFile) ([]SQLFromXML, error) { allStmtsFromXml, err := xmlParser.ParseXMLs(xmlContents, false) if err != nil { return nil, fmt.Errorf("parse sqls from xml failed: %v", err) } - sqls := []SQLsFromFile{} - for _, xmlContent := range xmlContents { - var sqlBuffer bytes.Buffer - ss := getSQLsByFilePath(xmlContent.FilePath, allStmtsFromXml) - if ss == nil { - continue - } - - for _, sql := range ss { - if sqlBuffer.String() != "" && !strings.HasSuffix(sqlBuffer.String(), ";") { - if _, err = sqlBuffer.WriteString(";"); err != nil { - return nil, fmt.Errorf("gather sqls from xml file failed: %v", err) - } - } - if _, err = sqlBuffer.WriteString(sql); err != nil { - return nil, fmt.Errorf("gather sqls from xml file failed: %v", err) - } - } - - sqls = append(sqls, SQLsFromFile{ - FilePath: xmlContent.FilePath, - SQLs: sqlBuffer.String(), + sqls := []SQLFromXML{} + for _, stmtInfo := range allStmtsFromXml { + sqls = append(sqls, SQLFromXML{ + FilePath: stmtInfo.FilePath, + StartLine: stmtInfo.StartLine, + SQL: stmtInfo.SQL, }) } return sqls, nil } -func getSqlsFromGit(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) { +func getSqlsFromGit(c echo.Context) (sqlsFromSQLFiles, sqlsFromJavaFiles []SQLsFromFile, sqlsFromXMLs []SQLFromXML, exist bool, err error) { // make a temp dir and clean up befor return dir, err := os.MkdirTemp("./", "git-repo-") if err != nil { - return nil, false, err + return nil, nil, nil, false, err } defer os.RemoveAll(dir) // read http url from form and check if it's a git url url := c.FormValue(GitHttpURL) if !utils.IsGitHttpURL(url) { - return nil, false, errors.New(errors.DataInvalid, fmt.Errorf("url is not a git url")) + return nil, nil, nil, false, errors.New(errors.DataInvalid, fmt.Errorf("url is not a git url")) } cloneOpts := &goGit.CloneOptions{ URL: url, @@ -410,7 +432,7 @@ func getSqlsFromGit(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) // clone from git _, err = goGit.PlainCloneContext(c.Request().Context(), dir, false, cloneOpts) if err != nil { - return nil, false, err + return nil, nil, nil, false, err } l := log.NewEntry().WithField("function", "getSqlsFromGit") var xmlContents []xmlParser.XmlFile @@ -419,7 +441,6 @@ func getSqlsFromGit(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) gitPath := strings.TrimPrefix(path, strings.TrimPrefix(dir, "./")) if !info.IsDir() { var sqlBuffer strings.Builder - var sqlsFromOneFile string switch { case strings.HasSuffix(path, ".xml"): content, err := os.ReadFile(path) @@ -437,7 +458,10 @@ func getSqlsFromGit(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) l.Errorf("skip file [%v]. because read file failed: %v", path, err) return nil } - sqlsFromOneFile = string(content) + sqlsFromSQLFiles = append(sqlsFromSQLFiles, SQLsFromFile{ + FilePath: gitPath, + SQLs: string(content), + }) case strings.HasSuffix(path, ".java"): sqls, err := javaParser.GetSqlFromJavaFile(path) if err != nil { @@ -453,31 +477,26 @@ func getSqlsFromGit(c echo.Context) (sqls []SQLsFromFile, exist bool, err error) return fmt.Errorf("gather sqls from java file failed: %v", err) } } - sqlsFromOneFile = sqlBuffer.String() + sqlsFromJavaFiles = append(sqlsFromJavaFiles, SQLsFromFile{ + FilePath: gitPath, + SQLs: sqlBuffer.String(), + }) } - - sqls = append(sqls, SQLsFromFile{ - FilePath: gitPath, - SQLs: sqlsFromOneFile, - }) } return nil }) if err != nil { - return nil, false, err + return nil, nil, nil, false, err } // parse xml content // xml文件需要把所有文件内容同时解析,否则会无法解析跨namespace引用的SQL - { - sqlsFromXmls, err := parseXMLsWithFilePath(xmlContents) - if err != nil { - return nil, false, err - } - sqls = append(sqls, sqlsFromXmls...) + sqlsFromXMLs, err = parseXMLsWithFilePath(xmlContents) + if err != nil { + return nil, nil, nil, false, err } - return sqls, true, nil + return sqlsFromSQLFiles, sqlsFromJavaFiles, sqlsFromXMLs, true, nil } type UpdateSQLAuditRecordReqV1 struct { diff --git a/sqle/api/controller/v1/task.go b/sqle/api/controller/v1/task.go index 124e78ace2..30e46d8768 100644 --- a/sqle/api/controller/v1/task.go +++ b/sqle/api/controller/v1/task.go @@ -15,7 +15,6 @@ import ( "github.com/actiontech/sqle/sqle/api/controller" "github.com/actiontech/sqle/sqle/common" "github.com/actiontech/sqle/sqle/dms" - driverV2 "github.com/actiontech/sqle/sqle/driver/v2" "github.com/actiontech/sqle/sqle/errors" "github.com/actiontech/sqle/sqle/log" "github.com/actiontech/sqle/sqle/model" @@ -85,8 +84,8 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) { } if exist { return getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromSQLFile, - SQLs: []SQLsFromFile{{SQLs: sqlsFromSQLFile}}, + SourceType: model.TaskSQLSourceFromSQLFile, + SQLsFromSQLFiles: []SQLsFromFile{{SQLs: sqlsFromSQLFile}}, }, nil } @@ -96,37 +95,45 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) { return getSQLFromFileResp{}, err } if exist { - sql, err := mybatis_parser.ParseXML(data) + sqls, err := mybatis_parser.ParseXMLs([]mybatis_parser.XmlFile{{Content: data}}, true) if err != nil { return getSQLFromFileResp{}, errors.New(errors.ParseMyBatisXMLFileError, err) } + sqlsFromXMLs := make([]SQLFromXML, len(sqls)) + for i := range sqls { + sqlsFromXMLs[i] = SQLFromXML{ + SQL: sqls[i].SQL, + } + } return getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromMyBatisXMLFile, - SQLs: []SQLsFromFile{{SQLs: sql}}, + SourceType: model.TaskSQLSourceFromMyBatisXMLFile, + SQLsFromXMLs: sqlsFromXMLs, }, nil } // If mybatis xml file is not exist, read it from zip file. - sqlsFromZip, exist, err := getSqlsFromZip(c) + sqlsFromSQLFiles, sqlsFromXML, exist, err := getSqlsFromZip(c) if err != nil { return getSQLFromFileResp{}, err } if exist { return getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromZipFile, - SQLs: sqlsFromZip, + SourceType: model.TaskSQLSourceFromZipFile, + SQLsFromSQLFiles: sqlsFromSQLFiles, + SQLsFromXMLs: sqlsFromXML, }, nil } // If zip file is not exist, read it from git repository - sqlsFromGit, exist, err := getSqlsFromGit(c) + sqlsFromSQLFiles, sqlsFromJavaFiles, sqlsFromXMLs, exist, err := getSqlsFromGit(c) if err != nil { return getSQLFromFileResp{}, err } if exist { return getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromGitRepository, - SQLs: sqlsFromGit, + SourceType: model.TaskSQLSourceFromGitRepository, + SQLsFromSQLFiles: append(sqlsFromSQLFiles, sqlsFromJavaFiles...), + SQLsFromXMLs: sqlsFromXMLs, }, nil } return getSQLFromFileResp{}, errors.New(errors.DataInvalid, fmt.Errorf("input sql is empty")) @@ -161,13 +168,8 @@ func CreateAndAuditTask(c echo.Context) error { if req.Sql != "" { sqls = getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromFormData, - SQLs: []SQLsFromFile{ - { - FilePath: "", - SQLs: req.Sql, - }, - }, + SourceType: model.TaskSQLSourceFromFormData, + SQLsFromFormData: req.Sql, } } else { sqls, err = getSQLFromFile(c) @@ -763,13 +765,8 @@ func AuditTaskGroupV1(c echo.Context) error { var sqls getSQLFromFileResp if req.Sql != "" { sqls = getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromFormData, - SQLs: []SQLsFromFile{ - { - FilePath: "", - SQLs: req.Sql, - }, - }, + SourceType: model.TaskSQLSourceFromFormData, + SQLsFromFormData: req.Sql, } } else { sqls, err = getSQLFromFile(c) @@ -817,29 +814,10 @@ func AuditTaskGroupV1(c echo.Context) error { } defer plugin.Close(context.TODO()) - allNodes := make(map[string][]driverV2.Node, len(sqls.SQLs)) - for _, sqlsFromOneFile := range sqls.SQLs { - nodes, err := plugin.Parse(context.TODO(), sqlsFromOneFile.SQLs) - if err != nil { - return controller.JSONBaseErrorReq(c, err) - } - allNodes[sqlsFromOneFile.FilePath] = nodes - } - for _, task := range tasks { - task.SQLSource = sqls.SourceType - var num uint = 1 - for filePath, nodes := range allNodes { - for _, node := range nodes { - task.ExecuteSQLs = append(task.ExecuteSQLs, &model.ExecuteSQL{ - BaseSQL: model.BaseSQL{ - Number: num, - Content: node.Text, - SourceFile: filePath, - }, - }) - num += 1 - } + err := addSQLsFromFileToTasks(sqls, task, plugin) + if err != nil { + return controller.JSONBaseErrorReq(c, errors.New(errors.GenericError, fmt.Errorf("add sqls from file to task failed: %v", err))) } } } From 3afc01daa5ef49d12337f20669f3811b73e284c0 Mon Sep 17 00:00:00 2001 From: luowei Date: Thu, 18 Jan 2024 18:09:42 +0800 Subject: [PATCH 06/10] rename struct --- sqle/api/controller/v1/sql_audit_record.go | 14 +++++++------- sqle/api/controller/v1/task.go | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sqle/api/controller/v1/sql_audit_record.go b/sqle/api/controller/v1/sql_audit_record.go index a67612725b..cf0785487b 100644 --- a/sqle/api/controller/v1/sql_audit_record.go +++ b/sqle/api/controller/v1/sql_audit_record.go @@ -170,11 +170,11 @@ func CreateSQLAuditRecord(c echo.Context) error { type getSQLFromFileResp struct { SourceType string SQLsFromFormData string - SQLsFromSQLFiles []SQLsFromFile + SQLsFromSQLFiles []SQLsFromSQLFile SQLsFromXMLs []SQLFromXML } -type SQLsFromFile struct { +type SQLsFromSQLFile struct { FilePath string SQLs string } @@ -305,7 +305,7 @@ func buildOfflineTaskForAudit(userId uint64, dbType string, sqls getSQLFromFileR return task, nil } -func getSqlsFromZip(c echo.Context) (sqlsFromSQLFile []SQLsFromFile, sqlsFromXML []SQLFromXML, exist bool, err error) { +func getSqlsFromZip(c echo.Context) (sqlsFromSQLFile []SQLsFromSQLFile, sqlsFromXML []SQLFromXML, exist bool, err error) { file, err := c.FormFile(InputZipFileName) if err == http.ErrMissingFile { return nil, nil, false, nil @@ -357,7 +357,7 @@ func getSqlsFromZip(c echo.Context) (sqlsFromSQLFile []SQLsFromFile, sqlsFromXML Content: string(content), }) } else if strings.HasSuffix(srcFile.Name, ".sql") { - sqlsFromSQLFile = append(sqlsFromSQLFile, SQLsFromFile{ + sqlsFromSQLFile = append(sqlsFromSQLFile, SQLsFromSQLFile{ FilePath: srcFile.Name, SQLs: string(content), }) @@ -405,7 +405,7 @@ func parseXMLsWithFilePath(xmlContents []xmlParser.XmlFile) ([]SQLFromXML, error return sqls, nil } -func getSqlsFromGit(c echo.Context) (sqlsFromSQLFiles, sqlsFromJavaFiles []SQLsFromFile, sqlsFromXMLs []SQLFromXML, exist bool, err error) { +func getSqlsFromGit(c echo.Context) (sqlsFromSQLFiles, sqlsFromJavaFiles []SQLsFromSQLFile, sqlsFromXMLs []SQLFromXML, exist bool, err error) { // make a temp dir and clean up befor return dir, err := os.MkdirTemp("./", "git-repo-") if err != nil { @@ -458,7 +458,7 @@ func getSqlsFromGit(c echo.Context) (sqlsFromSQLFiles, sqlsFromJavaFiles []SQLsF l.Errorf("skip file [%v]. because read file failed: %v", path, err) return nil } - sqlsFromSQLFiles = append(sqlsFromSQLFiles, SQLsFromFile{ + sqlsFromSQLFiles = append(sqlsFromSQLFiles, SQLsFromSQLFile{ FilePath: gitPath, SQLs: string(content), }) @@ -477,7 +477,7 @@ func getSqlsFromGit(c echo.Context) (sqlsFromSQLFiles, sqlsFromJavaFiles []SQLsF return fmt.Errorf("gather sqls from java file failed: %v", err) } } - sqlsFromJavaFiles = append(sqlsFromJavaFiles, SQLsFromFile{ + sqlsFromJavaFiles = append(sqlsFromJavaFiles, SQLsFromSQLFile{ FilePath: gitPath, SQLs: sqlBuffer.String(), }) diff --git a/sqle/api/controller/v1/task.go b/sqle/api/controller/v1/task.go index 30e46d8768..319751c379 100644 --- a/sqle/api/controller/v1/task.go +++ b/sqle/api/controller/v1/task.go @@ -85,7 +85,7 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) { if exist { return getSQLFromFileResp{ SourceType: model.TaskSQLSourceFromSQLFile, - SQLsFromSQLFiles: []SQLsFromFile{{SQLs: sqlsFromSQLFile}}, + SQLsFromSQLFiles: []SQLsFromSQLFile{{SQLs: sqlsFromSQLFile}}, }, nil } From af370b9c71cbd3c5e7a12de764d5e5a7ac42d509 Mon Sep 17 00:00:00 2001 From: luowei Date: Thu, 18 Jan 2024 18:59:29 +0800 Subject: [PATCH 07/10] return file path and start line for SQLs which upload by single SQL file or XML file --- sqle/api/controller/base.go | 14 +++++++------- sqle/api/controller/v1/rule.go | 2 +- sqle/api/controller/v1/task.go | 14 +++++++++----- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/sqle/api/controller/base.go b/sqle/api/controller/base.go index 2946e2e7d5..5f05aa3159 100644 --- a/sqle/api/controller/base.go +++ b/sqle/api/controller/base.go @@ -123,26 +123,26 @@ func JSONOnlySupportForEnterpriseVersionErr(c echo.Context) error { return c.JSON(http.StatusOK, NewBaseReq(errors.NewOnlySupportForEnterpriseVersion())) } -// ReadFileContent read content from http body by name if file exist, +// ReadFile read content from http body by name if file exist, // the name is a http form data key, not file name. -func ReadFileContent(c echo.Context, name string) (content string, fileExist bool, err error) { +func ReadFile(c echo.Context, name string) (fileName, content string, fileExist bool, err error) { file, err := c.FormFile(name) if err == http.ErrMissingFile { - return "", false, nil + return "", "", false, nil } if err != nil { - return "", false, errors.New(errors.ReadUploadFileError, err) + return "", "", false, errors.New(errors.ReadUploadFileError, err) } src, err := file.Open() if err != nil { - return "", false, errors.New(errors.ReadUploadFileError, err) + return "", "", false, errors.New(errors.ReadUploadFileError, err) } defer src.Close() data, err := ioutil.ReadAll(src) if err != nil { - return "", false, errors.New(errors.ReadUploadFileError, err) + return "", "", false, errors.New(errors.ReadUploadFileError, err) } - return string(data), true, nil + return file.Filename, string(data), true, nil } // subjectUser should be admin user. diff --git a/sqle/api/controller/v1/rule.go b/sqle/api/controller/v1/rule.go index a59f9c45ac..bb59f9e1d2 100644 --- a/sqle/api/controller/v1/rule.go +++ b/sqle/api/controller/v1/rule.go @@ -1211,7 +1211,7 @@ type ParseProjectRuleTemplateFileResDataV1 struct { // @router /v1/rule_templates/parse [post] func ParseProjectRuleTemplateFile(c echo.Context) error { // 读取+解析文件 - file, exist, err := controller.ReadFileContent(c, "rule_template_file") + _, file, exist, err := controller.ReadFile(c, "rule_template_file") if err != nil { return controller.JSONBaseErrorReq(c, err) } diff --git a/sqle/api/controller/v1/task.go b/sqle/api/controller/v1/task.go index 319751c379..25cb0b8b32 100644 --- a/sqle/api/controller/v1/task.go +++ b/sqle/api/controller/v1/task.go @@ -78,19 +78,21 @@ const ( func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) { // Read it from sql file. - sqlsFromSQLFile, exist, err := controller.ReadFileContent(c, InputSQLFileName) + fileName, sqlsFromSQLFile, exist, err := controller.ReadFile(c, InputSQLFileName) if err != nil { return getSQLFromFileResp{}, err } if exist { return getSQLFromFileResp{ - SourceType: model.TaskSQLSourceFromSQLFile, - SQLsFromSQLFiles: []SQLsFromSQLFile{{SQLs: sqlsFromSQLFile}}, + SourceType: model.TaskSQLSourceFromSQLFile, + SQLsFromSQLFiles: []SQLsFromSQLFile{{ + FilePath: fileName, + SQLs: sqlsFromSQLFile}}, }, nil } // If sql_file is not exist, read it from mybatis xml file. - data, exist, err := controller.ReadFileContent(c, InputMyBatisXMLFileName) + fileName, data, exist, err := controller.ReadFile(c, InputMyBatisXMLFileName) if err != nil { return getSQLFromFileResp{}, err } @@ -102,7 +104,9 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) { sqlsFromXMLs := make([]SQLFromXML, len(sqls)) for i := range sqls { sqlsFromXMLs[i] = SQLFromXML{ - SQL: sqls[i].SQL, + FilePath: fileName, + StartLine: sqls[i].StartLine, + SQL: sqls[i].SQL, } } return getSQLFromFileResp{ From bfa3902d7a3a9f81777c77c85b034ba4eda0e184 Mon Sep 17 00:00:00 2001 From: luowei Date: Fri, 19 Jan 2024 18:13:11 +0800 Subject: [PATCH 08/10] update vendor --- go.mod | 4 ++-- go.sum | 8 ++++---- .../mybatis-mapper-2-sql/ast/mapper.go | 6 +++--- .../mybatis-mapper-2-sql/ast/mappers.go | 12 ++++++----- .../mybatis-mapper-2-sql/ast/query.go | 11 ++++++---- .../actiontech/mybatis-mapper-2-sql/parser.go | 20 ++++++++++++------- vendor/github.com/pingcap/parser/ast/ast.go | 4 ++++ vendor/github.com/pingcap/parser/ast/base.go | 13 +++++++++++- vendor/github.com/pingcap/parser/lexer.go | 10 ++++++++++ vendor/github.com/pingcap/parser/parser.go | 2 ++ vendor/github.com/pingcap/parser/parser.y | 2 ++ vendor/github.com/pingcap/parser/yy_parser.go | 1 + vendor/modules.txt | 6 +++--- 13 files changed, 70 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index ce512d9f59..5847245225 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/actiontech/dms v0.0.0-20231214070254-f5cbf6d30d45 github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62 - github.com/actiontech/mybatis-mapper-2-sql v0.5.0 + github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20240119100929-4c4ddc90b1c2 github.com/agiledragon/gomonkey v2.0.2+incompatible github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/alibabacloud-go/darabonba-openapi v0.1.18 @@ -181,6 +181,6 @@ replace ( cloud.google.com/go/compute/metadata => cloud.google.com/go/compute/metadata v0.1.0 github.com/labstack/echo/v4 => github.com/labstack/echo/v4 v4.6.1 github.com/pingcap/log => github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 - github.com/pingcap/parser => github.com/sjjian/parser v0.0.0-20231122073510-03e191068cf1 + github.com/pingcap/parser => github.com/sjjian/parser v0.0.0-20240119100847-3235e2338d0e google.golang.org/grpc => google.golang.org/grpc v1.29.0 ) diff --git a/go.sum b/go.sum index 0180cfa245..69af3427c2 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/actiontech/dms v0.0.0-20231214070254-f5cbf6d30d45 h1:/gAzpuj3c5Agdbhp github.com/actiontech/dms v0.0.0-20231214070254-f5cbf6d30d45/go.mod h1:jG8WjxiSBcUccweOqRPONDx95j88FwZytoa1/PI8I44= github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62 h1:JM7WnLzlvXOGE90KKd+aigi+qUDS+U5dLwQMNpTKZxE= github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62/go.mod h1:adDZHhAf2LRMx2h0JzofPXn12x2XlyQjVE116KXquwo= -github.com/actiontech/mybatis-mapper-2-sql v0.5.0 h1:TGovwZpLT+DUE5W0ZeSNE//LQLpVuQx8ghx0r8rPVBY= -github.com/actiontech/mybatis-mapper-2-sql v0.5.0/go.mod h1:ZMmUEDfbjm8oWxSAZkejqeOzlXa1BWNCfhNIxCMu7lw= +github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20240119100929-4c4ddc90b1c2 h1:Txeii4CZzDNbvvtGoJSFNcBTHyLD0uR6Sd16waQZSxY= +github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20240119100929-4c4ddc90b1c2/go.mod h1:ZMmUEDfbjm8oWxSAZkejqeOzlXa1BWNCfhNIxCMu7lw= github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw= github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -839,8 +839,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sjjian/parser v0.0.0-20231122073510-03e191068cf1 h1:0TrsSYbQFneiCIbZF3YWS0WJD5MP4J/CEoglDGrADUg= -github.com/sjjian/parser v0.0.0-20231122073510-03e191068cf1/go.mod h1:Qq2tnreUXwVo7NAKAHmbWFsgqpDUkxwhJCClY+ZCudA= +github.com/sjjian/parser v0.0.0-20240119100847-3235e2338d0e h1:f17wdoGuuBUt0U/YCidu9NIzZeLkkhooIBpODzIKuGs= +github.com/sjjian/parser v0.0.0-20240119100847-3235e2338d0e/go.mod h1:Qq2tnreUXwVo7NAKAHmbWFsgqpDUkxwhJCClY+ZCudA= github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= diff --git a/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mapper.go b/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mapper.go index 7588f30472..9bd186fab7 100644 --- a/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mapper.go +++ b/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mapper.go @@ -69,8 +69,8 @@ func (m *Mapper) GetStmt(ctx *Context) (string, error) { return strings.TrimSuffix(buff.String(), "\n"), nil } -func (m *Mapper) GetStmts(ctx *Context, skipErrorQuery bool) ([]string, error) { - var stmts []string +func (m *Mapper) GetStmts(ctx *Context, skipErrorQuery bool) ([]StmtInfo, error) { + var stmts []StmtInfo if len(ctx.Sqls) == 0 { ctx.Sqls = m.SqlNodes } @@ -78,7 +78,7 @@ func (m *Mapper) GetStmts(ctx *Context, skipErrorQuery bool) ([]string, error) { for _, a := range m.QueryNodes { data, err := a.GetStmt(ctx) if err == nil { - stmts = append(stmts, data) + stmts = append(stmts, StmtInfo{SQL: data, StartLine: a.StartLine}) continue } if skipErrorQuery { diff --git a/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mappers.go b/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mappers.go index 894d2641ff..2fd5283a14 100644 --- a/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mappers.go +++ b/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/mappers.go @@ -24,8 +24,9 @@ func (s *Mappers) AddMapper(ms ...*Mapper) error { } type StmtInfo struct { - FilePath string - SQL string + FilePath string + StartLine uint64 + SQL string } func (s *Mappers) GetStmts(skipErrorQuery bool) ([]StmtInfo, error) { @@ -43,10 +44,11 @@ func (s *Mappers) GetStmts(skipErrorQuery bool) ([]StmtInfo, error) { if err != nil { return nil, fmt.Errorf("get sqls from mapper failed, namespace: %v, err: %v", m.NameSpace, err) } - for _, sql := range stmt { + for _, info := range stmt { stmts = append(stmts, StmtInfo{ - FilePath: m.FilePath, - SQL: sql, + FilePath: m.FilePath, + StartLine: info.StartLine, + SQL: info.SQL, }) } } diff --git a/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/query.go b/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/query.go index 7482d144c7..20523df89c 100644 --- a/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/query.go +++ b/vendor/github.com/actiontech/mybatis-mapper-2-sql/ast/query.go @@ -9,12 +9,15 @@ import ( type QueryNode struct { *ChildrenNode - Id string - Type string + Id string + Type string + StartLine uint64 } -func NewQueryNode() *QueryNode { - n := &QueryNode{} +func NewQueryNode(startLine uint64) *QueryNode { + n := &QueryNode{ + StartLine: startLine, + } n.ChildrenNode = NewNode() return n } diff --git a/vendor/github.com/actiontech/mybatis-mapper-2-sql/parser.go b/vendor/github.com/actiontech/mybatis-mapper-2-sql/parser.go index cc73398150..9a1ea2abe4 100644 --- a/vendor/github.com/actiontech/mybatis-mapper-2-sql/parser.go +++ b/vendor/github.com/actiontech/mybatis-mapper-2-sql/parser.go @@ -95,7 +95,11 @@ func ParseXMLQuery(data string, skipErrorQuery bool) ([]string, error) { if err != nil { return nil, err } - return stmts, nil + sqls := []string{} + for _, stmt := range stmts { + sqls = append(sqls, stmt.SQL) + } + return sqls, nil } func parse(d *xml.Decoder) (node ast.Node, err error) { @@ -121,7 +125,7 @@ func parse(d *xml.Decoder) (node ast.Node, err error) { } func parseMyBatis(d *xml.Decoder, start *xml.StartElement) (node ast.Node, err error) { - node, err = scanMyBatis(start) + node, err = scanMyBatis(d, start) if err != nil { return nil, err } @@ -172,7 +176,7 @@ func parseMyBatis(d *xml.Decoder, start *xml.StartElement) (node ast.Node, err e return node, nil } -func scanMyBatis(start *xml.StartElement) (ast.Node, error) { +func scanMyBatis(d *xml.Decoder, start *xml.StartElement) (ast.Node, error) { var node ast.Node switch start.Name.Local { case "mapper": @@ -184,7 +188,8 @@ func scanMyBatis(start *xml.StartElement) (ast.Node, error) { case "property": node = ast.NewPropertyNode() case "select", "update", "delete", "insert": - node = ast.NewQueryNode() + startLine, _ := d.InputPos() + node = ast.NewQueryNode(uint64(startLine)) case "if": node = ast.NewIfNode() case "choose": @@ -207,7 +212,7 @@ func scanMyBatis(start *xml.StartElement) (ast.Node, error) { // ref: https://ibatis.apache.org/docs/java/pdf/iBATIS-SqlMaps-2_cn.pdf func parseIBatis(d *xml.Decoder, start *xml.StartElement) (node ast.Node, err error) { - node, err = scanIBatis(start) + node, err = scanIBatis(d, start) if err != nil { return nil, err } @@ -258,7 +263,7 @@ func parseIBatis(d *xml.Decoder, start *xml.StartElement) (node ast.Node, err er return node, nil } -func scanIBatis(start *xml.StartElement) (ast.Node, error) { +func scanIBatis(d *xml.Decoder, start *xml.StartElement) (ast.Node, error) { var node ast.Node switch start.Name.Local { case "sqlMap": @@ -268,7 +273,8 @@ func scanIBatis(start *xml.StartElement) (ast.Node, error) { case "include": node = ast.NewIncludeNode() case "select", "update", "delete", "insert", "statement": - node = ast.NewQueryNode() + startLine, _ := d.InputPos() + node = ast.NewQueryNode(uint64(startLine)) case "isEqual", "isNotEqual", "isGreaterThan", "isGreaterEqual", "isLessEqual", "isPropertyAvailable", "isNotPropertyAvailable", "isNull", "isNotNull", "isEmpty", "isNotEmpty": node = ast.NewConditionStmt() diff --git a/vendor/github.com/pingcap/parser/ast/ast.go b/vendor/github.com/pingcap/parser/ast/ast.go index b2628bedd8..8bb959d424 100644 --- a/vendor/github.com/pingcap/parser/ast/ast.go +++ b/vendor/github.com/pingcap/parser/ast/ast.go @@ -41,6 +41,10 @@ type Node interface { Text() string // SetText sets original text to the Node. SetText(text string) + // StartLine returns the start line of original text of the element. + StartLine() int + // SetStartLine sets the start line of original text of the element. + SetStartLine(startLine int) } // Flags indicates whether an expression contains certain types of expression. diff --git a/vendor/github.com/pingcap/parser/ast/base.go b/vendor/github.com/pingcap/parser/ast/base.go index 984d8e4d95..364f77c026 100644 --- a/vendor/github.com/pingcap/parser/ast/base.go +++ b/vendor/github.com/pingcap/parser/ast/base.go @@ -18,7 +18,8 @@ import "github.com/pingcap/parser/types" // node is the struct implements node interface except for Accept method. // Node implementations should embed it in. type node struct { - text string + text string + startLine int } // SetText implements Node interface. @@ -31,6 +32,16 @@ func (n *node) Text() string { return n.text } +// SetText implements Node interface. +func (n *node) SetStartLine(startLine int) { + n.startLine = startLine +} + +// Text implements Node interface. +func (n *node) StartLine() int { + return n.startLine +} + // stmtNode implements StmtNode interface. // Statement implementations should embed it in. type stmtNode struct { diff --git a/vendor/github.com/pingcap/parser/lexer.go b/vendor/github.com/pingcap/parser/lexer.go index 030ec96b89..cdb6b68c1d 100644 --- a/vendor/github.com/pingcap/parser/lexer.go +++ b/vendor/github.com/pingcap/parser/lexer.go @@ -57,6 +57,8 @@ type Scanner struct { // It's used to substring sql in syntax error message. lastScanOffset int + lastScanStartLine int + // lastKeyword records the previous keyword returned by scan(). // determine whether an optimizer hint should be parsed or ignored. lastKeyword int @@ -93,9 +95,17 @@ func (s *Scanner) stmtText() string { text := s.r.s[s.stmtStartPos:endPos] s.stmtStartPos = endPos + + trimedText := strings.TrimLeftFunc(text, unicode.IsSpace) + LFs := strings.Count(trimedText, "\n") + s.lastScanStartLine = s.r.pos().Line - LFs return text } +func (s *Scanner) startLine() int { + return s.lastScanStartLine +} + // Errorf tells scanner something is wrong. // Scanner satisfies yyLexer interface which need this function. func (s *Scanner) Errorf(format string, a ...interface{}) (err error) { diff --git a/vendor/github.com/pingcap/parser/parser.go b/vendor/github.com/pingcap/parser/parser.go index 2bedb3868b..25bfc54117 100644 --- a/vendor/github.com/pingcap/parser/parser.go +++ b/vendor/github.com/pingcap/parser/parser.go @@ -15412,6 +15412,7 @@ yynewstate: s := yyS[yypt-0].statement if lexer, ok := yylex.(stmtTexter); ok { s.SetText(lexer.stmtText()) + s.SetStartLine(lexer.startLine()) } parser.result = append(parser.result, s) } @@ -15421,6 +15422,7 @@ yynewstate: s := yyS[yypt-0].statement if lexer, ok := yylex.(stmtTexter); ok { s.SetText(lexer.stmtText()) + s.SetStartLine(lexer.startLine()) } parser.result = append(parser.result, s) } diff --git a/vendor/github.com/pingcap/parser/parser.y b/vendor/github.com/pingcap/parser/parser.y index 82ed8bc1ad..f24e413578 100644 --- a/vendor/github.com/pingcap/parser/parser.y +++ b/vendor/github.com/pingcap/parser/parser.y @@ -9486,6 +9486,7 @@ StatementList: s := $1 if lexer, ok := yylex.(stmtTexter); ok { s.SetText(lexer.stmtText()) + s.SetStartLine(lexer.startLine()) } parser.result = append(parser.result, s) } @@ -9496,6 +9497,7 @@ StatementList: s := $3 if lexer, ok := yylex.(stmtTexter); ok { s.SetText(lexer.stmtText()) + s.SetStartLine(lexer.startLine()) } parser.result = append(parser.result, s) } diff --git a/vendor/github.com/pingcap/parser/yy_parser.go b/vendor/github.com/pingcap/parser/yy_parser.go index 614dd05419..35c7a8e7a9 100644 --- a/vendor/github.com/pingcap/parser/yy_parser.go +++ b/vendor/github.com/pingcap/parser/yy_parser.go @@ -78,6 +78,7 @@ type Parser struct { type stmtTexter interface { stmtText() string + startLine() int } // New returns a Parser object. diff --git a/vendor/modules.txt b/vendor/modules.txt index 52a84b0570..67c5049511 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -63,7 +63,7 @@ github.com/actiontech/dms/pkg/dms-common/register ## explicit; go 1.19 github.com/actiontech/java-sql-extractor/java_antlr github.com/actiontech/java-sql-extractor/parser -# github.com/actiontech/mybatis-mapper-2-sql v0.5.0 +# github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20240119100929-4c4ddc90b1c2 ## explicit; go 1.14 github.com/actiontech/mybatis-mapper-2-sql github.com/actiontech/mybatis-mapper-2-sql/ast @@ -554,7 +554,7 @@ github.com/pingcap/errors # github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 => github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 ## explicit; go 1.13 github.com/pingcap/log -# github.com/pingcap/parser v3.0.12+incompatible => github.com/sjjian/parser v0.0.0-20231122073510-03e191068cf1 +# github.com/pingcap/parser v3.0.12+incompatible => github.com/sjjian/parser v0.0.0-20240119100847-3235e2338d0e ## explicit; go 1.13 github.com/pingcap/parser github.com/pingcap/parser/ast @@ -920,5 +920,5 @@ vitess.io/vitess/go/vt/vtgate/evalengine # cloud.google.com/go/compute/metadata => cloud.google.com/go/compute/metadata v0.1.0 # github.com/labstack/echo/v4 => github.com/labstack/echo/v4 v4.6.1 # github.com/pingcap/log => github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 -# github.com/pingcap/parser => github.com/sjjian/parser v0.0.0-20231122073510-03e191068cf1 +# github.com/pingcap/parser => github.com/sjjian/parser v0.0.0-20240119100847-3235e2338d0e # google.golang.org/grpc => google.golang.org/grpc v1.29.0 From c1d5308146803130fb204aefc73415dbdfdacd89 Mon Sep 17 00:00:00 2001 From: luowei Date: Fri, 19 Jan 2024 18:22:38 +0800 Subject: [PATCH 09/10] fix unit test --- sqle/server/sqled_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqle/server/sqled_test.go b/sqle/server/sqled_test.go index f0ba502009..f24a64bf49 100644 --- a/sqle/server/sqled_test.go +++ b/sqle/server/sqled_test.go @@ -170,7 +170,7 @@ func Test_action_audit_UpdateTask(t *testing.T) { mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta("INSERT INTO `execute_sql_detail`")). - WithArgs(model.MockTime, model.MockTime, nil, 0, 0, act.task.ExecuteSQLs[0].Content, "", "", 0, "", 0, 0, "", "", "", model.SQLAuditStatusFinished, `[{"level":"normal","message":"白名单","rule_name":""}]`, "2882fdbb7d5bcda7b49ea0803493467e", "normal"). + WithArgs(model.MockTime, model.MockTime, nil, 0, 0, act.task.ExecuteSQLs[0].Content, "", "", 0, "", 0, 0, "", "", "", 0, model.SQLAuditStatusFinished, `[{"level":"normal","message":"白名单","rule_name":""}]`, "2882fdbb7d5bcda7b49ea0803493467e", "normal"). WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() From 46285c4974507c8f6c3200b0eb418821e75b2a54 Mon Sep 17 00:00:00 2001 From: luowei Date: Fri, 19 Jan 2024 18:36:05 +0800 Subject: [PATCH 10/10] fix go lint error --- sqle/api/controller/v1/sql_audit_record.go | 12 ------------ sqle/cmd/scannerd/scanners/common/file.go | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/sqle/api/controller/v1/sql_audit_record.go b/sqle/api/controller/v1/sql_audit_record.go index cf0785487b..b1cd9aaada 100644 --- a/sqle/api/controller/v1/sql_audit_record.go +++ b/sqle/api/controller/v1/sql_audit_record.go @@ -16,7 +16,6 @@ import ( javaParser "github.com/actiontech/java-sql-extractor/parser" xmlParser "github.com/actiontech/mybatis-mapper-2-sql" - xmlAst "github.com/actiontech/mybatis-mapper-2-sql/ast" "github.com/actiontech/sqle/sqle/api/controller" "github.com/actiontech/sqle/sqle/common" "github.com/actiontech/sqle/sqle/dms" @@ -377,17 +376,6 @@ func getSqlsFromZip(c echo.Context) (sqlsFromSQLFile []SQLsFromSQLFile, sqlsFrom return sqlsFromSQLFile, sqlsFromXML, true, nil } -func getSQLsByFilePath(filePath string, stmtsInfo []xmlAst.StmtInfo) []string { - sqls := []string{} - for _, info := range stmtsInfo { - if info.FilePath != filePath { - continue - } - sqls = append(sqls, info.SQL) - } - return sqls -} - func parseXMLsWithFilePath(xmlContents []xmlParser.XmlFile) ([]SQLFromXML, error) { allStmtsFromXml, err := xmlParser.ParseXMLs(xmlContents, false) if err != nil { diff --git a/sqle/cmd/scannerd/scanners/common/file.go b/sqle/cmd/scannerd/scanners/common/file.go index 4d1c3723b4..cf57dcb39b 100644 --- a/sqle/cmd/scannerd/scanners/common/file.go +++ b/sqle/cmd/scannerd/scanners/common/file.go @@ -3,7 +3,6 @@ package common import ( "context" "fmt" - "github.com/actiontech/sqle/sqle/utils" "io/ioutil" "os" "path" @@ -12,6 +11,7 @@ import ( mybatisParser "github.com/actiontech/mybatis-mapper-2-sql" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" + "github.com/actiontech/sqle/sqle/utils" ) func GetSQLFromPath(pathName string, skipErrorQuery, skipErrorFile bool, fileSuffix string) (allSQL []driverV2.Node, err error) {