diff --git a/.changeset/yellow-apples-punch.md b/.changeset/yellow-apples-punch.md new file mode 100644 index 000000000..fca7f048e --- /dev/null +++ b/.changeset/yellow-apples-punch.md @@ -0,0 +1,5 @@ +--- +"@livekit/protocol": patch +--- + +add redact format option diff --git a/livekit/livekit_models.pb.go b/livekit/livekit_models.pb.go index afa6a19ea..b5f248cd8 100644 --- a/livekit/livekit_models.pb.go +++ b/livekit/livekit_models.pb.go @@ -21,6 +21,7 @@ package livekit import ( + _ "github.com/livekit/protocol/livekit/logger" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" @@ -5771,7 +5772,7 @@ var File_livekit_models_proto protoreflect.FileDescriptor const file_livekit_models_proto_rawDesc = "" + "\n" + - "\x14livekit_models.proto\x12\alivekit\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x15livekit_metrics.proto\"=\n" + + "\x14livekit_models.proto\x12\alivekit\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x15livekit_metrics.proto\x1a\x14logger/options.proto\"=\n" + "\n" + "Pagination\x12\x19\n" + "\bafter_id\x18\x01 \x01(\tR\aafterId\x12\x14\n" + diff --git a/livekit/logger/options.pb.go b/livekit/logger/options.pb.go index fc3cd3f6c..99e467571 100644 --- a/livekit/logger/options.pb.go +++ b/livekit/logger/options.pb.go @@ -30,12 +30,22 @@ var file_logger_options_proto_extTypes = []protoimpl.ExtensionInfo{ Tag: "varint,50001,opt,name=redact", Filename: "logger/options.proto", }, + { + ExtendedType: (*descriptorpb.FieldOptions)(nil), + ExtensionType: (*string)(nil), + Field: 50002, + Name: "logger.redact_format", + Tag: "bytes,50002,opt,name=redact_format", + Filename: "logger/options.proto", + }, } // Extension fields to descriptorpb.FieldOptions. var ( // optional bool redact = 50001; E_Redact = &file_logger_options_proto_extTypes[0] + // optional string redact_format = 50002; + E_RedactFormat = &file_logger_options_proto_extTypes[1] ) var File_logger_options_proto protoreflect.FileDescriptor @@ -43,17 +53,19 @@ var File_logger_options_proto protoreflect.FileDescriptor const file_logger_options_proto_rawDesc = "" + "\n" + "\x14logger/options.proto\x12\x06logger\x1a google/protobuf/descriptor.proto:7\n" + - "\x06redact\x12\x1d.google.protobuf.FieldOptions\x18ц\x03 \x01(\bR\x06redactB,Z*github.com/livekit/protocol/livekit/loggerb\x06proto3" + "\x06redact\x12\x1d.google.protobuf.FieldOptions\x18ц\x03 \x01(\bR\x06redact:D\n" + + "\rredact_format\x12\x1d.google.protobuf.FieldOptions\x18҆\x03 \x01(\tR\fredactFormatB,Z*github.com/livekit/protocol/livekit/loggerb\x06proto3" var file_logger_options_proto_goTypes = []any{ (*descriptorpb.FieldOptions)(nil), // 0: google.protobuf.FieldOptions } var file_logger_options_proto_depIdxs = []int32{ 0, // 0: logger.redact:extendee -> google.protobuf.FieldOptions - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 0, // [0:1] is the sub-list for extension extendee + 0, // 1: logger.redact_format:extendee -> google.protobuf.FieldOptions + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 0, // [0:2] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name } @@ -69,7 +81,7 @@ func file_logger_options_proto_init() { RawDescriptor: unsafe.Slice(unsafe.StringData(file_logger_options_proto_rawDesc), len(file_logger_options_proto_rawDesc)), NumEnums: 0, NumMessages: 0, - NumExtensions: 1, + NumExtensions: 2, NumServices: 0, }, GoTypes: file_logger_options_proto_goTypes, diff --git a/logger/proto.go b/logger/proto.go index 6ff2f7c68..25a1a0f55 100644 --- a/logger/proto.go +++ b/logger/proto.go @@ -15,15 +15,21 @@ package logger import ( + "bytes" "encoding/base64" "fmt" "strconv" + "text/template" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" + + "github.com/puzpuzpuz/xsync/v3" "github.com/livekit/protocol/livekit/logger" + "github.com/livekit/protocol/utils/must" ) func Proto(val proto.Message) zapcore.ObjectMarshaler { @@ -49,7 +55,7 @@ func (p protoMarshaller) MarshalLogObject(e zapcore.ObjectEncoder) error { v := p.m.Get(f) if proto.HasExtension(f.Options(), logger.E_Redact) { - e.AddString(k, "") + e.AddString(k, marshalRedacted(f, v)) continue } @@ -183,3 +189,57 @@ func marshalProtoBytes(b []byte) string { return fmt.Sprintf("%s... (%.2fGB)", s, float64(n)/float64(1<<30)) } } + +var redactTemplates = xsync.NewMapOf[string, *template.Template]() + +func marshalRedacted(f protoreflect.FieldDescriptor, v protoreflect.Value) string { + if !proto.HasExtension(f.Options(), logger.E_RedactFormat) { + return "" + } + + text := proto.GetExtension(f.Options(), logger.E_RedactFormat).(string) + tpl, _ := redactTemplates.LoadOrCompute(text, func() *template.Template { + return template.Must(template.New("format").Parse(text)) + }) + + var b bytes.Buffer + must.Do(tpl.Execute(&b, redactTemplateData{f, v})) + return b.String() +} + +type redactTemplateData struct { + f protoreflect.FieldDescriptor + v protoreflect.Value +} + +func (d redactTemplateData) TextName() string { + return d.f.TextName() +} + +func (d redactTemplateData) Size() string { + if !d.v.IsValid() { + return "0" + } + + msg := dynamicpb.NewMessage(d.f.ContainingMessage()) + + switch { + case d.f.IsList(): + dst := msg.Mutable(d.f).List() + src := d.v.List() + for i := 0; i < src.Len(); i++ { + dst.Append(src.Get(i)) + } + case d.f.IsMap(): + dst := msg.Mutable(d.f).Map() + src := d.v.Map() + src.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool { + dst.Set(k, v) + return true + }) + default: + msg.Set(d.f, d.v) + } + + return strconv.Itoa(proto.Size(msg.Interface())) +} diff --git a/protobufs/livekit_models.proto b/protobufs/livekit_models.proto index 287edf210..1bf42abc3 100644 --- a/protobufs/livekit_models.proto +++ b/protobufs/livekit_models.proto @@ -22,6 +22,7 @@ option ruby_package = "LiveKit::Proto"; import "google/protobuf/timestamp.proto"; import "livekit_metrics.proto"; +import "logger/options.proto"; message Pagination { string after_id = 1; // list entities which IDs are greater diff --git a/protobufs/logger/options.proto b/protobufs/logger/options.proto index c5c406af6..b120ea8d5 100644 --- a/protobufs/logger/options.proto +++ b/protobufs/logger/options.proto @@ -7,4 +7,5 @@ import "google/protobuf/descriptor.proto"; extend google.protobuf.FieldOptions { bool redact = 50001; + string redact_format = 50002; }