From 354ab6f01d3d34b0133e8a18e578b06721e19f99 Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Mon, 13 Apr 2026 05:02:23 -0700 Subject: [PATCH] [proto] add evaluation completed event contract --- README.md | 8 +- contract_test.go | 37 ++ contractfixtures/fixtures.go | 63 +++ contractfixtures/fixtures_test.go | 32 ++ eventhelpers/eventhelpers.go | 8 + eventhelpers/eventhelpers_test.go | 64 +++ gen/go/events/v1/evaluation.pb.go | 397 ++++++++++++++++++ gen/ts/events/v1/evaluation_pb.ts | 160 +++++++ package.json | 6 +- proto/events/v1/evaluation.proto | 39 ++ ...uation_completed_technical_capability.json | 50 +++ 11 files changed, 860 insertions(+), 4 deletions(-) create mode 100644 gen/go/events/v1/evaluation.pb.go create mode 100644 gen/ts/events/v1/evaluation_pb.ts create mode 100644 proto/events/v1/evaluation.proto create mode 100644 proto/events/v1/testdata/cloud_event_evaluation_completed_technical_capability.json diff --git a/README.md b/README.md index 7f34b8a..b3ccb7d 100644 --- a/README.md +++ b/README.md @@ -237,8 +237,8 @@ bus. - Use `github.com/evalops/proto/eventhelpers` to build and unpack canonical envelopes instead of re-implementing `Any` packing, protojson marshaling, and type assertions in each service. The package exports `NewCloudEvent`, - `NewChange`, `MarshalProtoJSON`, `UnpackChange`, and - `UnpackTapEventData`. `NewCloudEvent` also stamps + `NewChange`, `MarshalProtoJSON`, `UnpackChange`, + `UnpackEvaluationCompleted`, and `UnpackTapEventData`. `NewCloudEvent` also stamps `extensions.dataschema=buf.build/evalops/proto/` so published envelopes stay self-describing across service boundaries. - Use `config/v1.FeatureFlagSnapshot` when a repo needs a shared on-disk or @@ -278,7 +278,9 @@ generated types. High-risk boundary fixtures belong there too. The current catalog includes canonical `events/v1.CloudEvent` examples for `pipeline.changes.activity.create` with `outcome=replied` and `parker.changes.work_relationship.update` with -`status=terminated`. It also includes a Tap -> Pipeline boundary fixture for +`status=terminated`, plus `evaluation.completed` with a typed +`events/v1.EvaluationCompleted` payload for the Fermata -> Pipeline capability +signal seam. It also includes a Tap -> Pipeline boundary fixture for `ensemble.tap.hubspot.deal.updated` with a qualified stage change and a real UUID tenant, so downstream consumers can pin the semantics they depend on instead of only the wire shape. diff --git a/contract_test.go b/contract_test.go index 6acc0df..f474c99 100644 --- a/contract_test.go +++ b/contract_test.go @@ -589,6 +589,43 @@ func TestCloudEventPipelineActivityCreateRepliedFixtureMatchesProtoContract(t *t } } +func TestCloudEventEvaluationCompletedTechnicalCapabilityFixtureMatchesProtoContract(t *testing.T) { + t.Parallel() + + var message eventsv1.CloudEvent + loadProtoJSONFixture(t, filepath.Join("proto", "events", "v1", "testdata", "cloud_event_evaluation_completed_technical_capability.json"), &message) + + if message.GetType() != "evaluation.completed" { + t.Fatalf("expected type evaluation.completed, got %q", message.GetType()) + } + if message.GetSubject() != "product.evaluation.completed" { + t.Fatalf("expected subject product.evaluation.completed, got %q", message.GetSubject()) + } + if message.GetTenantId() != "11111111-1111-1111-1111-111111111111" { + t.Fatalf("expected tenant_id 11111111-1111-1111-1111-111111111111, got %q", message.GetTenantId()) + } + if got := message.GetExtensions()["dataschema"].GetStringValue(); got != "buf.build/evalops/proto/events.v1.EvaluationCompleted" { + t.Fatalf("expected dataschema buf.build/evalops/proto/events.v1.EvaluationCompleted, got %q", got) + } + + var unpacked eventsv1.EvaluationCompleted + if err := message.GetData().UnmarshalTo(&unpacked); err != nil { + t.Fatalf("unpack EvaluationCompleted payload: %v", err) + } + if unpacked.GetSignalType() != "technical_capability" { + t.Fatalf("expected signal_type technical_capability, got %q", unpacked.GetSignalType()) + } + if unpacked.GetRun().GetId() != "run-1" { + t.Fatalf("expected run.id run-1, got %q", unpacked.GetRun().GetId()) + } + if unpacked.GetMetrics().GetSuccessRate() != 0.9 { + t.Fatalf("expected metrics.success_rate 0.9, got %v", unpacked.GetMetrics().GetSuccessRate()) + } + if got := unpacked.GetCompanyDomains(); len(got) != 1 || got[0] != "acme.com" { + t.Fatalf("expected company_domains [acme.com], got %#v", got) + } +} + func TestCloudEventParkerWorkRelationshipUpdateTerminatedFixtureMatchesProtoContract(t *testing.T) { t.Parallel() diff --git a/contractfixtures/fixtures.go b/contractfixtures/fixtures.go index d4393b4..eef2993 100644 --- a/contractfixtures/fixtures.go +++ b/contractfixtures/fixtures.go @@ -18,6 +18,7 @@ import ( const ( ConfigFeatureFlagSnapshot = "config/v1/testdata/feature_flag_snapshot.json" + EventEvaluationCompletedTechnicalCapability = "events/v1/testdata/cloud_event_evaluation_completed_technical_capability.json" EventPipelineActivityCreateReplied = "events/v1/testdata/cloud_event_pipeline_activity_create_replied.json" EventParkerWorkRelationshipUpdateTerminated = "events/v1/testdata/cloud_event_parker_work_relationship_update_terminated.json" EventTapHubspotDealQualified = "events/v1/testdata/cloud_event_tap_hubspot_deal_qualified.json" @@ -30,6 +31,56 @@ const ( ) var embeddedFixtures = map[string][]byte{ + EventEvaluationCompletedTechnicalCapability: []byte(`{ + "spec_version": "1.0", + "id": "evt_eval_completed_technical_capability_1", + "type": "evaluation.completed", + "source": "fermata", + "subject": "product.evaluation.completed", + "time": "2026-04-13T12:00:00Z", + "data_content_type": "application/protobuf", + "tenant_id": "11111111-1111-1111-1111-111111111111", + "data": { + "@type": "type.googleapis.com/events.v1.EvaluationCompleted", + "signal_type": "technical_capability", + "summary": "Claude beats GPT-4 on latency by 40% on enterprise workloads.", + "success_rate": 0.9, + "company_domains": [ + "acme.com" + ], + "company_names": [ + "Acme Corporation" + ], + "deal_ids": [ + "deal-123" + ], + "run": { + "id": "run-1", + "test_suite_id": "suite-1", + "test_suite_name": "Latency benchmark", + "name": "Enterprise latency proof", + "description": "Claude beats GPT-4 on latency by 40% on enterprise workloads.", + "tags": [ + "pipeline:signal_type=technical_capability", + "pipeline:company_domain=acme.com", + "pipeline:company_name=Acme Corporation", + "pipeline:deal_id=deal-123" + ], + "completed_at": "2026-04-13T12:00:00Z" + }, + "metrics": { + "total_tests": "20", + "passed_tests": "18", + "failed_tests": "2", + "total_cost": 4.2, + "duration": 12.5, + "success_rate": 0.9 + } + }, + "extensions": { + "dataschema": "buf.build/evalops/proto/events.v1.EvaluationCompleted" + } +}`), MeterRecordUsageRequestLLMGatewayResponses: []byte(`{ "team_id": "team_eng", "agent_id": "agent_456", @@ -130,6 +181,18 @@ func LoadChangeFixture(name string) (*eventsv1.CloudEvent, *eventsv1.Change, err return envelope, change, nil } +func LoadEvaluationCompletedFixture(name string) (*eventsv1.CloudEvent, *eventsv1.EvaluationCompleted, error) { + envelope, err := LoadCloudEvent(name) + if err != nil { + return nil, nil, err + } + message, err := eventhelpers.UnpackEvaluationCompleted(envelope) + if err != nil { + return nil, nil, fmt.Errorf("unmarshal evaluation fixture %q: %w", name, err) + } + return envelope, message, nil +} + func LoadTapFixture(name string) (*eventsv1.CloudEvent, *tapv1.TapEventData, error) { envelope, err := LoadCloudEvent(name) if err != nil { diff --git a/contractfixtures/fixtures_test.go b/contractfixtures/fixtures_test.go index e50ad0f..fa1092b 100644 --- a/contractfixtures/fixtures_test.go +++ b/contractfixtures/fixtures_test.go @@ -5,6 +5,7 @@ import ( "testing" configv1 "github.com/evalops/proto/gen/go/config/v1" + eventsv1 "github.com/evalops/proto/gen/go/events/v1" meterv1 "github.com/evalops/proto/gen/go/meter/v1" ) @@ -61,6 +62,37 @@ func TestLoadTapFixture(t *testing.T) { } } +func TestLoadEvaluationCompletedFixture(t *testing.T) { + t.Parallel() + + envelope, message, err := LoadEvaluationCompletedFixture(EventEvaluationCompletedTechnicalCapability) + if err != nil { + t.Fatalf("load evaluation fixture: %v", err) + } + if envelope.GetType() != "evaluation.completed" { + t.Fatalf("unexpected type %q", envelope.GetType()) + } + if got := envelope.GetExtensions()["dataschema"].GetStringValue(); got != "buf.build/evalops/proto/events.v1.EvaluationCompleted" { + t.Fatalf("unexpected dataschema %q", got) + } + if message.GetSignalType() != "technical_capability" { + t.Fatalf("unexpected signal_type %q", message.GetSignalType()) + } + if message.GetRun().GetId() != "run-1" { + t.Fatalf("unexpected run.id %q", message.GetRun().GetId()) + } + if message.GetMetrics().GetSuccessRate() != 0.9 { + t.Fatalf("unexpected metrics.success_rate %v", message.GetMetrics().GetSuccessRate()) + } + if envelope.GetData().GetTypeUrl() != "type.googleapis.com/events.v1.EvaluationCompleted" { + t.Fatalf("unexpected type URL %q", envelope.GetData().GetTypeUrl()) + } + var direct eventsv1.EvaluationCompleted + if err := envelope.GetData().UnmarshalTo(&direct); err != nil { + t.Fatalf("unmarshal direct evaluation payload: %v", err) + } +} + func TestUnmarshalProtoJSONLoadsMeterFixture(t *testing.T) { t.Parallel() diff --git a/eventhelpers/eventhelpers.go b/eventhelpers/eventhelpers.go index 5f8e229..ebeb05b 100644 --- a/eventhelpers/eventhelpers.go +++ b/eventhelpers/eventhelpers.go @@ -144,6 +144,14 @@ func UnpackChange(envelope *eventsv1.CloudEvent) (*eventsv1.Change, error) { return message, nil } +func UnpackEvaluationCompleted(envelope *eventsv1.CloudEvent) (*eventsv1.EvaluationCompleted, error) { + message := &eventsv1.EvaluationCompleted{} + if err := UnpackData(envelope, message); err != nil { + return nil, err + } + return message, nil +} + func UnpackTapEventData(envelope *eventsv1.CloudEvent) (*tapv1.TapEventData, error) { message := &tapv1.TapEventData{} if err := UnpackData(envelope, message); err != nil { diff --git a/eventhelpers/eventhelpers_test.go b/eventhelpers/eventhelpers_test.go index 9b52f20..cfdf49c 100644 --- a/eventhelpers/eventhelpers_test.go +++ b/eventhelpers/eventhelpers_test.go @@ -112,6 +112,62 @@ func TestUnpackTapEventDataRoundTrip(t *testing.T) { } } +func TestUnpackEvaluationCompletedRoundTrip(t *testing.T) { + t.Parallel() + + message := &eventsv1.EvaluationCompleted{ + SignalType: "technical_capability", + Summary: "Claude beats GPT-4 on latency by 40% on enterprise workloads.", + SuccessRate: float64ptr(0.9), + CompanyDomains: []string{"acme.com"}, + CompanyNames: []string{"Acme Corporation"}, + DealIds: []string{"deal-123"}, + Run: &eventsv1.EvaluationRun{ + Id: "run-1", + TestSuiteId: "suite-1", + TestSuiteName: "Latency benchmark", + Name: "Enterprise latency proof", + Description: "Claude beats GPT-4 on latency by 40% on enterprise workloads.", + Tags: []string{"pipeline:signal_type=technical_capability"}, + }, + Metrics: &eventsv1.EvaluationMetrics{ + TotalTests: int64ptr(20), + PassedTests: int64ptr(18), + FailedTests: int64ptr(2), + TotalCost: float64ptr(4.2), + Duration: float64ptr(12.5), + SuccessRate: float64ptr(0.9), + }, + } + + envelope, err := NewCloudEvent( + "evt_eval_123", + "evaluation.completed", + "fermata", + "product.evaluation.completed", + "11111111-1111-1111-1111-111111111111", + time.Date(2026, 4, 13, 12, 0, 0, 0, time.UTC), + message, + ) + if err != nil { + t.Fatalf("NewCloudEvent() error = %v", err) + } + + unpacked, err := UnpackEvaluationCompleted(envelope) + if err != nil { + t.Fatalf("UnpackEvaluationCompleted() error = %v", err) + } + if unpacked.GetSignalType() != "technical_capability" { + t.Fatalf("unexpected signal_type %q", unpacked.GetSignalType()) + } + if unpacked.GetMetrics().GetSuccessRate() != 0.9 { + t.Fatalf("unexpected success_rate %v", unpacked.GetMetrics().GetSuccessRate()) + } + if got := envelope.GetExtensions()["dataschema"].GetStringValue(); got != "buf.build/evalops/proto/events.v1.EvaluationCompleted" { + t.Fatalf("unexpected dataschema %q", got) + } +} + func TestNewChangeBuildsCanonicalMessageFromJSONPayload(t *testing.T) { t.Parallel() @@ -159,3 +215,11 @@ func mustStruct(t *testing.T, fields map[string]any) *structpb.Struct { } return message } + +func int64ptr(value int64) *int64 { + return &value +} + +func float64ptr(value float64) *float64 { + return &value +} diff --git a/gen/go/events/v1/evaluation.pb.go b/gen/go/events/v1/evaluation.pb.go new file mode 100644 index 0000000..637a54c --- /dev/null +++ b/gen/go/events/v1/evaluation.pb.go @@ -0,0 +1,397 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: events/v1/evaluation.proto + +package eventsv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// EvaluationCompleted is the canonical typed payload for evaluation.completed +// events that downstream services consume as pipeline signals. +type EvaluationCompleted struct { + state protoimpl.MessageState `protogen:"open.v1"` + SignalType string `protobuf:"bytes,1,opt,name=signal_type,json=signalType,proto3" json:"signal_type,omitempty"` + Summary string `protobuf:"bytes,2,opt,name=summary,proto3" json:"summary,omitempty"` + SuccessRate *float64 `protobuf:"fixed64,3,opt,name=success_rate,json=successRate,proto3,oneof" json:"success_rate,omitempty"` + CompanyDomains []string `protobuf:"bytes,4,rep,name=company_domains,json=companyDomains,proto3" json:"company_domains,omitempty"` + CompanyNames []string `protobuf:"bytes,5,rep,name=company_names,json=companyNames,proto3" json:"company_names,omitempty"` + DealIds []string `protobuf:"bytes,6,rep,name=deal_ids,json=dealIds,proto3" json:"deal_ids,omitempty"` + Run *EvaluationRun `protobuf:"bytes,7,opt,name=run,proto3" json:"run,omitempty"` + Metrics *EvaluationMetrics `protobuf:"bytes,8,opt,name=metrics,proto3" json:"metrics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluationCompleted) Reset() { + *x = EvaluationCompleted{} + mi := &file_events_v1_evaluation_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluationCompleted) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluationCompleted) ProtoMessage() {} + +func (x *EvaluationCompleted) ProtoReflect() protoreflect.Message { + mi := &file_events_v1_evaluation_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluationCompleted.ProtoReflect.Descriptor instead. +func (*EvaluationCompleted) Descriptor() ([]byte, []int) { + return file_events_v1_evaluation_proto_rawDescGZIP(), []int{0} +} + +func (x *EvaluationCompleted) GetSignalType() string { + if x != nil { + return x.SignalType + } + return "" +} + +func (x *EvaluationCompleted) GetSummary() string { + if x != nil { + return x.Summary + } + return "" +} + +func (x *EvaluationCompleted) GetSuccessRate() float64 { + if x != nil && x.SuccessRate != nil { + return *x.SuccessRate + } + return 0 +} + +func (x *EvaluationCompleted) GetCompanyDomains() []string { + if x != nil { + return x.CompanyDomains + } + return nil +} + +func (x *EvaluationCompleted) GetCompanyNames() []string { + if x != nil { + return x.CompanyNames + } + return nil +} + +func (x *EvaluationCompleted) GetDealIds() []string { + if x != nil { + return x.DealIds + } + return nil +} + +func (x *EvaluationCompleted) GetRun() *EvaluationRun { + if x != nil { + return x.Run + } + return nil +} + +func (x *EvaluationCompleted) GetMetrics() *EvaluationMetrics { + if x != nil { + return x.Metrics + } + return nil +} + +type EvaluationRun struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + TestSuiteId string `protobuf:"bytes,2,opt,name=test_suite_id,json=testSuiteId,proto3" json:"test_suite_id,omitempty"` + TestSuiteName string `protobuf:"bytes,3,opt,name=test_suite_name,json=testSuiteName,proto3" json:"test_suite_name,omitempty"` + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,5,opt,name=description,proto3" json:"description,omitempty"` + Tags []string `protobuf:"bytes,6,rep,name=tags,proto3" json:"tags,omitempty"` + CompletedAt *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=completed_at,json=completedAt,proto3" json:"completed_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluationRun) Reset() { + *x = EvaluationRun{} + mi := &file_events_v1_evaluation_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluationRun) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluationRun) ProtoMessage() {} + +func (x *EvaluationRun) ProtoReflect() protoreflect.Message { + mi := &file_events_v1_evaluation_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluationRun.ProtoReflect.Descriptor instead. +func (*EvaluationRun) Descriptor() ([]byte, []int) { + return file_events_v1_evaluation_proto_rawDescGZIP(), []int{1} +} + +func (x *EvaluationRun) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *EvaluationRun) GetTestSuiteId() string { + if x != nil { + return x.TestSuiteId + } + return "" +} + +func (x *EvaluationRun) GetTestSuiteName() string { + if x != nil { + return x.TestSuiteName + } + return "" +} + +func (x *EvaluationRun) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *EvaluationRun) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *EvaluationRun) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *EvaluationRun) GetCompletedAt() *timestamppb.Timestamp { + if x != nil { + return x.CompletedAt + } + return nil +} + +type EvaluationMetrics struct { + state protoimpl.MessageState `protogen:"open.v1"` + TotalTests *int64 `protobuf:"varint,1,opt,name=total_tests,json=totalTests,proto3,oneof" json:"total_tests,omitempty"` + PassedTests *int64 `protobuf:"varint,2,opt,name=passed_tests,json=passedTests,proto3,oneof" json:"passed_tests,omitempty"` + FailedTests *int64 `protobuf:"varint,3,opt,name=failed_tests,json=failedTests,proto3,oneof" json:"failed_tests,omitempty"` + TotalCost *float64 `protobuf:"fixed64,4,opt,name=total_cost,json=totalCost,proto3,oneof" json:"total_cost,omitempty"` + Duration *float64 `protobuf:"fixed64,5,opt,name=duration,proto3,oneof" json:"duration,omitempty"` + SuccessRate *float64 `protobuf:"fixed64,6,opt,name=success_rate,json=successRate,proto3,oneof" json:"success_rate,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EvaluationMetrics) Reset() { + *x = EvaluationMetrics{} + mi := &file_events_v1_evaluation_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EvaluationMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvaluationMetrics) ProtoMessage() {} + +func (x *EvaluationMetrics) ProtoReflect() protoreflect.Message { + mi := &file_events_v1_evaluation_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvaluationMetrics.ProtoReflect.Descriptor instead. +func (*EvaluationMetrics) Descriptor() ([]byte, []int) { + return file_events_v1_evaluation_proto_rawDescGZIP(), []int{2} +} + +func (x *EvaluationMetrics) GetTotalTests() int64 { + if x != nil && x.TotalTests != nil { + return *x.TotalTests + } + return 0 +} + +func (x *EvaluationMetrics) GetPassedTests() int64 { + if x != nil && x.PassedTests != nil { + return *x.PassedTests + } + return 0 +} + +func (x *EvaluationMetrics) GetFailedTests() int64 { + if x != nil && x.FailedTests != nil { + return *x.FailedTests + } + return 0 +} + +func (x *EvaluationMetrics) GetTotalCost() float64 { + if x != nil && x.TotalCost != nil { + return *x.TotalCost + } + return 0 +} + +func (x *EvaluationMetrics) GetDuration() float64 { + if x != nil && x.Duration != nil { + return *x.Duration + } + return 0 +} + +func (x *EvaluationMetrics) GetSuccessRate() float64 { + if x != nil && x.SuccessRate != nil { + return *x.SuccessRate + } + return 0 +} + +var File_events_v1_evaluation_proto protoreflect.FileDescriptor + +const file_events_v1_evaluation_proto_rawDesc = "" + + "\n" + + "\x1aevents/v1/evaluation.proto\x12\tevents.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xd6\x02\n" + + "\x13EvaluationCompleted\x12\x1f\n" + + "\vsignal_type\x18\x01 \x01(\tR\n" + + "signalType\x12\x18\n" + + "\asummary\x18\x02 \x01(\tR\asummary\x12&\n" + + "\fsuccess_rate\x18\x03 \x01(\x01H\x00R\vsuccessRate\x88\x01\x01\x12'\n" + + "\x0fcompany_domains\x18\x04 \x03(\tR\x0ecompanyDomains\x12#\n" + + "\rcompany_names\x18\x05 \x03(\tR\fcompanyNames\x12\x19\n" + + "\bdeal_ids\x18\x06 \x03(\tR\adealIds\x12*\n" + + "\x03run\x18\a \x01(\v2\x18.events.v1.EvaluationRunR\x03run\x126\n" + + "\ametrics\x18\b \x01(\v2\x1c.events.v1.EvaluationMetricsR\ametricsB\x0f\n" + + "\r_success_rate\"\xf4\x01\n" + + "\rEvaluationRun\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\"\n" + + "\rtest_suite_id\x18\x02 \x01(\tR\vtestSuiteId\x12&\n" + + "\x0ftest_suite_name\x18\x03 \x01(\tR\rtestSuiteName\x12\x12\n" + + "\x04name\x18\x04 \x01(\tR\x04name\x12 \n" + + "\vdescription\x18\x05 \x01(\tR\vdescription\x12\x12\n" + + "\x04tags\x18\x06 \x03(\tR\x04tags\x12=\n" + + "\fcompleted_at\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\vcompletedAt\"\xd5\x02\n" + + "\x11EvaluationMetrics\x12$\n" + + "\vtotal_tests\x18\x01 \x01(\x03H\x00R\n" + + "totalTests\x88\x01\x01\x12&\n" + + "\fpassed_tests\x18\x02 \x01(\x03H\x01R\vpassedTests\x88\x01\x01\x12&\n" + + "\ffailed_tests\x18\x03 \x01(\x03H\x02R\vfailedTests\x88\x01\x01\x12\"\n" + + "\n" + + "total_cost\x18\x04 \x01(\x01H\x03R\ttotalCost\x88\x01\x01\x12\x1f\n" + + "\bduration\x18\x05 \x01(\x01H\x04R\bduration\x88\x01\x01\x12&\n" + + "\fsuccess_rate\x18\x06 \x01(\x01H\x05R\vsuccessRate\x88\x01\x01B\x0e\n" + + "\f_total_testsB\x0f\n" + + "\r_passed_testsB\x0f\n" + + "\r_failed_testsB\r\n" + + "\v_total_costB\v\n" + + "\t_durationB\x0f\n" + + "\r_success_rateB4Z2github.com/evalops/proto/gen/go/events/v1;eventsv1b\x06proto3" + +var ( + file_events_v1_evaluation_proto_rawDescOnce sync.Once + file_events_v1_evaluation_proto_rawDescData []byte +) + +func file_events_v1_evaluation_proto_rawDescGZIP() []byte { + file_events_v1_evaluation_proto_rawDescOnce.Do(func() { + file_events_v1_evaluation_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_events_v1_evaluation_proto_rawDesc), len(file_events_v1_evaluation_proto_rawDesc))) + }) + return file_events_v1_evaluation_proto_rawDescData +} + +var file_events_v1_evaluation_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_events_v1_evaluation_proto_goTypes = []any{ + (*EvaluationCompleted)(nil), // 0: events.v1.EvaluationCompleted + (*EvaluationRun)(nil), // 1: events.v1.EvaluationRun + (*EvaluationMetrics)(nil), // 2: events.v1.EvaluationMetrics + (*timestamppb.Timestamp)(nil), // 3: google.protobuf.Timestamp +} +var file_events_v1_evaluation_proto_depIdxs = []int32{ + 1, // 0: events.v1.EvaluationCompleted.run:type_name -> events.v1.EvaluationRun + 2, // 1: events.v1.EvaluationCompleted.metrics:type_name -> events.v1.EvaluationMetrics + 3, // 2: events.v1.EvaluationRun.completed_at:type_name -> google.protobuf.Timestamp + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_events_v1_evaluation_proto_init() } +func file_events_v1_evaluation_proto_init() { + if File_events_v1_evaluation_proto != nil { + return + } + file_events_v1_evaluation_proto_msgTypes[0].OneofWrappers = []any{} + file_events_v1_evaluation_proto_msgTypes[2].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_events_v1_evaluation_proto_rawDesc), len(file_events_v1_evaluation_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_events_v1_evaluation_proto_goTypes, + DependencyIndexes: file_events_v1_evaluation_proto_depIdxs, + MessageInfos: file_events_v1_evaluation_proto_msgTypes, + }.Build() + File_events_v1_evaluation_proto = out.File + file_events_v1_evaluation_proto_goTypes = nil + file_events_v1_evaluation_proto_depIdxs = nil +} diff --git a/gen/ts/events/v1/evaluation_pb.ts b/gen/ts/events/v1/evaluation_pb.ts new file mode 100644 index 0000000..ead888c --- /dev/null +++ b/gen/ts/events/v1/evaluation_pb.ts @@ -0,0 +1,160 @@ +// @generated by protoc-gen-es v2.11.0 with parameter "target=ts,import_extension=.js" +// @generated from file events/v1/evaluation.proto (package events.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; +import type { Timestamp } from "@bufbuild/protobuf/wkt"; +import { file_google_protobuf_timestamp } from "@bufbuild/protobuf/wkt"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file events/v1/evaluation.proto. + */ +export const file_events_v1_evaluation: GenFile = /*@__PURE__*/ + fileDesc("ChpldmVudHMvdjEvZXZhbHVhdGlvbi5wcm90bxIJZXZlbnRzLnYxIv8BChNFdmFsdWF0aW9uQ29tcGxldGVkEhMKC3NpZ25hbF90eXBlGAEgASgJEg8KB3N1bW1hcnkYAiABKAkSGQoMc3VjY2Vzc19yYXRlGAMgASgBSACIAQESFwoPY29tcGFueV9kb21haW5zGAQgAygJEhUKDWNvbXBhbnlfbmFtZXMYBSADKAkSEAoIZGVhbF9pZHMYBiADKAkSJQoDcnVuGAcgASgLMhguZXZlbnRzLnYxLkV2YWx1YXRpb25SdW4SLQoHbWV0cmljcxgIIAEoCzIcLmV2ZW50cy52MS5FdmFsdWF0aW9uTWV0cmljc0IPCg1fc3VjY2Vzc19yYXRlIq4BCg1FdmFsdWF0aW9uUnVuEgoKAmlkGAEgASgJEhUKDXRlc3Rfc3VpdGVfaWQYAiABKAkSFwoPdGVzdF9zdWl0ZV9uYW1lGAMgASgJEgwKBG5hbWUYBCABKAkSEwoLZGVzY3JpcHRpb24YBSABKAkSDAoEdGFncxgGIAMoCRIwCgxjb21wbGV0ZWRfYXQYByABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIo0CChFFdmFsdWF0aW9uTWV0cmljcxIYCgt0b3RhbF90ZXN0cxgBIAEoA0gAiAEBEhkKDHBhc3NlZF90ZXN0cxgCIAEoA0gBiAEBEhkKDGZhaWxlZF90ZXN0cxgDIAEoA0gCiAEBEhcKCnRvdGFsX2Nvc3QYBCABKAFIA4gBARIVCghkdXJhdGlvbhgFIAEoAUgEiAEBEhkKDHN1Y2Nlc3NfcmF0ZRgGIAEoAUgFiAEBQg4KDF90b3RhbF90ZXN0c0IPCg1fcGFzc2VkX3Rlc3RzQg8KDV9mYWlsZWRfdGVzdHNCDQoLX3RvdGFsX2Nvc3RCCwoJX2R1cmF0aW9uQg8KDV9zdWNjZXNzX3JhdGVCNFoyZ2l0aHViLmNvbS9ldmFsb3BzL3Byb3RvL2dlbi9nby9ldmVudHMvdjE7ZXZlbnRzdjFiBnByb3RvMw", [file_google_protobuf_timestamp]); + +/** + * EvaluationCompleted is the canonical typed payload for evaluation.completed + * events that downstream services consume as pipeline signals. + * + * @generated from message events.v1.EvaluationCompleted + */ +export type EvaluationCompleted = Message<"events.v1.EvaluationCompleted"> & { + /** + * @generated from field: string signal_type = 1; + */ + signalType: string; + + /** + * @generated from field: string summary = 2; + */ + summary: string; + + /** + * @generated from field: optional double success_rate = 3; + */ + successRate?: number; + + /** + * @generated from field: repeated string company_domains = 4; + */ + companyDomains: string[]; + + /** + * @generated from field: repeated string company_names = 5; + */ + companyNames: string[]; + + /** + * @generated from field: repeated string deal_ids = 6; + */ + dealIds: string[]; + + /** + * @generated from field: events.v1.EvaluationRun run = 7; + */ + run?: EvaluationRun; + + /** + * @generated from field: events.v1.EvaluationMetrics metrics = 8; + */ + metrics?: EvaluationMetrics; +}; + +/** + * Describes the message events.v1.EvaluationCompleted. + * Use `create(EvaluationCompletedSchema)` to create a new message. + */ +export const EvaluationCompletedSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_events_v1_evaluation, 0); + +/** + * @generated from message events.v1.EvaluationRun + */ +export type EvaluationRun = Message<"events.v1.EvaluationRun"> & { + /** + * @generated from field: string id = 1; + */ + id: string; + + /** + * @generated from field: string test_suite_id = 2; + */ + testSuiteId: string; + + /** + * @generated from field: string test_suite_name = 3; + */ + testSuiteName: string; + + /** + * @generated from field: string name = 4; + */ + name: string; + + /** + * @generated from field: string description = 5; + */ + description: string; + + /** + * @generated from field: repeated string tags = 6; + */ + tags: string[]; + + /** + * @generated from field: google.protobuf.Timestamp completed_at = 7; + */ + completedAt?: Timestamp; +}; + +/** + * Describes the message events.v1.EvaluationRun. + * Use `create(EvaluationRunSchema)` to create a new message. + */ +export const EvaluationRunSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_events_v1_evaluation, 1); + +/** + * @generated from message events.v1.EvaluationMetrics + */ +export type EvaluationMetrics = Message<"events.v1.EvaluationMetrics"> & { + /** + * @generated from field: optional int64 total_tests = 1; + */ + totalTests?: bigint; + + /** + * @generated from field: optional int64 passed_tests = 2; + */ + passedTests?: bigint; + + /** + * @generated from field: optional int64 failed_tests = 3; + */ + failedTests?: bigint; + + /** + * @generated from field: optional double total_cost = 4; + */ + totalCost?: number; + + /** + * @generated from field: optional double duration = 5; + */ + duration?: number; + + /** + * @generated from field: optional double success_rate = 6; + */ + successRate?: number; +}; + +/** + * Describes the message events.v1.EvaluationMetrics. + * Use `create(EvaluationMetricsSchema)` to create a new message. + */ +export const EvaluationMetricsSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_events_v1_evaluation, 2); + diff --git a/package.json b/package.json index 016ba97..4d3db7a 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,10 @@ "types": "./gen/dist/events/v1/cloudevent_pb.d.ts", "import": "./gen/dist/events/v1/cloudevent_pb.js" }, + "./events/v1/evaluation_pb": { + "types": "./gen/dist/events/v1/evaluation_pb.d.ts", + "import": "./gen/dist/events/v1/evaluation_pb.js" + }, "./governance/v1/governance_pb": { "types": "./gen/dist/governance/v1/governance_pb.d.ts", "import": "./gen/dist/governance/v1/governance_pb.js" @@ -86,7 +90,7 @@ }, "scripts": { "build": "tsc -p tsconfig.package.json", - "check:package": "npm run build && node --input-type=module -e \"const config = await import('@evalops/proto/config/v1/config_pb'); const memory = await import('@evalops/proto/memory/v1/memory_pb'); const meter = await import('@evalops/proto/meter/v1/meter_pb'); const events = await import('@evalops/proto/events/v1/cloudevent_pb'); const prompts = await import('@evalops/proto/prompts/v1/prompts_pb'); if (!config.FeatureFlagSnapshotSchema || !memory.RecallRequestSchema || !meter.RecordUsageRequestSchema || !events.CloudEventSchema || !prompts.ResolveRequestSchema) throw new Error('package exports failed');\"", + "check:package": "npm run build && node --input-type=module -e \"const config = await import('@evalops/proto/config/v1/config_pb'); const memory = await import('@evalops/proto/memory/v1/memory_pb'); const meter = await import('@evalops/proto/meter/v1/meter_pb'); const cloudEvent = await import('@evalops/proto/events/v1/cloudevent_pb'); const evaluation = await import('@evalops/proto/events/v1/evaluation_pb'); const prompts = await import('@evalops/proto/prompts/v1/prompts_pb'); if (!config.FeatureFlagSnapshotSchema || !memory.RecallRequestSchema || !meter.RecordUsageRequestSchema || !cloudEvent.CloudEventSchema || !evaluation.EvaluationCompletedSchema || !prompts.ResolveRequestSchema) throw new Error('package exports failed');\"", "prepare": "npm run build" }, "repository": { diff --git a/proto/events/v1/evaluation.proto b/proto/events/v1/evaluation.proto new file mode 100644 index 0000000..ba7e9d9 --- /dev/null +++ b/proto/events/v1/evaluation.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package events.v1; + +option go_package = "github.com/evalops/proto/gen/go/events/v1;eventsv1"; + +import "google/protobuf/timestamp.proto"; + +// EvaluationCompleted is the canonical typed payload for evaluation.completed +// events that downstream services consume as pipeline signals. +message EvaluationCompleted { + string signal_type = 1; + string summary = 2; + optional double success_rate = 3; + repeated string company_domains = 4; + repeated string company_names = 5; + repeated string deal_ids = 6; + EvaluationRun run = 7; + EvaluationMetrics metrics = 8; +} + +message EvaluationRun { + string id = 1; + string test_suite_id = 2; + string test_suite_name = 3; + string name = 4; + string description = 5; + repeated string tags = 6; + google.protobuf.Timestamp completed_at = 7; +} + +message EvaluationMetrics { + optional int64 total_tests = 1; + optional int64 passed_tests = 2; + optional int64 failed_tests = 3; + optional double total_cost = 4; + optional double duration = 5; + optional double success_rate = 6; +} diff --git a/proto/events/v1/testdata/cloud_event_evaluation_completed_technical_capability.json b/proto/events/v1/testdata/cloud_event_evaluation_completed_technical_capability.json new file mode 100644 index 0000000..ed83b3f --- /dev/null +++ b/proto/events/v1/testdata/cloud_event_evaluation_completed_technical_capability.json @@ -0,0 +1,50 @@ +{ + "spec_version": "1.0", + "id": "evt_eval_completed_technical_capability_1", + "type": "evaluation.completed", + "source": "fermata", + "subject": "product.evaluation.completed", + "time": "2026-04-13T12:00:00Z", + "data_content_type": "application/protobuf", + "tenant_id": "11111111-1111-1111-1111-111111111111", + "data": { + "@type": "type.googleapis.com/events.v1.EvaluationCompleted", + "signal_type": "technical_capability", + "summary": "Claude beats GPT-4 on latency by 40% on enterprise workloads.", + "success_rate": 0.9, + "company_domains": [ + "acme.com" + ], + "company_names": [ + "Acme Corporation" + ], + "deal_ids": [ + "deal-123" + ], + "run": { + "id": "run-1", + "test_suite_id": "suite-1", + "test_suite_name": "Latency benchmark", + "name": "Enterprise latency proof", + "description": "Claude beats GPT-4 on latency by 40% on enterprise workloads.", + "tags": [ + "pipeline:signal_type=technical_capability", + "pipeline:company_domain=acme.com", + "pipeline:company_name=Acme Corporation", + "pipeline:deal_id=deal-123" + ], + "completed_at": "2026-04-13T12:00:00Z" + }, + "metrics": { + "total_tests": "20", + "passed_tests": "18", + "failed_tests": "2", + "total_cost": 4.2, + "duration": 12.5, + "success_rate": 0.9 + } + }, + "extensions": { + "dataschema": "buf.build/evalops/proto/events.v1.EvaluationCompleted" + } +}