-
Notifications
You must be signed in to change notification settings - Fork 134
/
conversions.go
76 lines (66 loc) · 2.38 KB
/
conversions.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
package convert
import (
"context"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/armadaproject/armada/internal/common/compress"
"github.com/armadaproject/armada/internal/common/eventutil"
"github.com/armadaproject/armada/internal/common/ingest"
"github.com/armadaproject/armada/internal/common/ingest/metrics"
"github.com/armadaproject/armada/internal/eventingester/model"
)
// EventConverter converts event sequences into events that we can store in Redis
type EventConverter struct {
Compressor compress.Compressor
MaxMessageBatchSize uint
metrics *metrics.Metrics
}
func NewEventConverter(compressor compress.Compressor, maxMessageBatchSize uint, metrics *metrics.Metrics) ingest.InstructionConverter[*model.BatchUpdate] {
return &EventConverter{
Compressor: compressor,
MaxMessageBatchSize: maxMessageBatchSize,
metrics: metrics,
}
}
func (ec *EventConverter) Convert(ctx context.Context, sequencesWithIds *ingest.EventSequencesWithIds) *model.BatchUpdate {
// Remove all groups as they are potentially quite large
for _, es := range sequencesWithIds.EventSequences {
es.Groups = nil
}
sequences := eventutil.CompactEventSequences(sequencesWithIds.EventSequences)
sequences, err := eventutil.LimitSequencesByteSize(sequences, ec.MaxMessageBatchSize, false)
if err != nil {
// This should never happen. We pass strict=false to theabove sequence
panic(errors.WithMessage(err, "Failed to limit sequence by size"))
}
events := make([]*model.Event, 0)
for _, es := range sequences {
// Remove the jobset Name and the queue from the proto as this will be stored as the key
queue := es.Queue
jobset := es.JobSetName
es.JobSetName = ""
es.Queue = ""
bytes, err := proto.Marshal(es)
if err != nil {
ec.metrics.RecordPulsarMessageError(metrics.PulsarMessageErrorProcessing)
log.WithError(err).Warnf("Could not marshall proto for msg")
continue
}
compressedBytes, err := ec.Compressor.Compress(bytes)
if err != nil {
ec.metrics.RecordPulsarMessageError(metrics.PulsarMessageErrorProcessing)
log.WithError(err).Warnf("Could not compress event")
continue
}
events = append(events, &model.Event{
Queue: queue,
Jobset: jobset,
Event: compressedBytes,
})
}
return &model.BatchUpdate{
MessageIds: sequencesWithIds.MessageIds,
Events: events,
}
}