forked from redpanda-data/connect
-
Notifications
You must be signed in to change notification settings - Fork 0
/
execute.go
109 lines (96 loc) · 3.41 KB
/
execute.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
package processor
import (
"context"
"github.com/dafanshu/benthos/v4/internal/message"
)
// ExecuteAll attempts to execute a slice of processors to a message. Returns
// N resulting messages or a response. The response may indicate either a NoAck
// in the event of the message being buffered or an unrecoverable error.
func ExecuteAll(ctx context.Context, procs []V1, msgs ...message.Batch) ([]message.Batch, error) {
resultMsgs := make([]message.Batch, len(msgs))
copy(resultMsgs, msgs)
for i := 0; len(resultMsgs) > 0 && i < len(procs); i++ {
var nextResultMsgs []message.Batch
for _, m := range resultMsgs {
rMsgs, err := procs[i].ProcessBatch(ctx, m)
if err != nil {
// We immediately return if a processor hits an unrecoverable
// error on a message.
return nil, err
}
nextResultMsgs = append(nextResultMsgs, rMsgs...)
}
resultMsgs = nextResultMsgs
}
return resultMsgs, nil
}
// ExecuteTryAll attempts to execute a slice of processors to messages, if a
// message has failed a processing step it is prevented from being sent to
// subsequent processors. Returns N resulting messages or a response. The
// response may indicate either a NoAck in the event of the message being
// buffered or an unrecoverable error.
func ExecuteTryAll(ctx context.Context, procs []V1, msgs ...message.Batch) ([]message.Batch, error) {
resultMsgs := make([]message.Batch, len(msgs))
copy(resultMsgs, msgs)
for i := 0; len(resultMsgs) > 0 && i < len(procs); i++ {
var nextResultMsgs []message.Batch
for _, m := range resultMsgs {
// Skip messages that failed a prior stage.
if m.Get(0).ErrorGet() != nil {
nextResultMsgs = append(nextResultMsgs, m)
continue
}
rMsgs, err := procs[i].ProcessBatch(ctx, m)
if err != nil {
// We immediately return if a processor hits an unrecoverable
// error on a message.
return nil, err
}
nextResultMsgs = append(nextResultMsgs, rMsgs...)
}
resultMsgs = nextResultMsgs
}
return resultMsgs, nil
}
type catchMessage struct {
batches []message.Batch
caught bool
}
// ExecuteCatchAll attempts to execute a slice of processors to only messages
// that have failed a processing step. Returns N resulting messages or a
// response.
func ExecuteCatchAll(ctx context.Context, procs []V1, msgs ...message.Batch) ([]message.Batch, error) {
// Preserves the original order of messages before entering the catch block.
// Only processors that have failed a previous stage are "caught", and will
// remain caught until all catch processors are executed.
catchBatches := make([]catchMessage, len(msgs))
for i, m := range msgs {
catchBatches[i] = catchMessage{
batches: []message.Batch{m},
caught: m.Get(0).ErrorGet() != nil,
}
}
for i := 0; i < len(procs); i++ {
for j := 0; j < len(catchBatches); j++ {
if !catchBatches[j].caught || len(catchBatches[j].batches) == 0 {
continue
}
var nextResultBatches []message.Batch
for _, m := range catchBatches[j].batches {
rMsgs, resultRes := procs[i].ProcessBatch(ctx, m)
if resultRes != nil {
// We immediately return if a processor hits an unrecoverable
// error on a message.
return nil, resultRes
}
nextResultBatches = append(nextResultBatches, rMsgs...)
}
catchBatches[j].batches = nextResultBatches
}
}
var resultBatches []message.Batch
for _, b := range catchBatches {
resultBatches = append(resultBatches, b.batches...)
}
return resultBatches, nil
}