-
Notifications
You must be signed in to change notification settings - Fork 42
/
timeouts.go
95 lines (77 loc) · 2.97 KB
/
timeouts.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
package flow
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"time"
)
func (engine *engine) scheduleTimeout(_ context.Context, im *instanceMemory, oldController string, t time.Time, soft bool) {
var err error
deadline := t
prefixes := []string{"soft", "hard"}
prefix := prefixes[1]
if soft {
prefix = prefixes[0]
}
oldId := fmt.Sprintf("timeout:%s:%s:%d", im.ID().String(), prefix, im.Step()-1)
id := fmt.Sprintf("timeout:%s:%s:%d", im.ID().String(), prefix, im.Step())
if im.Step() == 0 {
id = fmt.Sprintf("timeout:%s:%s", im.ID().String(), prefix)
}
// cancel existing timeouts
slog.Debug("Cancelling existing timeouts.", "namespace", im.Namespace(), "instance", im.ID(), "timeout_type", prefix, "step", im.Step(), "error", err)
engine.timers.deleteTimerByName(oldController, engine.pubsub.Hostname, oldId)
engine.timers.deleteTimerByName(oldController, engine.pubsub.Hostname, id)
// schedule timeout
args := &timeoutArgs{
InstanceId: im.ID().String(),
Step: im.Step(),
Soft: soft,
}
data, err := json.Marshal(args)
if err != nil {
panic(err)
}
err = engine.timers.addOneShot(id, timeoutFunction, deadline, data)
if err != nil {
slog.Error("Failed to schedule a timeout.", "namespace", im.Namespace(), "instance", im.ID(), "timeout_type", prefix, "step", im.Step(), "error", err)
} else {
slog.Debug("Successfully scheduled a new timeout.", "namespace", im.Namespace(), "instance", im.ID(), "timeout_type", prefix, "step", im.Step(), "error", err)
}
}
func (engine *engine) ScheduleHardTimeout(ctx context.Context, im *instanceMemory, oldController string, t time.Time) {
engine.scheduleTimeout(ctx, im, oldController, t, false)
}
func (engine *engine) ScheduleSoftTimeout(ctx context.Context, im *instanceMemory, oldController string, t time.Time) {
engine.scheduleTimeout(ctx, im, oldController, t, true)
}
type timeoutArgs struct {
InstanceId string
Step int
Soft bool
}
const timeoutFunction = "timeoutFunction"
func (engine *engine) hardCancelInstance(instanceId, code, message string) {
engine.cancelInstance(instanceId, code, message, false)
}
func (engine *engine) softCancelInstance(instanceId string, code, message string) {
engine.cancelInstance(instanceId, code, message, true)
}
func (engine *engine) timeoutHandler(input []byte) {
args := new(timeoutArgs)
err := json.Unmarshal(input, args)
if err != nil {
slog.Error("Failed to unmarshal timeout handler arguments.", "error", err)
return
}
if args.Soft {
slog.Error("Initiating soft cancellation due to timeout.", "instance", args.InstanceId)
engine.softCancelInstance(args.InstanceId, ErrCodeSoftTimeout, "operation timed out")
slog.Error("Soft cancellation complete.", "instance", args.InstanceId)
} else {
slog.Error("Initiating hard cancellation due to timeout.", "instance", args.InstanceId)
engine.hardCancelInstance(args.InstanceId, ErrCodeHardTimeout, "workflow timed out")
slog.Error("Hard cancellation complete.", "instance", args.InstanceId)
}
}