-
Notifications
You must be signed in to change notification settings - Fork 40
/
KinesisSourceStage.cs
225 lines (190 loc) · 7.35 KB
/
KinesisSourceStage.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
#region copyright
//-----------------------------------------------------------------------
// <copyright file="KinesisSourceStage.cs" company="Akka.NET Project">
// Copyright (C) 2009-2018 Lightbend Inc. <http://www.lightbend.com>
// Copyright (C) 2013-2018 .NET Foundation <https://github.com/akkadotnet/akka.net>
// </copyright>
//-----------------------------------------------------------------------
#endregion
using System;
using System.Collections.Generic;
using Akka.Actor;
using Akka.Streams.Stage;
using Amazon.Kinesis;
using Amazon.Kinesis.Model;
namespace Akka.Streams.Kinesis
{
internal sealed class KinesisSourceStage : GraphStage<SourceShape<Record>>
{
#region messages
sealed class GetShardIteratorSuccess
{
public readonly GetShardIteratorResponse Response;
public GetShardIteratorSuccess(GetShardIteratorResponse response)
{
Response = response;
}
}
sealed class GetShardIteratorFailure
{
public readonly Exception Cause;
public GetShardIteratorFailure(Exception cause)
{
Cause = cause;
}
}
sealed class GetRecordsSuccess
{
public readonly GetRecordsResponse Response;
public GetRecordsSuccess(GetRecordsResponse response)
{
Response = response;
}
}
sealed class GetRecordsFailure
{
public readonly Exception Cause;
public GetRecordsFailure(Exception cause)
{
Cause = cause;
}
}
sealed class Pump
{
public static readonly Pump Instance = new Pump();
private Pump() { }
}
#endregion
#region logic
sealed class Logic : TimerGraphStageLogic
{
private readonly ShardSettings _settings;
private readonly Outlet<Record> _out;
private readonly IAmazonKinesis _kinesisClient;
private string _iterator;
private readonly Queue<Record> _buffer = new Queue<Record>();
private StageActor _actor;
private IActorRef _self;
public Logic(KinesisSourceStage stage) : base(stage.Shape)
{
_settings = stage.Settings;
_out = stage.Outlet;
_kinesisClient = stage._clientFactory();
SetHandler(_out, onPull: () => _self.Tell(Pump.Instance));
}
public override void PreStart()
{
_actor = GetStageActor(AwaitingShardIterator);
_self = _actor.Ref;
GetShardIterator();
}
private void GetShardIterator()
{
var request = new GetShardIteratorRequest
{
ShardId = _settings.ShardId,
StreamName = _settings.StreamName,
ShardIteratorType = _settings.ShardIteratorType
};
if (_settings.AtTimestamp.HasValue)
request.Timestamp = _settings.AtTimestamp.Value;
_kinesisClient.GetShardIteratorAsync(request)
.PipeTo(_self,
success: result => new GetShardIteratorSuccess(result),
failure: ex => new GetShardIteratorFailure(ex));
}
private void AwaitingShardIterator((IActorRef actorRef, object message) args)
{
switch (args.message)
{
case GetShardIteratorSuccess s:
_iterator = s.Response.ShardIterator;
_actor.Become(AwaitingRecords);
RequestRecords(_self);
break;
case GetShardIteratorFailure f:
Log.Error(f.Cause, "Failed to get a shard iterator for shard {0}", _settings.ShardId);
FailStage(f.Cause);
break;
}
}
private void RequestRecords(IActorRef self)
{
_kinesisClient.GetRecordsAsync(new GetRecordsRequest
{
Limit = _settings.Limit,
ShardIterator = _iterator
}).PipeTo(self,
success: result => new GetRecordsSuccess(result),
failure: ex => new GetRecordsFailure(ex));
}
private void AwaitingRecords((IActorRef actorRef, object message) args)
{
switch (args.message)
{
case GetRecordsSuccess s:
var records = s.Response.Records;
if (string.IsNullOrEmpty(s.Response.NextShardIterator))
{
Log.Info("Shard {0} returned a null iterator and will now complete.", _settings.ShardId);
CompleteStage();
}
else
{
_iterator = s.Response.NextShardIterator;
}
if (records.Count > 0)
{
foreach (var record in records)
_buffer.Enqueue(record);
_actor.Become(Ready);
_self.Tell(Pump.Instance);
}
else
{
ScheduleOnce("GET_RECORDS", _settings.RefreshInterval);
}
break;
case GetRecordsFailure f:
Log.Error(f.Cause, "Failed to fetch records from Kinesis for shard {0}", _settings.ShardId);
FailStage(f.Cause);
break;
}
}
private void Ready((IActorRef actorRef, object message) args)
{
if (args.message is Pump)
{
if (IsAvailable(_out))
{
Push(_out, _buffer.Dequeue());
_self.Tell(Pump.Instance);
}
if (_buffer.Count == 0)
{
_actor.Become(AwaitingRecords);
RequestRecords(_self);
}
}
}
protected override void OnTimer(object timerKey)
{
if ("GETRECORDS" == (string)timerKey)
RequestRecords(_self);
}
}
#endregion
public ShardSettings Settings { get; }
private readonly Func<IAmazonKinesis> _clientFactory;
public KinesisSourceStage(ShardSettings settings, Func<IAmazonKinesis> clientFactory)
{
Settings = settings;
_clientFactory = clientFactory;
Shape = new SourceShape<Record>(Outlet);
}
public Outlet<Record> Outlet { get; } = new Outlet<Record>("Records.out");
public override SourceShape<Record> Shape { get; }
protected override GraphStageLogic CreateLogic(Attributes inheritedAttributes) =>
new Logic(this);
}
}