forked from blox/blox
/
kinesis_consumer.go
103 lines (92 loc) · 3.03 KB
/
kinesis_consumer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
// Copyright 2016-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.
package event
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/kinesis"
"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface"
log "github.com/cihub/seelog"
"github.com/pkg/errors"
"golang.org/x/net/context"
"time"
)
const (
kinesisWaitTimeSeconds = 10
kinesisStartingShardId = "shardId-000000000000"
kinesisShardIteratorType = kinesis.ShardIteratorTypeTrimHorizon
kinesisGetRecordsSize = 100
)
type kinesisEventConsumer struct {
kinesis kinesisiface.KinesisAPI
streamName string
processor Processor
iterator *string
}
func NewKinesisConsumer(kinesis kinesisiface.KinesisAPI, processor Processor, streamName string) (Consumer, error) {
if kinesis == nil {
return nil, errors.Errorf("The Kinesis API interface is not initialized")
}
if processor == nil {
return nil, errors.Errorf("The event processor is not initialized")
}
if streamName == "" {
return nil, errors.Errorf("The Kinesis stream name is empty")
}
return &kinesisEventConsumer{
kinesis: kinesis,
streamName: streamName,
processor: processor,
}, nil
}
func (kinesisConsumer *kinesisEventConsumer) PollForEvents(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
default:
kinesisConsumer.pollForMessages()
}
}
}
func (kinesisConsumer *kinesisEventConsumer) pollForMessages() {
if kinesisConsumer.iterator == nil {
iteratorRequest := &kinesis.GetShardIteratorInput{
ShardId: aws.String(kinesisStartingShardId),
ShardIteratorType: aws.String(kinesisShardIteratorType),
StreamName: aws.String(kinesisConsumer.streamName),
}
iteratorResponse, err := kinesisConsumer.kinesis.GetShardIterator(iteratorRequest)
if err != nil {
log.Errorf("%+v", errors.Wrapf(err, "Could not get shard iterator"))
return
}
kinesisConsumer.iterator = iteratorResponse.ShardIterator
}
recordsRequest := &kinesis.GetRecordsInput{
Limit: aws.Int64(kinesisGetRecordsSize),
ShardIterator: kinesisConsumer.iterator,
}
recordsResponse, err := kinesisConsumer.kinesis.GetRecords(recordsRequest)
if err != nil {
log.Errorf("%+v", errors.Wrapf(err, "Unable to get records from kinesis"))
kinesisConsumer.iterator = nil
return
}
for _, record := range recordsResponse.Records {
kinesisConsumer.processor.ProcessEvent(string(record.Data[:]))
}
kinesisConsumer.iterator = recordsResponse.NextShardIterator
if len(recordsResponse.Records) == 0 {
time.Sleep(kinesisWaitTimeSeconds * time.Second)
}
}