/
kinesis.go
94 lines (78 loc) · 2.95 KB
/
kinesis.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package aws
import (
"context"
"math"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/kinesis"
"github.com/aws/aws-sdk-go-v2/service/kinesis/types"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
kinesisv1 "github.com/lyft/clutch/backend/api/aws/kinesis/v1"
)
func (c *client) DescribeKinesisStream(ctx context.Context, account, region, streamName string) (*kinesisv1.Stream, error) {
cl, err := c.getAccountRegionClient(account, region)
if err != nil {
return nil, err
}
input := &kinesis.DescribeStreamSummaryInput{StreamName: aws.String(streamName)}
result, err := cl.kinesis.DescribeStreamSummary(ctx, input)
if err != nil {
return nil, err
}
currentShardCount := aws.ToInt32(result.StreamDescriptionSummary.OpenShardCount)
if currentShardCount < 0 {
return nil, status.Error(codes.Internal, "AWS returned a negative value for the current shard count")
}
var ret = &kinesisv1.Stream{
StreamName: aws.ToString(result.StreamDescriptionSummary.StreamName),
Account: account,
Region: region,
CurrentShardCount: aws.ToInt32(result.StreamDescriptionSummary.OpenShardCount),
}
return ret, nil
}
// we limit the resizing of shard sizes to increments of 25% ranging from [50%, 200%] of the current shard count
// https://docs.aws.amazon.com/kinesis/latest/APIReference/API_UpdateShardCount.html
func getRecommendedShardSizes(currentShardCount int32) map[int32]bool {
sizes := make(map[int32]bool)
base := 0.5
for i := 0; i < 7; i++ {
recommendedValue := int32(math.Ceil(float64(currentShardCount) * base))
sizes[recommendedValue] = true
base += 0.25
}
return sizes
}
func isRecommendedChange(stream *kinesis.DescribeStreamSummaryOutput, targetShardCount int32) error {
currentShardCount := aws.ToInt32(stream.StreamDescriptionSummary.OpenShardCount)
if currentShardCount < 0 {
return status.Error(codes.Internal, "AWS returned a negative value for the current shard count")
}
recommendedValues := getRecommendedShardSizes(currentShardCount)
if !recommendedValues[targetShardCount] {
return status.Error(codes.FailedPrecondition, "new shard count should be a 25% increment of current shard count ranging from 50-200%")
}
return nil
}
func (c *client) UpdateKinesisShardCount(ctx context.Context, account, region, streamName string, targetShardCount int32) error {
cl, err := c.getAccountRegionClient(account, region)
if err != nil {
return err
}
describeInput := &kinesis.DescribeStreamSummaryInput{StreamName: aws.String(streamName)}
result, err := cl.kinesis.DescribeStreamSummary(ctx, describeInput)
if err != nil {
return err
}
err = isRecommendedChange(result, targetShardCount)
if err != nil {
return err
}
input := &kinesis.UpdateShardCountInput{
StreamName: aws.String(streamName),
TargetShardCount: aws.Int32(targetShardCount),
ScalingType: types.ScalingTypeUniformScaling,
}
_, err = cl.kinesis.UpdateShardCount(ctx, input)
return err
}