/
sqs.go
288 lines (253 loc) · 8.48 KB
/
sqs.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
// Package sqs is for messaging with AWS SQS.
package sqs
import (
"context"
"errors"
"fmt"
"math"
"os"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/retry"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
smithy "github.com/aws/smithy-go"
)
type Raw struct {
Body string
ReceiptHandle string
Attributes map[string]string
}
type SQS struct {
client *sqs.Client
}
// New returns an SQS struct which wraps an SQS client using the default AWS credentials chain.
// This consults (in order) environment vars, config files, EC2 and ECS roles.
// It is an error if the AWS_REGION environment variable is not set.
// Requests with recoverable errors will be retried with the default retrier.
func New() (SQS, error) {
cfg, err := getConfig()
if err != nil {
return SQS{}, err
}
return SQS{client: sqs.NewFromConfig(cfg)}, nil
}
// NewWithMaxRetries returns the same as New(), but with the
// back off set to up to maxRetries times.
func NewWithMaxRetries(maxRetries int) (SQS, error) {
cfg, err := getConfig()
if err != nil {
return SQS{}, err
}
client := sqs.NewFromConfig(cfg, func(options *sqs.Options) {
options.Retryer = retry.AddWithMaxAttempts(options.Retryer, maxRetries)
})
return SQS{client: client}, nil
}
// getConfig returns the default AWS Config struct.
func getConfig() (aws.Config, error) {
if os.Getenv("AWS_REGION") == "" {
return aws.Config{}, errors.New("AWS_REGION is not set")
}
var cfg aws.Config
var err error
if awsEndpoint := os.Getenv("CUSTOM_AWS_ENDPOINT_URL"); awsEndpoint != "" {
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
SigningRegion: region,
URL: awsEndpoint,
}, nil
})
cfg, err = config.LoadDefaultConfig(
context.TODO(),
config.WithEndpointResolverWithOptions(customResolver))
} else {
cfg, err = config.LoadDefaultConfig(context.TODO())
}
if err != nil {
return aws.Config{}, err
}
return cfg, nil
}
// Ready returns whether the SQS client has been initialised.
func (s *SQS) Ready() bool {
return s.client != nil
}
// Receive receives a raw message or error from the queue.
// After a successful receive the message will be in flight
// until it is either deleted or the visibility timeout expires
// (at which point it is available for redelivery).
//
// Applications should be able to handle duplicate or out of order messages,
// and should back off on Receive error.
func (s *SQS) Receive(queueURL string, visibilityTimeout int32) (Raw, error) {
return s.ReceiveWithContext(context.TODO(), queueURL, visibilityTimeout)
}
// ReceiveWithAttributes is the same as Receive except that Queue Attributes can be requested
// to be received with the message.
func (s *SQS) ReceiveWithAttributes(queueURL string, visibilityTimeout int32, attrs []types.QueueAttributeName) (Raw, error) {
return s.ReceiveWithContextAttributes(context.TODO(), queueURL, visibilityTimeout, attrs)
}
// ReceiveWithContextAttributes by context and Queue Attributes,
// so that system stop signal can be received by the context.
// to receive system stop signal, register the context with signal.NotifyContext before passing in this function,
// when system stop signal is received, an error with message '... context canceled' will be returned
// which can be used to safely stop the system
func (s *SQS) ReceiveWithContextAttributes(ctx context.Context, queueURL string, visibilityTimeout int32, attrs []types.QueueAttributeName) (Raw, error) {
input := sqs.ReceiveMessageInput{
QueueUrl: aws.String(queueURL),
MaxNumberOfMessages: 1,
VisibilityTimeout: visibilityTimeout,
WaitTimeSeconds: 20,
AttributeNames: attrs,
}
return s.receiveMessage(ctx, &input)
}
// receiveMessage is the common code used internally to receive an SQS message based
// on the provided input.
func (s *SQS) receiveMessage(ctx context.Context, input *sqs.ReceiveMessageInput) (Raw, error) {
for {
r, err := s.client.ReceiveMessage(ctx, input)
if err != nil {
return Raw{}, err
}
switch {
case r == nil || len(r.Messages) == 0:
// no message received
continue
case len(r.Messages) == 1:
raw := r.Messages[0]
m := Raw{
Body: aws.ToString(raw.Body),
ReceiptHandle: aws.ToString(raw.ReceiptHandle),
Attributes: raw.Attributes,
}
return m, nil
case len(r.Messages) > 1:
return Raw{}, fmt.Errorf("received more than 1 message: %d", len(r.Messages))
}
}
}
// receive with context so that system stop signal can be received,
// to receive system stop signal, register the context with signal.NotifyContext before passing in this function,
// when system stop signal is received, an error with message '... context canceled' will be returned
// which can be used to safely stop the system
func (s *SQS) ReceiveWithContext(ctx context.Context, queueURL string, visibilityTimeout int32) (Raw, error) {
input := sqs.ReceiveMessageInput{
QueueUrl: aws.String(queueURL),
MaxNumberOfMessages: 1,
VisibilityTimeout: visibilityTimeout,
WaitTimeSeconds: 20,
}
return s.receiveMessage(ctx, &input)
}
// Delete deletes the message referred to by receiptHandle from the queue.
func (s *SQS) Delete(queueURL, receiptHandle string) error {
params := sqs.DeleteMessageInput{
QueueUrl: aws.String(queueURL),
ReceiptHandle: aws.String(receiptHandle),
}
_, err := s.client.DeleteMessage(context.TODO(), ¶ms)
return err
}
// Send sends the message body to the SQS queue referred to by queueURL.
func (s *SQS) Send(queueURL string, body string) error {
params := sqs.SendMessageInput{
QueueUrl: aws.String(queueURL),
MessageBody: aws.String(body),
}
_, err := s.client.SendMessage(context.TODO(), ¶ms)
return err
}
// SendWithDelay is the same as Send but adds a delay (in seconds) before sending.
func (s *SQS) SendWithDelay(queueURL string, body string, delay int32) error {
params := sqs.SendMessageInput{
QueueUrl: aws.String(queueURL),
MessageBody: aws.String(body),
DelaySeconds: delay,
}
_, err := s.client.SendMessage(context.TODO(), ¶ms)
return err
}
// SendFifoMessage puts a message onto the given AWS SQS queue.
func (s *SQS) SendFifoMessage(queue, group, dedupe string, msg []byte) (string, error) {
var id *string
if dedupe != "" {
id = aws.String(dedupe)
}
params := sqs.SendMessageInput{
MessageBody: aws.String(string(msg)),
QueueUrl: aws.String(queue),
MessageGroupId: aws.String(group),
MessageDeduplicationId: id,
}
output, err := s.client.SendMessage(context.TODO(), ¶ms)
if err != nil {
return "", err
}
if id := output.MessageId; id != nil {
return *id, nil
}
return "", nil
}
// Leverage the sendbatch api for uploading large numbers of messages
func (s *SQS) SendBatch(ctx context.Context, queueURL string, bodies []string) error {
if len(bodies) > 11 {
return errors.New("too many messages to batch")
}
var err error
entries := make([]types.SendMessageBatchRequestEntry, len(bodies))
for j, body := range bodies {
entries[j] = types.SendMessageBatchRequestEntry{
Id: aws.String(fmt.Sprintf("gamitjob%d", j)),
MessageBody: aws.String(body),
}
}
_, err = s.client.SendMessageBatch(ctx, &sqs.SendMessageBatchInput{
Entries: entries,
QueueUrl: &queueURL,
})
return err
}
func (s *SQS) SendNBatch(ctx context.Context, queueURL string, bodies []string) error {
var (
bodiesLen = len(bodies)
maxlen = 10
times = int(math.Ceil(float64(bodiesLen) / float64(maxlen)))
)
for i := 0; i < times; i++ {
batch_end := maxlen * (i + 1)
if maxlen*(i+1) > bodiesLen {
batch_end = bodiesLen
}
var bodies_batch = bodies[maxlen*i : batch_end]
err := s.SendBatch(ctx, queueURL, bodies_batch)
if err != nil {
return err
}
}
return nil
}
// GetQueueUrl returns an AWS SQS queue URL given its name.
func (s *SQS) GetQueueUrl(name string) (string, error) {
params := sqs.GetQueueUrlInput{
QueueName: aws.String(name),
}
output, err := s.client.GetQueueUrl(context.TODO(), ¶ms)
if err != nil {
return "", err
}
if url := output.QueueUrl; url != nil {
return aws.ToString(url), nil
}
return "", nil
}
func Cancelled(err error) bool {
var opErr *smithy.OperationError
if errors.As(err, &opErr) {
return opErr.Service() == "SQS" && strings.Contains(opErr.Unwrap().Error(), "context canceled")
}
return false
}