-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Vladimir Dubov
committed
Jun 3, 2019
0 parents
commit 18f4e06
Showing
8 changed files
with
501 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # builder | ||
| FROM golang:1.12.5-alpine3.9 AS builder | ||
|
|
||
| ENV GO111MODULE=on | ||
|
|
||
| WORKDIR /go/src/sqs_exporter | ||
|
|
||
| COPY ./* /go/src/sqs_exporter/ | ||
|
|
||
| RUN apk --no-cache add git | ||
| RUN go mod download | ||
| RUN CGO_ENABLED=0 GOOS=linux go build -o sqs_prom | ||
|
|
||
| # final | ||
| FROM alpine:3.9 | ||
|
|
||
| RUN apk --no-cache add ca-certificates | ||
|
|
||
| WORKDIR /root/ | ||
|
|
||
| COPY --from=builder /go/src/sqs_exporter/sqs_prom . | ||
|
|
||
| EXPOSE 9108 | ||
|
|
||
| CMD ["./sqs_prom"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| # AWS SQS exporter | ||
|
|
||
| A Prometheus SQS metrics exporter | ||
|
|
||
| ## Metrics | ||
|
|
||
| | Metric | Description | | ||
| | ------ | ----------- | | ||
| | aws\_sqs\_approximate\_number\_of\_messages | Number of messages available | | ||
| | aws\_sqs\_approximate\_number\_of\_messages\_delayed | Number of messages delayed | | ||
| | aws\_sqs\_approximate\_number\_of\_messages\_not\_visible | Number of messages in flight | | ||
|
|
||
| ## Lables | ||
|
|
||
| - queue name | ||
| - tags - you can disable with flag | ||
|
|
||
| For more information see the [AWS SQS Documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-attributes.html) | ||
|
|
||
| ## Configuration | ||
|
|
||
| Flags: | ||
|
|
||
| - interval - how often to update queue list, env INTERVAL | ||
| - prefix - filter queue list by prefix (filtered on AWS API side), env PREFIX | ||
| - regex - filter queues by regex (filtered in app), env REGEX | ||
| - tags - add tags as lables, env TAGS | ||
|
|
||
| Credentials to AWS are provided in the following order: | ||
|
|
||
| - Environment variables (AWS\_ACCESS\_KEY\_ID and AWS\_SECRET\_ACCESS\_KEY) | ||
| - Shared credentials file (~/.aws/credentials) | ||
| - IAM role for Amazon EC2 | ||
|
|
||
| For more information see the [AWS SDK Documentation](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) | ||
|
|
||
| ### AWS IAM permissions | ||
|
|
||
| The app needs sqs list and read access to the sqs policies | ||
|
|
||
| ```json | ||
| { | ||
| "Version": "2012-10-17", | ||
| "Statement": [ | ||
| { | ||
| "Sid": "VisualEditor0", | ||
| "Effect": "Allow", | ||
| "Action": [ | ||
| "sqs:GetQueueAttributes", | ||
| "sqs:GetQueueUrl", | ||
| "sqs:ListDeadLetterSourceQueues", | ||
| "sqs:ListQueueTags" | ||
| "sqs:ListQueues", | ||
| ], | ||
| "Resource": "*" | ||
| } | ||
| ] | ||
| } | ||
| ``` | ||
|
|
||
| ## Running | ||
|
|
||
| **You need to specify the region you to connect to** | ||
| Running on an ec2 machine using IAM roles: | ||
| `docker run -e AWS_REGION=<region> -d -p 9108:9108 sqs-exporter` | ||
|
|
||
| Or running it externally: | ||
| `docker run -d -p 9108:9108 -e AWS_ACCESS_KEY_ID=<access_key> -e AWS_SECRET_ACCESS_KEY=<secret_key> -e AWS_REGION=<region> sqs-exporter` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log" | ||
| "regexp" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/sqs" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| ) | ||
|
|
||
| type collector struct { | ||
| client *sqs.SQS | ||
| mutex *sync.Mutex | ||
|
|
||
| queues []*queue | ||
| attributes []*string | ||
|
|
||
| metrics []*queueMetric | ||
| totalScrapes prometheus.Counter | ||
| } | ||
|
|
||
| func newCollector(ctx context.Context, updateInterval time.Duration, prefix *string, regex *regexp.Regexp, tagsAsLables bool) *collector { | ||
| sess := session.Must(session.NewSession()) | ||
|
|
||
| c := &collector{ | ||
| client: sqs.New(sess), | ||
| mutex: &sync.Mutex{}, | ||
|
|
||
| metrics: metrics, | ||
| attributes: buildAttributes(), | ||
|
|
||
| totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ | ||
| Name: prometheus.BuildFQName(namespace, subsystem, "total_scrapes"), | ||
| Help: "Current total AWS SQS scrapes.", | ||
| }), | ||
| } | ||
|
|
||
| go c.queueListUpdater(ctx, updateInterval, queuePrefix, regex, tagsAsLables) | ||
|
|
||
| return c | ||
| } | ||
|
|
||
| func (c *collector) Describe(ch chan<- *prometheus.Desc) { | ||
| for _, metric := range c.metrics { | ||
| ch <- generateDesc(metric.name, metric.help, nil, nil) | ||
| } | ||
| ch <- c.totalScrapes.Desc() | ||
| } | ||
|
|
||
| func (c *collector) Collect(ch chan<- prometheus.Metric) { | ||
| c.totalScrapes.Inc() | ||
| defer func() { | ||
| ch <- c.totalScrapes | ||
| }() | ||
|
|
||
| log.Printf("Collecting metrics for %d queues", len(c.queues)) | ||
|
|
||
| for _, q := range c.queues { | ||
| lables := []string{"name"} | ||
| values := []string{q.name} | ||
|
|
||
| for key, value := range q.tags { | ||
| lables = append(lables, fmt.Sprintf("tag_%s", key)) | ||
| values = append(values, *value) | ||
| } | ||
|
|
||
| if err := q.getQueueAttributes(c.client, c.attributes); err != nil { | ||
| log.Printf("Failed to get queue attributes: %v", err) | ||
| continue | ||
| } | ||
|
|
||
| for _, m := range c.metrics { | ||
| ch <- prometheus.MustNewConstMetric( | ||
| generateDesc(m.name, m.help, lables, nil), | ||
| m.Type, | ||
| q.getAttributeValue(m.attribute), | ||
| values..., | ||
| ) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (c *collector) queueListUpdater(ctx context.Context, updateInverval time.Duration, prefix *string, regex *regexp.Regexp, tagsAsLables bool) { | ||
| log.Printf("Start queue list updater") | ||
|
|
||
| queueListInput := &sqs.ListQueuesInput{} | ||
|
|
||
| if len(*prefix) > 0 { | ||
| queueListInput.QueueNamePrefix = prefix | ||
| } | ||
|
|
||
| f := func() error { | ||
| result, err := c.client.ListQueues(queueListInput) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var tmpQueues []*queue | ||
|
|
||
| for _, q := range result.QueueUrls { | ||
| queueName := getQueueName(*q) | ||
| if !regex.MatchString(queueName) { | ||
| continue | ||
| } | ||
|
|
||
| tmpQ := &queue{name: queueName, url: q} | ||
|
|
||
| if tagsAsLables { | ||
| tagsInput := &sqs.ListQueueTagsInput{ | ||
| QueueUrl: q, | ||
| } | ||
|
|
||
| tagsOutput, err := c.client.ListQueueTags(tagsInput) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| tmpQ.tags = tagsOutput.Tags | ||
| } | ||
| tmpQueues = append(tmpQueues, tmpQ) | ||
| } | ||
|
|
||
| log.Printf("Found %d queues", len(tmpQueues)) | ||
|
|
||
| c.mutex.Lock() | ||
| c.queues = tmpQueues | ||
| c.mutex.Unlock() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| if err := f(); err != nil { | ||
| log.Fatalf("Failed to get queue list: %v", err) | ||
| } | ||
|
|
||
| innerTicker := time.NewTicker(updateInverval) | ||
| defer innerTicker.Stop() | ||
|
|
||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return | ||
| case <-innerTicker.C: | ||
| if err := f(); err != nil { | ||
| log.Fatalf("Failed to get queue list: %v", err) | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| module sqs-exporter | ||
|
|
||
| go 1.12 | ||
|
|
||
| require ( | ||
| github.com/aws/aws-sdk-go v1.19.33 | ||
| github.com/prometheus/client_golang v0.9.3 | ||
| golang.org/x/text v0.3.2 // indirect | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= | ||
| github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= | ||
| github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= | ||
| github.com/aws/aws-sdk-go v1.19.33 h1:qz9ZQtxCUuwBKdc5QiY6hKuISYGeRQyLVA2RryDEDaQ= | ||
| github.com/aws/aws-sdk-go v1.19.33/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= | ||
| github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= | ||
| github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= | ||
| github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= | ||
| github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= | ||
| github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= | ||
| github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
| github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= | ||
| github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= | ||
| github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= | ||
| github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= | ||
| github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= | ||
| github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= | ||
| github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= | ||
| github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | ||
| github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= | ||
| github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | ||
| github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= | ||
| github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= | ||
| github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= | ||
| github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= | ||
| github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= | ||
| github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= | ||
| github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= | ||
| github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= | ||
| github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= | ||
| github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | ||
| github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
| github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= | ||
| github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= | ||
| github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= | ||
| github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= | ||
| github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= | ||
| github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= | ||
| github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= | ||
| github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= | ||
| github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= | ||
| github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= | ||
| github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= | ||
| github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= | ||
| github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= | ||
| github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= | ||
| github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= | ||
| github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
| github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= | ||
| github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= | ||
| golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | ||
| golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= | ||
| golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
| golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
| golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
| golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||
| golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||
| golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||
| golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= | ||
| golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= | ||
| golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= | ||
| gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= | ||
| gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | ||
| gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
Oops, something went wrong.