Add endpoint to AWS configs

benthosdev · Sep 29, 2018 · e56253b · e56253b
1 parent b9f013d
commit e56253b
Show file tree

Hide file tree

Showing 32 changed files with 344 additions and 443 deletions.
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -34,27 +34,4 @@ release:
   github:
     owner: Jeffail
     name: benthos
-  disable: false
-dockers:
-  -
-    image: jeffail/benthos
-    goos: linux
-    goarch: amd64
-    binary: benthos
-    skip_push: false
-    dockerfile: ./resources/docker/Dockerfile
-    tag_templates:
-      - "v{{ .Version }}"
-      - "v{{ .Major }}"
-      - "v{{ .Major }}.{{ .Minor }}"
-      - latest
-    extra_files:
-      - .git
-      - cmd
-      - config
-      - go.mod
-      - go.sum
-      - lib
-      - LICENSE
-      - Makefile
-      - vendor
+  disable: false
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: all deps rpm docker clean docs test test-race test-integration fmt lint install docker-zmq
+.PHONY: all deps rpm docker docker-deps docker-zmq docker-push clean docs test test-race test-integration fmt lint install
 
 TAGS =
 
@@ -7,14 +7,17 @@ DEST_DIR       = ./target
 PATHINSTBIN    = $(DEST_DIR)/bin
 PATHINSTDOCKER = $(DEST_DIR)/docker
 
-VERSION := $(shell git describe --tags || echo "v0.0.0")
-DATE    := $(shell date +"%Y-%m-%dT%H:%M:%SZ")
+VERSION   := $(shell git describe --tags || echo "v0.0.0")
+VER_MAJOR := $(shell echo $(VERSION) | cut -f1 -d.)
+VER_MINOR := $(shell echo $(VERSION) | cut -f2 -d.)
+VER_PATCH := $(shell echo $(VERSION) | cut -f3 -d.)
+DATE      := $(shell date +"%Y-%m-%dT%H:%M:%SZ")
 
 VER_FLAGS = -X main.Version=$(VERSION) \
 	-X main.DateBuilt=$(DATE)
 
 LD_FLAGS =
-GO_FLAGS = -mod=vendor
+GO_FLAGS =
 
 APPS = benthos
 all: $(APPS)
@@ -29,17 +32,24 @@ $(PATHINSTBIN)/%: $(wildcard lib/*/*.go lib/*/*/*.go lib/*/*/*/*.go cmd/*/*.go)
 $(APPS): %: $(PATHINSTBIN)/%
 
 docker:
-	@docker rmi jeffail/benthos:$(VERSION); true
 	@docker build -f ./resources/docker/Dockerfile . -t jeffail/benthos:$(VERSION)
-	@docker rmi jeffail/benthos:latest; true
+	@docker tag jeffail/benthos:$(VERSION) jeffail/benthos:$(VER_MAJOR)
+	@docker tag jeffail/benthos:$(VERSION) jeffail/benthos:$(VER_MAJOR).$(VER_MINOR)
 	@docker tag jeffail/benthos:$(VERSION) jeffail/benthos:latest
 
+docker-deps:
+	@docker build -f ./resources/docker/Dockerfile --target deps . -t jeffail/benthos:$(VERSION)-deps
+
 docker-zmq:
-	@docker rmi jeffail/benthos:$(VERSION)-zmq; true
 	@docker build -f ./resources/docker/Dockerfile.zmq . -t jeffail/benthos:$(VERSION)-zmq
 
-deps:
-	@go mod vendor
+docker-push:
+	@docker push jeffail/benthos:$(VERSION)-deps; true
+	@docker push jeffail/benthos:$(VERSION)-zmq; true
+	@docker push jeffail/benthos:$(VERSION); true
+	@docker push jeffail/benthos:$(VER_MAJOR); true
+	@docker push jeffail/benthos:$(VER_MAJOR).$(VER_MINOR); true
+	@docker push jeffail/benthos:latest; true
 
 fmt:
 	@go list -f {{.Dir}} ./... | xargs -I{} gofmt -w -s {}
@@ -55,7 +65,7 @@ test-race:
 	@go test $(GO_FLAGS) -short -race ./...
 
 test-integration:
-	@go test $(GO_FLAGS) -timeout 300s ./...
+	@go test $(GO_FLAGS) -timeout 600s ./...
 
 clean:
 	rm -rf $(PATHINSTBIN)

diff --git a/README.md b/README.md
@@ -123,21 +123,22 @@ with [a config file][env-config] where _all_ common fields can be set this way.
 
 ## Install
 
-Build with Go (1.11 or later):
+Grab a binary for your OS from [here.][releases]
+
+Or pull the docker image:
 
 ``` shell
-make deps
-make
+docker pull jeffail/benthos
 ```
 
-Or, pull the docker image:
+Build with Go (1.11 or later):
 
 ``` shell
-docker pull jeffail/benthos
+git clone git@github.com:Jeffail/benthos
+cd benthos
+make
 ```
 
-Or, [grab a binary for your OS from here.][releases]
-
 ### Docker Builds
 
 There's a multi-stage `Dockerfile` for creating a Benthos docker image which
@@ -169,6 +170,12 @@ install libzmq4 and use the compile time flag when building Benthos:
 make TAGS=ZMQ4
 ```
 
+Or to build a docker image with ZMQ support:
+
+``` shell
+make docker-zmq
+```
+
 ## Contributing
 
 Contributions are welcome, please [read the guidelines](CONTRIBUTING.md).

diff --git a/config/env/README.md b/config/env/README.md
@@ -190,6 +190,7 @@ INPUT_S3_CREDENTIALS_ROLE
 INPUT_S3_CREDENTIALS_SECRET
 INPUT_S3_CREDENTIALS_TOKEN
 INPUT_S3_DELETE_OBJECTS                     = false
+INPUT_S3_ENDPOINT
 INPUT_S3_PREFIX
 INPUT_S3_REGION                             = eu-west-1
 INPUT_S3_RETRIES                            = 3
@@ -202,6 +203,7 @@ INPUT_SQS_CREDENTIALS_ID
 INPUT_SQS_CREDENTIALS_ROLE
 INPUT_SQS_CREDENTIALS_SECRET
 INPUT_SQS_CREDENTIALS_TOKEN
+INPUT_SQS_ENDPOINT
 INPUT_SQS_REGION                            = eu-west-1
 INPUT_SQS_TIMEOUT_S                         = 5
 INPUT_SQS_URL
@@ -311,6 +313,7 @@ PROCESSOR_LAMBDA_CREDENTIALS_ID
 PROCESSOR_LAMBDA_CREDENTIALS_ROLE
 PROCESSOR_LAMBDA_CREDENTIALS_SECRET
 PROCESSOR_LAMBDA_CREDENTIALS_TOKEN
+PROCESSOR_LAMBDA_ENDPOINT
 PROCESSOR_LAMBDA_FUNCTION
 PROCESSOR_LAMBDA_PARALLEL                            = false
 PROCESSOR_LAMBDA_RATE_LIMIT
@@ -459,13 +462,15 @@ OUTPUT_S3_CREDENTIALS_ID
 OUTPUT_S3_CREDENTIALS_ROLE
 OUTPUT_S3_CREDENTIALS_SECRET
 OUTPUT_S3_CREDENTIALS_TOKEN
+OUTPUT_S3_ENDPOINT
 OUTPUT_S3_PATH                               = ${!count:files}-${!timestamp_unix_nano}.txt
 OUTPUT_S3_REGION                             = eu-west-1
 OUTPUT_S3_TIMEOUT_S                          = 5
 OUTPUT_SQS_CREDENTIALS_ID
 OUTPUT_SQS_CREDENTIALS_ROLE
 OUTPUT_SQS_CREDENTIALS_SECRET
 OUTPUT_SQS_CREDENTIALS_TOKEN
+OUTPUT_SQS_ENDPOINT
 OUTPUT_SQS_REGION                            = eu-west-1
 OUTPUT_SQS_URL
 OUTPUT_STDOUT_DELIMITER

diff --git a/config/env/default.yaml b/config/env/default.yaml
@@ -190,6 +190,7 @@ input:
           secret: ${INPUT_S3_CREDENTIALS_SECRET}
           token: ${INPUT_S3_CREDENTIALS_TOKEN}
         delete_objects: ${INPUT_S3_DELETE_OBJECTS:false}
+        endpoint: ${INPUT_S3_ENDPOINT}
         prefix: ${INPUT_S3_PREFIX}
         region: ${INPUT_S3_REGION:eu-west-1}
         retries: ${INPUT_S3_RETRIES:3}
@@ -204,6 +205,7 @@ input:
           role: ${INPUT_SQS_CREDENTIALS_ROLE}
           secret: ${INPUT_SQS_CREDENTIALS_SECRET}
           token: ${INPUT_SQS_CREDENTIALS_TOKEN}
+        endpoint: ${INPUT_SQS_ENDPOINT}
         region: ${INPUT_SQS_REGION:eu-west-1}
         timeout_s: ${INPUT_SQS_TIMEOUT_S:5}
         url: ${INPUT_SQS_URL}
@@ -342,6 +344,7 @@ pipeline:
         role: ${PROCESSOR_LAMBDA_CREDENTIALS_ROLE}
         secret: ${PROCESSOR_LAMBDA_CREDENTIALS_SECRET}
         token: ${PROCESSOR_LAMBDA_CREDENTIALS_TOKEN}
+      endpoint: ${PROCESSOR_LAMBDA_ENDPOINT}
       function: ${PROCESSOR_LAMBDA_FUNCTION}
       parallel: ${PROCESSOR_LAMBDA_PARALLEL:false}
       rate_limit: ${PROCESSOR_LAMBDA_RATE_LIMIT}
@@ -538,6 +541,7 @@ output:
           role: ${OUTPUT_S3_CREDENTIALS_ROLE}
           secret: ${OUTPUT_S3_CREDENTIALS_SECRET}
           token: ${OUTPUT_S3_CREDENTIALS_TOKEN}
+        endpoint: ${OUTPUT_S3_ENDPOINT}
         path: ${OUTPUT_S3_PATH:${!count:files}-${!timestamp_unix_nano}.txt}
         region: ${OUTPUT_S3_REGION:eu-west-1}
         timeout_s: ${OUTPUT_S3_TIMEOUT_S:5}
@@ -547,6 +551,7 @@ output:
           role: ${OUTPUT_SQS_CREDENTIALS_ROLE}
           secret: ${OUTPUT_SQS_CREDENTIALS_SECRET}
           token: ${OUTPUT_SQS_CREDENTIALS_TOKEN}
+        endpoint: ${OUTPUT_SQS_ENDPOINT}
         region: ${OUTPUT_SQS_REGION:eu-west-1}
         url: ${OUTPUT_SQS_URL}
       stdout:

diff --git a/config/everything.yaml b/config/everything.yaml
@@ -117,13 +117,13 @@ input:
       skip_cert_verify: false
       client_certs: []
   kinesis:
-    endpoint: ""
-    region: eu-west-1
     credentials:
       id: ""
       secret: ""
       token: ""
       role: ""
+    endpoint: ""
+    region: eu-west-1
     limit: 100
     stream: ""
     shard: "0"
@@ -225,6 +225,12 @@ input:
     commit_period_ms: 1000
     timeout_ms: 5000
   s3:
+    credentials:
+      id: ""
+      secret: ""
+      token: ""
+      role: ""
+    endpoint: ""
     region: eu-west-1
     bucket: ""
     prefix: ""
@@ -234,20 +240,16 @@ input:
     sqs_body_path: Records.s3.object.key
     sqs_envelope_path: ""
     sqs_max_messages: 10
-    credentials:
-      id: ""
-      secret: ""
-      token: ""
-      role: ""
     timeout_s: 5
   sqs:
-    region: eu-west-1
-    url: ""
     credentials:
       id: ""
       secret: ""
       token: ""
       role: ""
+    endpoint: ""
+    region: eu-west-1
+    url: ""
     timeout_s: 5
   stdin:
     multipart: false
@@ -508,6 +510,7 @@ pipeline:
         secret: ""
         token: ""
         role: ""
+      endpoint: ""
       region: eu-west-1
       function: ""
       timeout_ms: 5000
@@ -669,16 +672,16 @@ output:
       skip_cert_verify: false
       client_certs: []
   kinesis:
-    endpoint: ""
-    region: eu-west-1
-    stream: ""
-    hash_key: ""
-    partition_key: ""
     credentials:
       id: ""
       secret: ""
       token: ""
       role: ""
+    endpoint: ""
+    region: eu-west-1
+    stream: ""
+    hash_key: ""
+    partition_key: ""
     max_retries: 0
     backoff:
       initial_interval: 1s
@@ -729,23 +732,25 @@ output:
       max_interval: 3s
       max_elapsed_time: 0s
   s3:
-    region: eu-west-1
-    bucket: ""
-    path: ${!count:files}-${!timestamp_unix_nano}.txt
     credentials:
       id: ""
       secret: ""
       token: ""
       role: ""
+    endpoint: ""
+    region: eu-west-1
+    bucket: ""
+    path: ${!count:files}-${!timestamp_unix_nano}.txt
     timeout_s: 5
   sqs:
-    region: eu-west-1
-    url: ""
     credentials:
       id: ""
       secret: ""
       token: ""
       role: ""
+    endpoint: ""
+    region: eu-west-1
+    url: ""
   stdout:
     delimiter: ""
   switch:
@@ -769,16 +774,16 @@ resources:
     example:
       type: memory
       dynamodb:
-        consistent_read: false
         credentials:
           id: ""
           secret: ""
           token: ""
           role: ""
-        data_key: ""
         endpoint: ""
+        region: eu-west-1
+        consistent_read: false
+        data_key: ""
         hash_key: ""
-        region: ""
         table: ""
         ttl: ""
         ttl_key: ""

diff --git a/config/processors/lambda.json b/config/processors/lambda.json
@@ -28,6 +28,7 @@
 						"secret": "",
 						"token": ""
 					},
+					"endpoint": "",
 					"function": "",
 					"parallel": false,
 					"rate_limit": "",

diff --git a/config/processors/lambda.yaml b/config/processors/lambda.yaml
@@ -22,6 +22,7 @@ pipeline:
         role: ""
         secret: ""
         token: ""
+      endpoint: ""
       function: ""
       parallel: false
       rate_limit: ""

diff --git a/config/s3.json b/config/s3.json
@@ -16,6 +16,7 @@
 				"token": ""
 			},
 			"delete_objects": false,
+			"endpoint": "",
 			"prefix": "",
 			"region": "eu-west-1",
 			"retries": 3,
@@ -44,6 +45,7 @@
 				"secret": "",
 				"token": ""
 			},
+			"endpoint": "",
 			"path": "${!count:files}-${!timestamp_unix_nano}.txt",
 			"region": "eu-west-1",
 			"timeout_s": 5