Merge f8b1879 into 46437ab

Makefile

@@ -20,7 +20,7 @@ install: deps
 
 lint: testdeps
 	go get -v github.com/golang/lint/golint
-	for file in $$(find . -name '*.go' | grep -v '\.pb\.go\|\.pb\.gw\.go\|examples\|pubsub\/awssub_test\.go'); do \
+	for file in $$(find . -name '*.go' | grep -v '\.pb\.go\|\.pb\.gw\.go\|examples\|pubsub\/aws\/awssub_test\.go'); do \
 		golint $${file}; \
 		if [ -n "$$(golint $${file})" ]; then \
 			exit 1; \

README.md

@@ -14,26 +14,25 @@ This toolkit provides packages to put together server and pubsub daemons with th
 
 In this toolkit, you will find:
 
-## The `config` package
+## The `config` packages
 
-This package contains a handful of structs meant for managing common configuration options and credentials. There are currently configs for:
+The `config` package contains a handful of useful functions to load to configuration structs from JSON files, JSON blobs in Consul k/v or environment variables.
 
+The subpackages contain structs meant for managing common configuration options and credentials. There are currently configs for:
+
+* Go Kit Metrics
 * MySQL
 * MongoDB
 * Oracle
-* AWS (SNS, SQS, S3, DynamoDB, ElastiCache)
+* AWS (S3, DynamoDB, ElastiCache)
 * GCP
-* Kafka
 * Gorilla's `securecookie`
-* Gizmo Servers
-
-The package also has a generic `Config` type that contains all of the above types. It's meant to be a 'catch all' struct that most applications should be able to use.
 
-`config` also contains functions to load these config structs from JSON files, JSON blobs in Consul k/v or environment variables.
+The package also has a generic `Config` type in the `config/combined` subpackage that contains all of the above types. It's meant to be a 'catch all' convenience struct that many applications should be able to use.
 
 ## The `server` package
 
-This package is the bulk of the toolkit and relies on `config` for any managing `Server` implementations. A server must implement the following interface:
+This package is the bulk of the toolkit and relies on `server.Config` for any managing `Server` implementations. A server must implement the following interface:
 
 ```go
 // Server is the basic interface that defines what expect from any server.
@@ -154,9 +153,9 @@ type RPCService interface {
 
 The `Middleware(..)` functions offer each service a 'hook' to wrap each of its endpoints. This may be handy for adding additional headers or context to the request. This is also the point where other, third-party middleware could be easily plugged in (i.e. oauth, tracing, metrics, logging, etc.)
 
-## The `pubsub` package
+## The `pubsub` packages
 
-This package contains two generic interfaces for publishing data to queues and subscribing and consuming data from those queues.
+The base `pubsub` package contains two generic interfaces for publishing data to queues and subscribing and consuming data from those queues.
 
 ```go
 // Publisher is a generic interface to encapsulate how we want our publishers
@@ -185,13 +184,13 @@ type Subscriber interface {
 
 Where a `SubscriberMessage` is an interface that gives implementations a hook for acknowledging/delete messages. Take a look at the docs for each implementation in `pubsub` to see how they behave.
 
-There are currently 2 implementations of each type of `pubsub` interfaces:
+There are currently 3 implementations of each type of `pubsub` interfaces:
 
-For pubsub via Amazon's SNS/SQS, you can use the `SNSPublisher` and the `SQSSubscriber`.
+For pubsub via Amazon's SNS/SQS, you can use the `pubsub/aws` package.
 
-For pubsub via Google's Pubsub, you can use the `GCPPublisher` and the `GCPSubscriber`.
+For pubsub via Google's Pubsub, you can use the `pubsub/gcp` package.
 
-For pubsub via Kafka topics, you can use the `KakfaPublisher` and the `KafkaSubscriber`.
+For pubsub via Kafka topics, you can use the `pubsub/kafka` package.
 
 ## The `pubsub/pubsubtest` package
 

config/aws.go → config/aws/aws.go

@@ -1,10 +1,10 @@
-package config
+package aws
 
 import (
 	"fmt"
 	"log"
-	"time"
 
+	"github.com/NYTimes/gizmo/config"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -13,61 +13,37 @@ import (
 )
 
 const (
-	// AWSRegionUSEast1 is a helper constant for AWS configs.
-	AWSRegionUSEast1 = "us-east-1"
-	// AWSRegionUSWest is a helper constant for AWS configs.
-	AWSRegionUSWest = "us-west-1"
+	// RegionUSEast1 is a helper constant for AWS configs.
+	RegionUSEast1 = "us-east-1"
+	// RegionUSWest is a helper constant for AWS configs.
+	RegionUSWest = "us-west-1"
 )
 
 type (
-	// AWS holds common AWS credentials and keys.
-	AWS struct {
+	// Config holds common AWS credentials and keys.
+	Config struct {
 		SecretKey string `envconfig:"AWS_SECRET_KEY"`
 		AccessKey string `envconfig:"AWS_ACCESS_KEY"`
 
 		Region string `envconfig:"AWS_REGION"`
 	}
 
-	// SQS holds the info required to work with Amazon SQS
-	SQS struct {
-		AWS
-		QueueName string `envconfig:"AWS_SQS_NAME"`
-		// MaxMessages will override the DefaultSQSMaxMessages.
-		MaxMessages *int64 `envconfig:"AWS_SQS_MAX_MESSAGES"`
-		// TimeoutSeconds will override the DefaultSQSTimeoutSeconds.
-		TimeoutSeconds *int64 `envconfig:"AWS_SQS_TIMEOUT_SECONDS"`
-		// SleepInterval will override the DefaultSQSSleepInterval.
-		SleepInterval *time.Duration `envconfig:"AWS_SQS_SLEEP_INTERVAL"`
-		// DeleteBufferSize will override the DefaultSQSDeleteBufferSize.
-		DeleteBufferSize *int `envconfig:"AWS_SQS_DELETE_BUFFER_SIZE"`
-		// ConsumeBase64 is a flag to signal the subscriber to base64 decode the payload
-		// before returning it. If it is not set in the config, the flag will default
-		// to 'true'.
-		ConsumeBase64 *bool `envconfig:"AWS_SQS_CONSUME_BASE64"`
-	}
-
-	// SNS holds the info required to work with Amazon SNS.
-	SNS struct {
-		AWS
-		Topic string `envconfig:"AWS_SNS_TOPIC"`
-	}
-
 	// S3 holds the info required to work with Amazon S3.
 	S3 struct {
-		AWS
+		Config
 		Bucket string `envconfig:"AWS_S3_BUCKET_NAME"`
 	}
 
 	// DynamoDB holds some basic info required to work with Amazon DynamoDB.
 	DynamoDB struct {
-		AWS
+		Config
 		TableName string `envconfig:"AWS_DYNAMODB_TABLE_NAME"`
 	}
 
 	// ElastiCache holds the basic info required to work with
 	// Amazon ElastiCache.
 	ElastiCache struct {
-		AWS
+		Config
 		ClusterID string `envconfig:"AWS_ELASTICACHE_CLUSTER_ID"`
 	}
 )
@@ -107,41 +83,35 @@ func (e *ElastiCache) MustClient() *memcache.Client {
 	return memcache.New(nodes...)
 }
 
-// LoadAWSFromEnv will attempt to load the AWS structs
+// LoadConfigFromEnv will attempt to load the Config struct
+// from environment variables.
+func LoadConfigFromEnv() Config {
+	var aws Config
+	config.LoadEnvConfig(&aws)
+	return aws
+}
+
+// LoadDynamoDBFromEnv will attempt to load the DynamoDB struct
 // from environment variables. If not populated, nil
 // is returned.
-func LoadAWSFromEnv() (*AWS, *SNS, *SQS, *S3, *DynamoDB, *ElastiCache) {
-	var (
-		aws = &AWS{}
-		sns = &SNS{}
-		sqs = &SQS{}
-		s3  = &S3{}
-		ddb = &DynamoDB{}
-		ec  = &ElastiCache{}
-	)
-	LoadEnvConfig(aws)
-	if aws.AccessKey == "" {
-		aws = nil
-	}
-	LoadEnvConfig(sns)
-	if sns.Topic == "" {
-		sns = nil
-	}
-	LoadEnvConfig(sqs)
-	if sqs.QueueName == "" {
-		sqs = nil
-	}
-	LoadEnvConfig(s3)
-	if s3.Bucket == "" {
-		s3 = nil
-	}
-	LoadEnvConfig(ddb)
-	if ddb.TableName == "" {
-		ddb = nil
-	}
-	LoadEnvConfig(ec)
-	if ec.ClusterID == "" {
-		ec = nil
-	}
-	return aws, sns, sqs, s3, ddb, ec
+func LoadDynamoDBFromEnv() DynamoDB {
+	var ddb DynamoDB
+	config.LoadEnvConfig(&ddb)
+	return ddb
+}
+
+// LoadS3FromEnv will attempt to load the S3 struct
+// from environment variables.
+func LoadS3FromEnv() S3 {
+	var s3 S3
+	config.LoadEnvConfig(&s3)
+	return s3
+}
+
+// LoadElastiCacheFromEnv will attempt to load the ElasiCache struct
+// from environment variables.
+func LoadElastiCacheFromEnv() ElastiCache {
+	var el ElastiCache
+	config.LoadEnvConfig(&el)
+	return el
 }

config/combined/config.go

@@ -0,0 +1,98 @@
+package combined
+
+import (
+	"strings"
+
+	"github.com/NYTimes/gizmo/config"
+	"github.com/NYTimes/gizmo/config/aws"
+	"github.com/NYTimes/gizmo/config/cookie"
+	"github.com/NYTimes/gizmo/config/metrics"
+	"github.com/NYTimes/gizmo/config/mongodb"
+	"github.com/NYTimes/gizmo/config/mysql"
+	"github.com/NYTimes/gizmo/config/oracle"
+	awsps "github.com/NYTimes/gizmo/pubsub/aws"
+	"github.com/NYTimes/gizmo/pubsub/kafka"
+	"github.com/NYTimes/gizmo/server"
+)
+
+// Config is a generic struct to hold information for applications that
+// need to connect to databases, handle cookies, log events or emit metrics.
+// If you have a use case that does not fit this struct, you can
+// make a struct containing just the types that suit your needs and use
+// some of the helper functions in this package to load it from the environment.
+type Config struct {
+	Server *server.Config
+
+	AWS         aws.Config
+	SQS         awsps.SQSConfig
+	SNS         awsps.SNSConfig
+	S3          aws.S3
+	DynamoDB    aws.DynamoDB
+	ElastiCache aws.ElastiCache
+
+	Kafka *kafka.Config
+
+	Oracle oracle.Config
+
+	MySQL      *mysql.Config
+	MySQLSlave *mysql.Config
+
+	MongoDB *mongodb.Config
+
+	Cookie cookie.Config
+
+	// GraphiteHost is DEPRECATED. Please use the
+	// Metrics config with "Type":"graphite" and this
+	// value in the "Addr" field.
+	GraphiteHost *string `envconfig:"GRAPHITE_HOST"`
+
+	Metrics metrics.Config
+
+	LogLevel *string `envconfig:"APP_LOG_LEVEL"`
+	Log      *string `envconfig:"APP_LOG"`
+}
+
+// LoadConfigFromEnv will attempt to inspect the environment
+// of any valid config options and will return a populated
+// Config struct with what it found.
+// If you need a unique config object and want to use envconfig, you
+// will need to run the LoadXXFromEnv for each child struct in
+// your config struct. For an example on how to do this, check out the
+// guts of this function.
+func LoadConfigFromEnv() *Config {
+	var app Config
+	config.LoadEnvConfig(&app)
+	app.AWS = aws.LoadConfigFromEnv()
+	app.SNS = awsps.LoadSNSConfigFromEnv()
+	app.SQS = awsps.LoadSQSConfigFromEnv()
+	app.S3 = aws.LoadS3FromEnv()
+	app.DynamoDB = aws.LoadDynamoDBFromEnv()
+	app.ElastiCache = aws.LoadElastiCacheFromEnv()
+	app.MongoDB = mongodb.LoadConfigFromEnv()
+	app.Kafka = kafka.LoadConfigFromEnv()
+	app.MySQL = mysql.LoadConfigFromEnv()
+	app.Oracle = oracle.LoadConfigFromEnv()
+	app.Cookie = cookie.LoadConfigFromEnv()
+	app.Server = server.LoadConfigFromEnv()
+	app.Metrics = metrics.LoadConfigFromEnv()
+	return &app
+}
+
+// NewConfig will attempt to unmarshal the contents
+// of the given JSON string source into a Config struct.
+// The value of fileName can be either the path to a JSON
+// file or a path to a JSON string found in Consul's Key
+// Value storage (using the format consul:path/to/JSON/string).
+// If the value of fileName is empty, a blank Config will
+// be returned.
+func NewConfig(fileName string) *Config {
+	var c Config
+	if fileName == "" {
+		return &c
+	}
+	if strings.HasPrefix(fileName, "consul") {
+		return config.LoadJSONFromConsulKV(fileName, &c).(*Config)
+	}
+	config.LoadJSONFile(fileName, &c)
+	return &c
+}