Merge pull request #13 from sapcc/secret-handling

Use individual env vars for database parameters

.travis.yml

@@ -12,5 +12,5 @@ install:
   - go get -u github.com/mattn/goveralls
 
 script:
-  - make all check GO_BUILDFLAGS="-mod vendor"
+  - make check
   - $HOME/gopath/bin/goveralls -coverprofile=build/cover.out -service=travis-ci

README.md

@@ -65,12 +65,20 @@ All components receive configuration via environment variables. The following va
 | Variable | Default | Explanation |
 | -------- | ------- | ----------- |
 | `CASTELLUM_ASSET_MANAGERS` | *(required)* | A comma-separated list of all asset managers that can be enabled. This configures what kinds of assets Castellum can handle. See [`docs/asset-managers/`](./docs/asset-managers/) for which asset managers exist. |
-| `CASTELLUM_DB_URI` | *(required)* | A [libpq connection URI][pq-uri] that locates the Castellum database. The non-URI "connection string" format is not allowed; it must be a URI. |
+| `CASTELLUM_DB_USERNAME` | `postgres` | Username of the user that Castellum should use to connect to the database. |
+| `CASTELLUM_DB_PASSWORD` | *(optional)* | Password for the specified user. |
+| `CASTELLUM_DB_HOSTNAME` | `localhost` | Hostname of the database server. |
+| `CASTELLUM_DB_PORT` | `5432` | Port on which the PostgreSQL service is running on. |
+| `CASTELLUM_DB_NAME` | `castellum` | The name of the database. |
+| `CASTELLUM_DB_CONNECTION_OPTIONS` | *(optional)* | Database connection options. |
 | `CASTELLUM_HTTP_LISTEN_ADDRESS` | `:8080` | Listen address for the internal HTTP server. For `castellum observer/worker`, this just exposes Prometheus metrics on `/metrics`. For `castelum api`, this also exposes [the REST API](./docs/api-spec.md). |
 | `CASTELLUM_MAX_ASSET_SIZES` | *(optional)* | A comma-separated list of `<asset-type>=<max-size>` pairs. If present, only resource configurations honoring these constraints will be allowed. |
 | `CASTELLUM_OSLO_POLICY_PATH` | *(required)* | Path to the `policy.json` file for this service. See [*Oslo policy*](#oslo-policy) for details. |
-| `CASTELLUM_RABBITMQ_URI` | *(optional)* | RabbitMQ URI as per the [AMQP URI format](https://www.rabbitmq.com/uri-spec.html). If this variable is configured then Castellum will send audit events to the respective RabbitMQ server. |
-| `CASTELLUM_RABBITMQ_QUEUE_NAME` | *(required if `CASTELLUM_RABBITMQ_URI` is configured)* | Name for the queue that will hold the audit events. The events are published to the default exchange. |
+| `CASTELLUM_RABBITMQ_QUEUE_NAME` | *(required for enabling audit trail)* | Name for the queue that will hold the audit events. The events are published to the default exchange. |
+| `CASTELLUM_RABBITMQ_USERNAME` | *(optional)* | RabbitMQ Username. |
+| `CASTELLUM_RABBITMQ_PASSWORD` | *(optional)* | Password for the specified user. |
+| `CASTELLUM_RABBITMQ_HOSTNAME` | `localhost` | Hostname of the RabbitMQ server. |
+| `CASTELLUM_RABBITMQ_PORT` | `5672` |  Port number to which the underlying connection is made. |
 | `CASTELLUM_AUDIT_SILENT` | `false` | Disable audit event logging to standard output. |
 | `CASTELLUM_LOG_SCRAPES` | `false` | Whether to write a log line for each asset scrape operation. This can be useful to debug situations where Castellum does not create operations when it should, but it generates a lot of log traffic (one line per asset per 5 minutes, which e.g. for 2000 assets is about 1 GiB per week). |
 | `OS_...` | *(required)* | A full set of OpenStack auth environment variables for Castellum's service user. See [documentation for openstackclient][os-env] for details. |

go.mod

@@ -5,16 +5,18 @@ go 1.15
 require (
 	github.com/databus23/goslo.policy v0.0.0-20170317131957-3ae74dd07ebf
 	github.com/dlmiddlecote/sqlstats v1.0.1
-	github.com/gophercloud/gophercloud v0.13.0
+	github.com/gofrs/uuid v3.3.0+incompatible // indirect
+	github.com/gophercloud/gophercloud v0.14.0
 	github.com/gophercloud/utils v0.0.0-20200918191848-da0e919a012a
 	github.com/gorilla/mux v1.8.0
 	github.com/majewsky/sqlproxy v0.0.0-20190524080451-ba724456c4af
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/common v0.14.0
 	github.com/rs/cors v1.7.0
-	github.com/sapcc/go-bits v0.0.0-20201015124403-5d1815e53d18
+	github.com/sapcc/go-bits v0.0.0-20201126124612-fcd600685fdb
 	github.com/sapcc/gophercloud-limes v0.0.0-20200819120227-33ee0896b7e8
 	github.com/sapcc/hermes v0.0.0-20200928164006-62559084bfce
 	github.com/sapcc/limes v0.0.0-20200928094800-7845b54d12e7
+	github.com/streadway/amqp v1.0.0 // indirect
 	gopkg.in/gorp.v2 v2.2.0
 )

go.sum

@@ -94,6 +94,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -138,8 +140,8 @@ github.com/gophercloud/gophercloud v0.6.1-0.20191122030953-d8ac278c1c9d/go.mod h
 github.com/gophercloud/gophercloud v0.10.1-0.20200424014253-c3bfe50899e5/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss=
 github.com/gophercloud/gophercloud v0.12.0 h1:mZrie07npp6ODiwHZolTicr5jV8Ogn43AvAsSMm6Ork=
 github.com/gophercloud/gophercloud v0.12.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss=
-github.com/gophercloud/gophercloud v0.13.0 h1:1XkslZZRm6Ks0bLup+hBNth+KQf+0JA1UeoB7YKw9E8=
-github.com/gophercloud/gophercloud v0.13.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM=
+github.com/gophercloud/gophercloud v0.14.0 h1:c2Byo+YMxhHlTJ3TPptjQ4dOQ1YknTHDJ/9zClDH+84=
+github.com/gophercloud/gophercloud v0.14.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM=
 github.com/gophercloud/utils v0.0.0-20190313033024-0bcc8e728cb5 h1:8USoe8m65WcTOYy+MUu+EtLJJysSODnoNDNCEWhDMso=
 github.com/gophercloud/utils v0.0.0-20190313033024-0bcc8e728cb5/go.mod h1:SZ9FTKibIotDtCrxAU/evccoyu1yhKST6hgBvwTB5Eg=
 github.com/gophercloud/utils v0.0.0-20200918191848-da0e919a012a h1:Nn4npa87SL5uLA6rYbQp/u3JwVTeEJ7VfaWrJw97F5Q=
@@ -363,8 +365,8 @@ github.com/sapcc/go-bits v0.0.0-20190910095551-3863bd6b4943 h1:tOsa/z/MGmhxp5qlu
 github.com/sapcc/go-bits v0.0.0-20190910095551-3863bd6b4943/go.mod h1:YpW83RbRSHPedMBfuvdfYJMQbejxSxzK1Kj/39szShw=
 github.com/sapcc/go-bits v0.0.0-20200719195243-6f202ca5296a h1:qYxXWqfFKGnHo3wLSyyXewRCQygkoanKnu/syOiRzDE=
 github.com/sapcc/go-bits v0.0.0-20200719195243-6f202ca5296a/go.mod h1:r6AY7ym3xd4WK0Crz/D0Rf7J+oMg7haFiADpqiY1e3A=
-github.com/sapcc/go-bits v0.0.0-20201015124403-5d1815e53d18 h1:EOVOg2worqvf2hGE7kPCN6CZUaK8doRBOWDLynIyKqU=
-github.com/sapcc/go-bits v0.0.0-20201015124403-5d1815e53d18/go.mod h1:r6AY7ym3xd4WK0Crz/D0Rf7J+oMg7haFiADpqiY1e3A=
+github.com/sapcc/go-bits v0.0.0-20201126124612-fcd600685fdb h1:bvD4yq3nxjUryiy921Eld0i33PE1oST4GLxW6Wlhzk4=
+github.com/sapcc/go-bits v0.0.0-20201126124612-fcd600685fdb/go.mod h1:MvjxocNfMqx+sQ2xdfrLveBljrUmGPaJT/UHUwyZ8lM=
 github.com/sapcc/gophercloud-limes v0.0.0-20200819120227-33ee0896b7e8 h1:oeLSkpv6DhRiNmMnYPwlCm/HeGfogYslFQDdLSWhRTY=
 github.com/sapcc/gophercloud-limes v0.0.0-20200819120227-33ee0896b7e8/go.mod h1:8/BIFprjDN70kq0r6GyFgBmZHEWF6/8+q6wmcvSmHmM=
 github.com/sapcc/hermes v0.0.0-20190722215605-d772f40f9fc6 h1:iI+ew9JUORCqP+8pElkSayI/smvdiYoPRX6J4DmkVTI=
@@ -407,6 +409,8 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 h1:0ngsPmuP6XIjiFRN
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw=
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
+github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

internal/api/audit.go

@@ -34,45 +34,30 @@ import (
 //eventSink is a channel that receives audit events.
 var eventSink chan<- cadf.Event
 
-var (
-	showAuditOnStdout   bool
-	sendAuditToRabbitMQ bool
-	rabbitURI           string
-	rabbitQueueName     string
-)
+var showAuditOnStdout bool
 
-func init() {
+// StartAuditLogging starts audit logging for the API.
+func StartAuditLogging(rabbitURI, rabbitQueueName string) {
 	silenceAuditLogging, _ := strconv.ParseBool(os.Getenv("CASTELLUM_AUDIT_SILENT"))
 	showAuditOnStdout = !silenceAuditLogging
 
-	rabbitURI = os.Getenv("CASTELLUM_RABBITMQ_URI")
-	if rabbitURI != "" {
-		rabbitQueueName = os.Getenv("CASTELLUM_RABBITMQ_QUEUE_NAME")
-		if rabbitQueueName == "" {
-			logg.Fatal("missing required environment variable: CASTELLUM_RABBITMQ_QUEUE_NAME")
-		}
-		sendAuditToRabbitMQ = true
-	}
+	auditEventPublishSuccessCounter.Add(0)
+	auditEventPublishFailedCounter.Add(0)
 
-	if sendAuditToRabbitMQ {
-		auditEventPublishSuccessCounter.Add(0)
-		auditEventPublishFailedCounter.Add(0)
-
-		onSuccessFunc := func() {
-			auditEventPublishSuccessCounter.Inc()
-		}
-		onFailFunc := func() {
-			auditEventPublishFailedCounter.Inc()
-		}
-		s := make(chan cadf.Event, 20)
-		eventSink = s
-
-		go audittools.AuditTrail{
-			EventSink:           s,
-			OnSuccessfulPublish: onSuccessFunc,
-			OnFailedPublish:     onFailFunc,
-		}.Commit(rabbitURI, rabbitQueueName)
+	onSuccessFunc := func() {
+		auditEventPublishSuccessCounter.Inc()
+	}
+	onFailFunc := func() {
+		auditEventPublishFailedCounter.Inc()
 	}
+	s := make(chan cadf.Event, 20)
+	eventSink = s
+
+	go audittools.AuditTrail{
+		EventSink:           s,
+		OnSuccessfulPublish: onSuccessFunc,
+		OnFailedPublish:     onFailFunc,
+	}.Commit(rabbitURI, rabbitQueueName)
 }
 
 var observerUUID = audittools.GenerateUUID()

internal/db/model.go

@@ -310,12 +310,7 @@ func init() {
 }
 
 //Init connects to the database and initializes the schema and model types.
-func Init(urlStr string) (*gorp.DbMap, error) {
-	dbURL, err := url.Parse(urlStr)
-	if err != nil {
-		return nil, fmt.Errorf("malformed CASTELLUM_DB_URI: " + err.Error())
-	}
-
+func Init(dbURL *url.URL) (*gorp.DbMap, error) {
 	cfg := easypg.Configuration{
 		PostgresURL: dbURL,
 		Migrations:  SQLMigrations,

internal/test/db.go

@@ -21,6 +21,7 @@ package test
 import (
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"os"
 
 	"github.com/sapcc/castellum/internal/db"
@@ -31,15 +32,20 @@ import (
 //WithDB prepares a DB reference for this test, or fails the test if the DB
 //is not ready.
 func (t T) WithDB(fixtureFile *string, action func(dbi *gorp.DbMap)) {
-	var postgresURL string
+	var postgresURLStr string
 	if os.Getenv("TRAVIS") == "true" {
 		//cf. https://docs.travis-ci.com/user/database-setup/#postgresql
-		postgresURL = "postgres://postgres@localhost/castellum?sslmode=disable"
+		postgresURLStr = "postgres://postgres@localhost/castellum?sslmode=disable"
 	} else {
 		//suitable for use with ./testing/with-postgres-db.sh
-		postgresURL = "postgres://postgres@localhost:54321/castellum?sslmode=disable"
+		postgresURLStr = "postgres://postgres@localhost:54321/castellum?sslmode=disable"
 	}
-	dbi, err := db.Init(postgresURL)
+	dbURL, err := url.Parse(postgresURLStr)
+	if err != nil {
+		t.Fatalf("malformed database URL %q: %s", postgresURLStr, err.Error())
+	}
+
+	dbi, err := db.Init(dbURL)
 	if err != nil {
 		t.Error(err)
 		t.Log("Try prepending ./testing/with-postgres-db.sh to your command.")

main.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
@@ -63,7 +64,21 @@ func main() {
 	logg.ShowDebug, _ = strconv.ParseBool(os.Getenv("CASTELLUM_DEBUG"))
 
 	//initialize DB connection
-	dbi, err := db.Init(mustGetenv("CASTELLUM_DB_URI"))
+	dbUsername := envOrDefault("CASTELLUM_DB_USERNAME", "postgres")
+	dbPass := os.Getenv("CASTELLUM_DB_PASSWORD")
+	dbHost := envOrDefault("CASTELLUM_DB_HOSTNAME", "localhost")
+	dbPort := envOrDefault("CASTELLUM_DB_PORT", "5432")
+	dbName := envOrDefault("CASTELLUM_DB_NAME", "castellum")
+	dbConnOpts := os.Getenv("CASTELLUM_DB_CONNECTION_OPTIONS")
+
+	dbURL := &url.URL{
+		Scheme:   "postgres",
+		User:     url.UserPassword(dbUsername, dbPass),
+		Host:     dbHost + ":" + dbPort,
+		Path:     dbName,
+		RawQuery: dbConnOpts,
+	}
+	dbi, err := db.Init(dbURL)
 	if err != nil {
 		logg.Fatal(err.Error())
 	}
@@ -171,6 +186,14 @@ func mustGetenv(key string) string {
 	return val
 }
 
+func envOrDefault(key, defaultVal string) string {
+	val := os.Getenv(key)
+	if val == "" {
+		val = defaultVal
+	}
+	return val
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // task: API
 
@@ -195,6 +218,26 @@ func runAPI(cfg *core.Config, dbi *gorp.DbMap, team core.AssetManagerTeam, provi
 		AllowedHeaders: []string{"Content-Type", "User-Agent", "X-Auth-Token"},
 	}).Handler(handler)
 
+	//Start audit logging.
+	rabbitQueueName := os.Getenv("CASTELLUM_RABBITMQ_QUEUE_NAME")
+	if rabbitQueueName != "" {
+		var rabbitUserInfo string
+		if rabbitUsername := os.Getenv("CASTELLUM_RABBITMQ_USERNAME"); rabbitUsername != "" {
+			rabbitUserInfo += rabbitUsername
+		}
+		if rabbitPass := os.Getenv("CASTELLUM_RABBITMQ_PASSWORD"); rabbitPass != "" {
+			rabbitUserInfo += ":" + rabbitPass
+		}
+		if rabbitUserInfo != "" {
+			rabbitUserInfo += "@"
+		}
+		rabbitHost := envOrDefault("CASTELLUM_RABBITMQ_HOSTNAME", "localhost")
+		rabbitPort := envOrDefault("CASTELLUM_RABBITMQ_PORT", "5672")
+
+		rabbitURI := fmt.Sprintf("amqp://%s%s:%s/", rabbitUserInfo, rabbitHost, rabbitPort)
+		api.StartAuditLogging(rabbitURI, rabbitQueueName)
+	}
+
 	//metrics and healthcheck are deliberately not covered by any of the
 	//middlewares - we do not want to log those requests
 	http.Handle("/", handler)