WIP: Bugfixes and upgrades

.gitignore

@@ -1,5 +1,6 @@
 .build
 postgres_exporter
+postgres_exporter_integration_test
 *.tar.gz
 *.test
 *-stamp

.travis.yml

@@ -4,13 +4,26 @@ services:
 language: go
 go:
 - '1.7'
+# Make sure we have p2 and the postgres client.
+before_install:
+- sudo wget -O /usr/local/bin/p2 https://github.com/wrouesnel/p2cli/releases/download/r4/p2 &&
+  sudo chmod +x /usr/local/bin/p2
+- sudo wget -O /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/1.9.0-rc4/docker-compose-Linux-x86_64 &&
+  sudo chmod +x /usr/local/bin/docker-compose
+- sudo apt-get update && sudo apt-get install postgresql-client-common
+
 script:
 - make all
 - make docker
 - make test-integration
 after_success:
-- if [ "$TRAVIS_BRANCH" == "master" ]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER
-  -p $DOCKER_PASS ; docker push wrouesnel/postgres_exporter ; fi
+- docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+# Push a tagged build if a tag is found.
+- if [ ! -z "$TRAVIS_TAG" ]; then 
+    docker tag wrouesnel/postgres_exporter:latest wrouesnel/postgres_exporter:$TRAVIS_TAG ;
+  fi
+# Push a latest version
+- if [ "$TRAVIS_BRANCH" == "master" ]; then docker push wrouesnel/postgres_exporter ; fi
 env:
   global:
   - secure: RfoWQj5tEB/t3XL2tqJW7u7Qscpz1QBOfF9lMFpB4kAUMTtZU0zBbXfMo1JheGoJQQxD/7NLRHhbUWPT2489o3KKpRTQ7RHn3k8n5U7opH01bWX0+l/EPVmhlsKjSDSLGgmxz80j3I6C8ZV3qDUijSx7r90QUNHGbZtV7g+KtoUTpRV0zir/heK6qq9LHWNHbNsJyHK8qHmd6g1UzWIBaZPJ6a/n/rO2jq4uS1JR0VlIJPRF11HOLH8IjFQvVYpN7YbEslxyNsfQJUSP/7CghSLLVWPSATEjMm8a5GJVLc564+nYghm484psEtiMXkZ3n6ie7AT8aJrKfexWrwh2aCc+cK4PiyXrf4euZehZNYogmFCqWzd1LJKcN2uIkpBSuZQDm3e6c4qkkWGpx+RdFWtAMG8IgZLDbcuryxFNzMwHc2CJ009s9Zsa+g7D57csyR5LCZ8YtNGI3g8FmhwpCKvYkfKa9aijUEWyJMyT4Vhd/w7btMTuwYHgUQ85k4ov4Xjz5SNpAGgemig5G5w7PJj4NhGvIBz9weL154x/BFVjHOZZ6Y/bWgJIPoW1KM15x5K8QylWYEBUHtwiyVyXOxHqt6MOX1vYo1L37jMK88IErrfh/VmlxEhtN9wOghk8IudMfFwQtjIwiWlJf218wxMIzUjoyb5/25tU9f2OJrg=

Makefile

@@ -7,7 +7,11 @@ all: vet test postgres_exporter
 
 # Simple go build
 postgres_exporter: $(GO_SRC)
-	CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=git:$(shell git rev-parse HEAD)" -o postgres_exporter .
+	CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=$(shell git describe --dirty)" -o postgres_exporter .
+
+postgres_exporter_integration_test: $(GO_SRC)
+	CGO_ENABLED=0 go test -c -tags integration \
+	    -a -ldflags "-extldflags '-static' -X main.Version=git:$(shell git describe --dirty)" -o postgres_exporter_integration_test .
 
 # Take a go build and turn it into a minimal container
 docker: postgres_exporter
@@ -19,8 +23,8 @@ vet:
 test:
 	go test -v .
 
-test-integration:
-	tests/test-smoke
+test-integration: postgres_exporter postgres_exporter_integration_test
+	tests/test-smoke ./postgres_exporter ./postgres_exporter_integration_test
 
 # Do a self-contained docker build - we pull the official upstream container
 # and do a self-contained build.

README.md

@@ -34,10 +34,15 @@ Package vendoring is handled with [`govendor`](https://github.com/kardianos/gove
 
 ### Flags
 
-Name               | Description
--------------------|------------
-web.listen-address | Address to listen on for web interface and telemetry.
-web.telemetry-path | Path under which to expose metrics.
+* `web.listen-address` 
+  Address to listen on for web interface and telemetry.
+
+* `web.telemetry-path`
+  Path under which to expose metrics.
+
+* `config.expect-replication-stats`
+  The target database has replication turned on - log errors when
+  replication stats are missing.
 
 ### Setting the Postgres server's data source name
 

postgres_exporter_integration_test.go

@@ -0,0 +1,77 @@
+// These are specialized integration tests. We only build them when we're doing
+// a lot of additional work to keep the external docker environment they require
+// working.
+// +build integration
+
+package main
+
+import (
+	"os"
+	"testing"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"database/sql"
+	_ "github.com/lib/pq"
+	"fmt"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type IntegrationSuite struct{
+	e *Exporter
+}
+
+var _ = Suite(&IntegrationSuite{})
+
+func (s *IntegrationSuite) SetUpSuite(c *C) {
+	dsn := os.Getenv("DATA_SOURCE_NAME")
+	c.Assert(dsn, Not(Equals), "")
+
+	exporter := NewExporter(dsn, "")
+	c.Assert(exporter, NotNil)
+	// Assign the exporter to the suite
+	s.e = exporter
+
+	prometheus.MustRegister(exporter)
+}
+
+// TODO: it would be nice if this didn't mostly just recreate the scrape function
+func (s *IntegrationSuite) TestAllNamespacesReturnResults(c *C) {
+	// Setup a dummy channel to consume metrics
+	ch := make(chan prometheus.Metric, 100)
+	go func() {
+		for _ = range ch {}
+	}()
+
+	// Open a database connection
+	db, err := sql.Open("postgres", s.e.dsn)
+	c.Assert(db, NotNil)
+	c.Assert(err, IsNil)
+	defer db.Close()
+
+	// Do a version update
+	err = s.e.checkMapVersions(ch, db)
+	c.Assert(err, IsNil)
+
+	// Check the show variables work
+	nonFatalErrors := queryShowVariables(ch, db, s.e.variableMap)
+	if !c.Check(len(nonFatalErrors), Equals, 0) {
+		fmt.Println("## NONFATAL ERRORS FOUND")
+		for _, err := range nonFatalErrors {
+			fmt.Println(err)
+		}
+	}
+
+
+	// This should never happen in our test cases.
+	errMap := queryNamespaceMappings(ch, db, s.e.metricMap, s.e.queryOverrides)
+	if !c.Check(len(errMap), Equals, 0) {
+		fmt.Println("## NAMESPACE ERRORS FOUND")
+		for namespace, err := range errMap {
+			fmt.Println(namespace, ":", err)
+		}
+	}
+}

tests/docker-postgres-replication/Dockerfile

@@ -0,0 +1,7 @@
+FROM postgres:9.6
+MAINTAINER Daniel Dent (https://www.danieldent.com)
+ENV PG_MAX_WAL_SENDERS 8
+ENV PG_WAL_KEEP_SEGMENTS 8
+COPY setup-replication.sh /docker-entrypoint-initdb.d/
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh

tests/docker-postgres-replication/Dockerfile.p2

@@ -0,0 +1,7 @@
+FROM postgres:{{VERSION}}
+MAINTAINER Daniel Dent (https://www.danieldent.com)
+ENV PG_MAX_WAL_SENDERS 8
+ENV PG_WAL_KEEP_SEGMENTS 8
+COPY setup-replication.sh /docker-entrypoint-initdb.d/
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh

tests/docker-postgres-replication/README.md

@@ -0,0 +1,11 @@
+# Replicated postgres cluster in docker.
+
+Upstream is forked from https://github.com/DanielDent/docker-postgres-replication
+
+My version lives at https://github.com/wrouesnel/docker-postgres-replication
+
+This very simple docker-compose file lets us stand up a replicated postgres
+cluster so we can test streaming.
+
+# TODO:
+Pull in p2 and template the Dockerfile so we can test multiple versions.

tests/docker-postgres-replication/docker-compose.yml

@@ -0,0 +1,32 @@
+
+version: '2'
+
+services:
+  pg-master:
+    build: '.'
+    image: 'danieldent/postgres-replication'
+    restart: 'always'
+    environment:
+      POSTGRES_USER: 'postgres'
+      POSTGRES_PASSWORD: 'postgres'
+      PGDATA: '/var/lib/postgresql/data/pgdata'
+    volumes:
+     - '/var/lib/postgresql/data'
+    expose:
+     - '5432'
+
+  pg-slave:
+    build: '.'
+    image: 'danieldent/postgres-replication'
+    restart: 'always'
+    environment:
+      POSTGRES_USER: 'postgres'
+      POSTGRES_PASSWORD: 'postgres'
+      PGDATA: '/var/lib/postgresql/data/pgdata'
+      REPLICATE_FROM: 'pg-master'
+    volumes:
+     - '/var/lib/postgresql/data'
+    expose:
+     - '5432'
+    links:
+     - 'pg-master'

tests/docker-postgres-replication/docker-entrypoint.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+
+# Backwards compatibility for old variable names (deprecated)
+if [ "x$PGUSER"     != "x" ]; then
+    POSTGRES_USER=$PGUSER
+fi
+if [ "x$PGPASSWORD" != "x" ]; then
+    POSTGRES_PASSWORD=$PGPASSWORD
+fi
+
+# Forwards-compatibility for old variable names (pg_basebackup uses them)
+if [ "x$PGPASSWORD" = "x" ]; then
+    export PGPASSWORD=$POSTGRES_PASSWORD
+fi
+
+# Based on official postgres package's entrypoint script (https://hub.docker.com/_/postgres/)
+# Modified to be able to set up a slave. The docker-entrypoint-initdb.d hook provided is inadequate.
+
+set -e
+
+if [ "${1:0:1}" = '-' ]; then
+	set -- postgres "$@"
+fi
+
+if [ "$1" = 'postgres' ]; then
+	mkdir -p "$PGDATA"
+	chmod 700 "$PGDATA"
+	chown -R postgres "$PGDATA"
+
+	mkdir -p /run/postgresql
+	chmod g+s /run/postgresql
+	chown -R postgres /run/postgresql
+
+	# look specifically for PG_VERSION, as it is expected in the DB dir
+	if [ ! -s "$PGDATA/PG_VERSION" ]; then
+	    if [ "x$REPLICATE_FROM" == "x" ]; then
+		eval "gosu postgres initdb $POSTGRES_INITDB_ARGS"
+	    else
+            	until ping -c 1 -W 1 ${REPLICATE_FROM}
+            	do
+                	echo "Waiting for master to ping..."
+                	sleep 1s
+            	done
+            	until gosu postgres pg_basebackup -h ${REPLICATE_FROM} -D ${PGDATA} -U ${POSTGRES_USER} -vP -w
+            	do
+                	echo "Waiting for master to connect..."
+                	sleep 1s
+            	done
+	    fi
+
+		# check password first so we can output the warning before postgres
+		# messes it up
+		if [ ! -z "$POSTGRES_PASSWORD" ]; then
+			pass="PASSWORD '$POSTGRES_PASSWORD'"
+			authMethod=md5
+		else
+			# The - option suppresses leading tabs but *not* spaces. :)
+			cat >&2 <<-'EOWARN'
+				****************************************************
+				WARNING: No password has been set for the database.
+				         This will allow anyone with access to the
+				         Postgres port to access your database. In
+				         Docker's default configuration, this is
+				         effectively any other container on the same
+				         system.
+
+				         Use "-e POSTGRES_PASSWORD=password" to set
+				         it in "docker run".
+				****************************************************
+			EOWARN
+
+			pass=
+			authMethod=trust
+		fi
+
+		if [ "x$REPLICATE_FROM" == "x" ]; then
+
+		{ echo; echo "host replication all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null
+		{ echo; echo "host all all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null
+
+		# internal start of server in order to allow set-up using psql-client		
+		# does not listen on external TCP/IP and waits until start finishes
+		gosu postgres pg_ctl -D "$PGDATA" \
+			-o "-c listen_addresses='localhost'" \
+			-w start
+
+		: ${POSTGRES_USER:=postgres}
+		: ${POSTGRES_DB:=$POSTGRES_USER}
+		export POSTGRES_USER POSTGRES_DB
+
+		psql=( psql -v ON_ERROR_STOP=1 )
+
+		if [ "$POSTGRES_DB" != 'postgres' ]; then
+			"${psql[@]}" --username postgres <<-EOSQL
+				CREATE DATABASE "$POSTGRES_DB" ;
+			EOSQL
+			echo
+		fi
+
+		if [ "$POSTGRES_USER" = 'postgres' ]; then
+			op='ALTER'
+		else
+			op='CREATE'
+		fi
+		"${psql[@]}" --username postgres <<-EOSQL
+			$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
+		EOSQL
+		echo
+
+		fi
+
+		psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
+
+		echo
+		for f in /docker-entrypoint-initdb.d/*; do
+			case "$f" in
+				*.sh)     echo "$0: running $f"; . "$f" ;;
+				*.sql)    echo "$0: running $f"; "${psql[@]}" < "$f"; echo ;;
+				*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
+				*)        echo "$0: ignoring $f" ;;
+			esac
+			echo
+		done
+
+	if [ "x$REPLICATE_FROM" == "x" ]; then
+		gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
+	fi
+
+		echo
+		echo 'PostgreSQL init process complete; ready for start up.'
+		echo
+	fi
+
+    # We need this health check so we know when it's started up.
+    touch /tmp/.postgres_init_complete
+
+	exec gosu postgres "$@"
+fi
+
+exec "$@"

tests/docker-postgres-replication/setup-replication.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ "x$REPLICATE_FROM" == "x" ]; then
+
+cat >> ${PGDATA}/postgresql.conf <<EOF
+wal_level = hot_standby
+max_wal_senders = $PG_MAX_WAL_SENDERS
+wal_keep_segments = $PG_WAL_KEEP_SEGMENTS
+hot_standby = on
+EOF
+
+else
+
+cat > ${PGDATA}/recovery.conf <<EOF
+standby_mode = on
+primary_conninfo = 'host=${REPLICATE_FROM} port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}'
+trigger_file = '/tmp/touch_me_to_promote_to_me_master'
+EOF
+chown postgres ${PGDATA}/recovery.conf
+chmod 600 ${PGDATA}/recovery.conf
+
+fi