From 02cda05012e12e3284b28cfe2bacf343e634dadc Mon Sep 17 00:00:00 2001 From: Marc Tamsky Date: Sat, 16 Sep 2017 02:41:38 -0700 Subject: [PATCH 1/3] add shellcheck linter to Makefile, add package to .travis.yml --- .travis.yml | 10 ++++++++++ Makefile | 13 +++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0bb1e296..42bdbbf0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,13 @@ services: language: go go: - '1.9' +addons: + apt: + sources: + - debian-sid # Grab ShellCheck from the Debian repo + packages: + - shellcheck + # Make sure we have p2 and the postgres client. before_install: - go get -v github.com/mattn/goveralls @@ -19,6 +26,7 @@ script: - make test-integration - make cover.out - $HOME/gopath/bin/goveralls -coverprofile=cover.out -service=travis-ci + after_success: - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS # Push a tagged build if a tag is found. @@ -28,11 +36,13 @@ after_success: fi # Push a latest version - if [ "$TRAVIS_BRANCH" == "master" ]; then docker push wrouesnel/postgres_exporter ; fi + env: global: - secure: RfoWQj5tEB/t3XL2tqJW7u7Qscpz1QBOfF9lMFpB4kAUMTtZU0zBbXfMo1JheGoJQQxD/7NLRHhbUWPT2489o3KKpRTQ7RHn3k8n5U7opH01bWX0+l/EPVmhlsKjSDSLGgmxz80j3I6C8ZV3qDUijSx7r90QUNHGbZtV7g+KtoUTpRV0zir/heK6qq9LHWNHbNsJyHK8qHmd6g1UzWIBaZPJ6a/n/rO2jq4uS1JR0VlIJPRF11HOLH8IjFQvVYpN7YbEslxyNsfQJUSP/7CghSLLVWPSATEjMm8a5GJVLc564+nYghm484psEtiMXkZ3n6ie7AT8aJrKfexWrwh2aCc+cK4PiyXrf4euZehZNYogmFCqWzd1LJKcN2uIkpBSuZQDm3e6c4qkkWGpx+RdFWtAMG8IgZLDbcuryxFNzMwHc2CJ009s9Zsa+g7D57csyR5LCZ8YtNGI3g8FmhwpCKvYkfKa9aijUEWyJMyT4Vhd/w7btMTuwYHgUQ85k4ov4Xjz5SNpAGgemig5G5w7PJj4NhGvIBz9weL154x/BFVjHOZZ6Y/bWgJIPoW1KM15x5K8QylWYEBUHtwiyVyXOxHqt6MOX1vYo1L37jMK88IErrfh/VmlxEhtN9wOghk8IudMfFwQtjIwiWlJf218wxMIzUjoyb5/25tU9f2OJrg= - secure: WP96T7yshE03XsPVc9ICbwZXZ6nVsQDCQ9NGKnIEQa4T1Swu5uLVzxjGeowHPykKbKphQmT8inMniBxB48OLc3VVqNxVI+00ppLPEf7n79w2wVbwFOEa6TiOdws+0wOphkeSYc0L+o2aSfoMKJHF+rVW9tmM2tszVjofYHhdWjqloc2pqsfOnqbR7icfpmzMWKyezIE20YOIBsiKZJTKXiZ1SaG9ExkNwuZ7L+HRF1yeI0OdAM4VfEzBK1Gwicy2BtrbyHnl4zgcSoIBmuzo+pNuvqyGmBn3C221M6ki7NoDJDfW5brcvDmiMODWGnka7iW0nt5aUbVtURM8BhWZR0uINo30aYjr4j39UBq8y+mqYV0dp/dMEmy2fa1mogr+DuHUNVSg59Au45AZeom8N6FT03nlg+RcG/tV1skvP/mn9n9CKsyfvC4Rf3jp4+LTiJ9JIch74MecRYVwlpKM+i8s6uDftt3mvxeIYdK+NEMcfwKMv8KTwuxRo/3KRhif7z2cOE+oMbT5POWO19bfboRPCs4xiMTcqpx8dJVs41SacY52PPgjFSnyVrKvzAyjn6mePjLhpoPZueHZuJYPNa9QC8JcASMlCI7lf2Eq+2Dmp2JxmndkRs/cIfHgmO4gtiNM7Vb/rlML1D/8LYPWU/Rdp82/yDffC0ugMNovXt0= - secure: RRQH4Tr94OblZoqls50BIoyK1OvK9fALs4pAq1Uk5dksY1NWnomheQzOaHzbVfMfXc4zXAzppZIqxUDGn8GiSLbtJL6pnxsxYNGoCGdS8lMjjKWXbCAs8TBJobi3krOOjqgbhOWTpyluTEShnBcg7CjrRQUa/ChS3uE5kl21/4eIl9Be6Q08Eqm3p1yvMAyAgIL6Y6oPAAtBa6zIsi2MSNlryz3RKHJO7AheilppYx3E8B03A+a/oqvTTcw6w/RwBYxB8MYfSLC0jSssZz5pUSX/byUaklGFhQLnKAzJyhrMOvRyMVcO4PHaLgVi1eAKQz6eLQh7uEiIqKh19cuvTbZHDgu8zMpLDTxOW9U95e4kbjOZ5pWZ7E5QTrb24RZIt42JGbmme7PWOvy3zNbWHwfwiOF1qwYwISUcj2KFCpes8mDGt6iv46LfdlU0uoZdZu3MAiTiW0u2SD5hIeFq4XYesPtkS/TKFoAbB5Tu1qbxdmYu5NqmfvmxsmeNEm4inFJ5ap3fRRCVo668Z6qRMuQ1URcEfOz8iEftP9CnwSOXRuiuMo+W9GgckRuDZcPyQMCftq8+PhB+SjK57zrAd4Kxqf6kVHV16tcWqmFjfJJUFqmL+gpjT/VMEVDY2FOnbOARjkeLTjVC4dADBjxfJ6wmlLrfHdUm4GinbaHq0iA= + deploy: skip_cleanup: true provider: releases diff --git a/Makefile b/Makefile index 26ccb8a65..d186cf666 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,14 @@ LINTER_DEADLINE ?= 30s export PATH := $(TOOLDIR)/bin:$(PATH) SHELL := env PATH=$(PATH) /bin/bash +# We may want to move shell scripts to have a common *.sh suffix, +# allowing us to use 'find *.sh', like we do for GO_SRC. +hash := \# +SHELL_SRC := $(shell grep -l '$(hash)!/bin/bash' -r . \ + --exclude-dir 'tools' \ + --exclude Makefile) -all: style lint test postgres_exporter +all: style lint shellcheck test postgres_exporter # Cross compilation (e.g. if you are on a Mac) cross: docker-build docker @@ -35,6 +41,9 @@ docker: postgres_exporter style: tools gometalinter --disable-all --enable=gofmt --vendor +shellcheck: + shellcheck $(SHELL_SRC) + lint: tools @echo Using $(CONCURRENT_LINTERS) processes gometalinter -j $(CONCURRENT_LINTERS) --deadline=$(LINTER_DEADLINE) --disable=gotype --disable=gocyclo $(GO_DIRS) @@ -76,4 +85,4 @@ tools: clean: rm -f postgres_exporter postgres_exporter_integration_test -.PHONY: tools docker-build docker lint fmt test vet push cross clean +.PHONY: tools docker-build docker lint shellcheck fmt test vet push cross clean From 1604b72e051290d06f640dafc68f54c52078edc7 Mon Sep 17 00:00:00 2001 From: Marc Tamsky Date: Sat, 16 Sep 2017 02:42:33 -0700 Subject: [PATCH 2/3] update shell scripts to address new linter's findings and harmonize styles across scripts. --- postgres_exporter_integration_test_script | 8 +- postgres_metrics_added_and_removed | 4 +- postgres_metrics_parse_script | 5 +- .../docker-entrypoint.sh | 8 +- .../setup-replication.sh | 10 +- tests/test-smoke | 204 +++++++++++------- 6 files changed, 141 insertions(+), 98 deletions(-) diff --git a/postgres_exporter_integration_test_script b/postgres_exporter_integration_test_script index 03f2945af..d6a5ee35e 100755 --- a/postgres_exporter_integration_test_script +++ b/postgres_exporter_integration_test_script @@ -7,9 +7,9 @@ shift output_cov=$1 shift -echo "mode: count" > $output_cov +echo "mode: count" > "${output_cov}" test_cov=$(mktemp) -$test_binary -test.coverprofile=$test_cov $@ -tail -n +2 $test_cov >> $output_cov -rm -f $test_cov +${test_binary} -test.coverprofile="${test_cov}" "$@" +tail -n +2 "${test_cov}" >> "${output_cov}" +rm -f "${test_cov}" diff --git a/postgres_metrics_added_and_removed b/postgres_metrics_added_and_removed index d33618904..f3e1684ce 100755 --- a/postgres_metrics_added_and_removed +++ b/postgres_metrics_added_and_removed @@ -7,5 +7,5 @@ version=$2 old_version=$3 new_version=$4 -comm -23 $old_version $new_version > .metrics.${type}.${version}.removed -comm -13 $old_version $new_version > .metrics.${type}.${version}.added +comm -23 "${old_version}" "${new_version}" > ".metrics.${type}.${version}.removed" +comm -13 "${old_version}" "${new_version}" > ".metrics.${type}.${version}.added" diff --git a/postgres_metrics_parse_script b/postgres_metrics_parse_script index 393e39a1a..a499fb668 100755 --- a/postgres_metrics_parse_script +++ b/postgres_metrics_parse_script @@ -7,10 +7,9 @@ for raw_prom in $(echo .*.prom) ; do # Strip, sort and deduplicate the label names - cat $raw_prom | grep -v '#' | \ + grep -v "#" "${raw_prom}" | \ rev | cut -d' ' -f2- | \ rev | cut -d'{' -f1 | \ sort | \ - uniq > ${raw_prom}.unique - + uniq > "${raw_prom}.unique" done diff --git a/tests/docker-postgres-replication/docker-entrypoint.sh b/tests/docker-postgres-replication/docker-entrypoint.sh index 0203e4b79..0b085514d 100755 --- a/tests/docker-postgres-replication/docker-entrypoint.sh +++ b/tests/docker-postgres-replication/docker-entrypoint.sh @@ -36,12 +36,12 @@ if [ "$1" = 'postgres' ]; then if [ "x$REPLICATE_FROM" == "x" ]; then eval "gosu postgres initdb $POSTGRES_INITDB_ARGS" else - until ping -c 1 -W 1 ${REPLICATE_FROM} + until ping -c 1 -W 1 "${REPLICATE_FROM}" do echo "Waiting for master to ping..." sleep 1s done - until gosu postgres pg_basebackup -h ${REPLICATE_FROM} -D ${PGDATA} -U ${POSTGRES_USER} -vP -w + until gosu postgres pg_basebackup -h "${REPLICATE_FROM}" -D "${PGDATA}" -U "${POSTGRES_USER}" -vP -w do echo "Waiting for master to connect..." sleep 1s @@ -84,10 +84,13 @@ if [ "$1" = 'postgres' ]; then -o "-c listen_addresses='localhost'" \ -w start + # shellcheck disable=SC2086 : ${POSTGRES_USER:=postgres} + # shellcheck disable=SC2086 : ${POSTGRES_DB:=$POSTGRES_USER} export POSTGRES_USER POSTGRES_DB + # shellcheck disable=SC2191 psql=( psql -v ON_ERROR_STOP=1 ) if [ "$POSTGRES_DB" != 'postgres' ]; then @@ -112,6 +115,7 @@ if [ "$1" = 'postgres' ]; then psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" ) echo + # shellcheck disable=SC1090 for f in /docker-entrypoint-initdb.d/*; do case "$f" in *.sh) echo "$0: running $f"; . "$f" ;; diff --git a/tests/docker-postgres-replication/setup-replication.sh b/tests/docker-postgres-replication/setup-replication.sh index 460c54891..ea3a794b2 100755 --- a/tests/docker-postgres-replication/setup-replication.sh +++ b/tests/docker-postgres-replication/setup-replication.sh @@ -1,8 +1,8 @@ #!/bin/bash -if [ "x$REPLICATE_FROM" == "x" ]; then +if [[ -z "$REPLICATE_FROM" ]]; then -cat >> ${PGDATA}/postgresql.conf <> "${PGDATA}/postgresql.conf" < ${PGDATA}/recovery.conf < "${PGDATA}/recovery.conf" <&2 exit 1 fi - - if [ -z $port ]; then + + if [[ -z ${port} ]]; then echo "No port specified." 1>&2 exit 1 fi - + + # shellcheck disable=SC2155 local wait_start=$(date +%s) echo "Waiting for postgres to start listening..." - while ! pg_isready --host=$ip --port=$port &> /dev/null; do - if [ $(( $(date +%s) - $wait_start )) -gt $TIMEOUT ]; then + while ! pg_isready --host="${ip}" --port="${port}" &> /dev/null; do + if [[ $(( $(date +%s) - wait_start )) -gt ${TIMEOUT} ]]; then echo "Timed out waiting for postgres to start!" 1>&2 - exit 1 + exit 1 fi sleep 1 done } wait_for_exporter() { + # shellcheck disable=SC2155 local wait_start=$(date +%s) echo "Waiting for exporter to start..." - while ! nc -z localhost $exporter_port ; do - if [ $(( $(date +%s) - $wait_start )) -gt $TIMEOUT ]; then + while ! nc -z localhost ${exporter_port} ; do + if [[ $(( $(date +%s) - wait_start )) -gt ${TIMEOUT} ]]; then echo "Timed out waiting for exporter!" 1>&2 exit 1 fi @@ -69,92 +96,105 @@ smoketest_postgres() { local CONTAINER_NAME=postgres_exporter-test-smoke local TIMEOUT=30 local IMAGE_NAME=postgres - - local CUR_IMAGE=$IMAGE_NAME:$version - + + local CUR_IMAGE=${IMAGE_NAME}:${version} + echo "#######################" echo "Standalone Postgres $version" echo "#######################" - local docker_cmd="docker run -d -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD -p 127.0.0.1:55432:5432 $CUR_IMAGE" - echo "Docker Cmd: $docker_cmd" - - CONTAINER_NAME=$($docker_cmd) - trap "docker logs $CONTAINER_NAME ; docker kill $CONTAINER_NAME ; docker rm -v $CONTAINER_NAME; exit 1" EXIT INT TERM + local docker_cmd="docker run -d -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -p 127.0.0.1:55432:5432 ${CUR_IMAGE}" + echo "Docker Cmd: ${docker_cmd}" + + CONTAINER_NAME=$( ${docker_cmd} ) + trap 'docker logs ${CONTAINER_NAME} ; + docker kill ${CONTAINER_NAME} ; + docker rm -v ${CONTAINER_NAME}; exit 1' EXIT INT TERM wait_for_postgres localhost 55432 # Run the test binary. - DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@localhost:55432/?sslmode=disable" $test_binary --log.level=debug || exit $? + DATA_SOURCE_NAME="postgresql://postgres:${POSTGRES_PASSWORD}@localhost:55432/?sslmode=disable" \ + ${test_binary} --log.level=debug || exit $? # Extract a raw metric list. - DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@localhost:55432/?sslmode=disable" $postgres_exporter --log.level=debug --web.listen-address=:$exporter_port & + DATA_SOURCE_NAME="postgresql://postgres:${POSTGRES_PASSWORD}@localhost:55432/?sslmode=disable" \ + ${postgres_exporter} --log.level=debug --web.listen-address=:${exporter_port} & exporter_pid=$! + # shellcheck disable=SC2064 trap "docker logs $CONTAINER_NAME ; docker kill $CONTAINER_NAME ; docker rm -v $CONTAINER_NAME; kill $exporter_pid; exit 1" EXIT INT TERM wait_for_exporter # Dump the metrics to a file. - wget -q -O - http://localhost:$exporter_port/metrics 1> $METRICS_DIR/.metrics.single.$version.prom - if [ "$?" != "0" ]; then - echo "Failed on postgres $version ($DOCKER_IMAGE)" 1>&2 - kill $exporter_pid - exit 1 - fi - - kill $exporter_pid - docker kill $CONTAINER_NAME - docker rm -v $CONTAINER_NAME + wget -q -O - \ + http://localhost:${exporter_port}/metrics \ + 1>"${METRICS_DIR}/.metrics.single.${version}.prom" || + { + echo "Failed on postgres ${version} (${DOCKER_IMAGE})" 1>&2 + kill ${exporter_pid} + exit 1 + } + + kill ${exporter_pid} + docker kill "${CONTAINER_NAME}" + docker rm -v "${CONTAINER_NAME}" trap - EXIT INT TERM - + echo "#######################" - echo "Replicated Postgres $version" + echo "Replicated Postgres ${version}" echo "#######################" old_pwd=$(pwd) - cd docker-postgres-replication + cd docker-postgres-replication || exit 1 + - VERSION=$version p2 -t Dockerfile.p2 -o Dockerfile - if [ "$?" != "0" ]; then - echo "Templating failed" 1>&2 - exit 1 - fi + VERSION=${version} p2 -t Dockerfile.p2 -o Dockerfile || { + echo "Templating failed" 1>&2 ; + exit 1 ; + } + trap "docker-compose logs; docker-compose down ; docker-compose rm -v; exit 1" EXIT INT TERM - local compose_cmd="POSTGRES_PASSWORD=$POSTGRES_PASSWORD docker-compose up -d --force-recreate --build" - echo "Compose Cmd: $compose_cmd" - eval $compose_cmd - + local compose_cmd="POSTGRES_PASSWORD=${POSTGRES_PASSWORD} docker-compose up -d --force-recreate --build" + echo "Compose Cmd: ${compose_cmd}" + eval "${compose_cmd}" + master_container=$(docker-compose ps -q pg-master) slave_container=$(docker-compose ps -q pg-slave) - master_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $master_container) - slave_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $slave_container) - echo "Got master IP: $master_ip" - wait_for_postgres $master_ip 5432 - wait_for_postgres $slave_ip 5432 - - DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@$master_ip:5432/?sslmode=disable" $test_binary --log.level=debug || exit $? - - DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@$master_ip:5432/?sslmode=disable" $postgres_exporter --log.level=debug --web.listen-address=:$exporter_port & + master_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${master_container}") + slave_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${slave_container}") + echo "Got master IP: ${master_ip}" + wait_for_postgres "${master_ip}" 5432 + wait_for_postgres "${slave_ip}" 5432 + + DATA_SOURCE_NAME="postgresql://postgres:${POSTGRES_PASSWORD}@${master_ip}:5432/?sslmode=disable" ${test_binary} --log.level=debug || exit $? + + DATA_SOURCE_NAME="postgresql://postgres:${POSTGRES_PASSWORD}@${master_ip}:5432/?sslmode=disable" ${postgres_exporter} --log.level=debug --web.listen-address=:${exporter_port} & exporter_pid=$! - trap "docker-compose logs; docker-compose down ; docker-compose rm -v ; kill $exporter_pid; exit 1" EXIT INT TERM + # shellcheck disable=SC2064 + trap "docker-compose logs; docker-compose down ; docker-compose rm -v ; kill ${exporter_pid}; exit 1" EXIT INT TERM wait_for_exporter - wget -q -O - http://localhost:$exporter_port/metrics 1> $METRICS_DIR/.metrics.replicated.$version.prom - if [ "$?" != "0" ]; then - echo "Failed on postgres $version ($DOCKER_IMAGE)" 1>&2 - exit 1 - fi + wget -q -O - \ + http://localhost:${exporter_port}/metrics \ + 1>"${METRICS_DIR}/.metrics.replicated.${version}.prom" || + { + echo "Failed on postgres ${version} (${DOCKER_IMAGE})" 1>&2 ; + exit 1 ; + } - kill $exporter_pid + kill ${exporter_pid} docker-compose down docker-compose rm -v trap - EXIT INT TERM - - cd $old_pwd + + cd "${old_pwd}" || exit 1 } +### START OF MAIN PROGRAM + # Start pulling the docker images in advance -for version in ${VERSIONS[@]}; do - docker pull postgres:$version > /dev/null & +for version in "${VERSIONS[@]}"; do + docker pull "postgres:${version}" > /dev/null & done -for version in ${VERSIONS[@]}; do - echo "Testing postgres version $version" - smoketest_postgres $version +for version in "${VERSIONS[@]}"; do + echo "Testing postgres version ${version}" + smoketest_postgres "${version}" done From e977ad3cc542b66002a8362f257598f246a916b5 Mon Sep 17 00:00:00 2001 From: Will Rouesnel Date: Sun, 11 Nov 2018 14:28:11 +1100 Subject: [PATCH 3/3] Fix bad merge syntax. --- cmd/postgres_exporter/tests/test-smoke | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/tests/test-smoke b/cmd/postgres_exporter/tests/test-smoke index 5695a8f48..010891afc 100755 --- a/cmd/postgres_exporter/tests/test-smoke +++ b/cmd/postgres_exporter/tests/test-smoke @@ -3,7 +3,7 @@ SOURCE="${BASH_SOURCE[0]}" METRICS_DIR=$(pwd) -done + # resolve $SOURCE until the file is no longer a symlink while [[ -h "${SOURCE}" ]]; do DIR="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"