PostgreSQL 11 support + Dockerfile improvements

docker/Dockerfile

@@ -10,28 +10,36 @@ ADD ./tsearch_data /usr/share/postgresql/$PG_SERVER_VERSION/tsearch_data
 # logging ON; memory setting – for 2CPU/4096MB/SSD
 ADD ./postgresql_${PG_SERVER_VERSION}_tweak.conf /postgresql.tweak.conf
 
+RUN if [ "$PG_SERVER_VERSION" = "11" ]; then \
+      export PG_CLIENT_VERSION=11; \
+    else \
+      export PG_CLIENT_VERSION=10; \
+    fi
+
+RUN if [ "$PG_SERVER_VERSION" = "11" ]; then \
+      export PG_EXOTIC="postgresql-$PG_SERVER_VERSION-rum postgresql-$PG_SERVER_VERSION-powa"; \
+    else \
+      export PG_EXOTIC="postgresql-$PG_SERVER_VERSION-rum postgresql-$PG_SERVER_VERSION-powa postgresql-$PG_SERVER_VERSION-plsh"; \
+    fi
+
 # install Postgres and postgres-specific software:
 #   - desired version of Postgres server,
-#   - psql version 10
+#   - psql version 10+ (required by postgres_dba toolset)
 #   - postgres_dba and pspg
-#   - pgbadger (modified, not lowercasing DB object names, auto_explain compatibility)
+#   - pgbadger 10+
+#   - postgres_dba toolset
 RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \
-      && echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main"> /etc/apt/sources.list.d/pgdg.list \
-      && apt-get update && apt-get install -y sudo postgresql-$PG_SERVER_VERSION \
-      && apt-get install -y postgresql-contrib-$PG_SERVER_VERSION postgresql-plpython-$PG_SERVER_VERSION \
-      && apt-get install -y postgresql-$PG_SERVER_VERSION-plsh postgresql-server-dev-$PG_SERVER_VERSION \
-      && apt-get install -y postgresql-$PG_SERVER_VERSION-rum \
-      && apt-get install -y git postgresql-client-10 pspg pgreplay jq etcd libjson-xs-perl \
-      && perl -MCPAN -e'install Text::CSV_XS' \
+      && echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main $PG_SERVER_VERSION" > /etc/apt/sources.list.d/pgdg.list \
+      && apt-get update \
+      && apt-get install -y sudo postgresql-$PG_SERVER_VERSION \
+      && apt-get install -y postgresql-contrib-$PG_SERVER_VERSION postgresql-plpython-$PG_SERVER_VERSION postgresql-server-dev-$PG_SERVER_VERSION \
+      && apt-get install -y "$PG_EXOTIC" \
+      && apt-get install -y git postgresql-client-$PG_CLIENT_VERSION pspg pgreplay jq etcd libjson-xs-perl vim \
       && git clone https://github.com/NikolayS/postgres_dba.git /root/postgres_dba \
-      && git clone https://github.com/darold/pgbadger.git /root/pgbadger
-
-# additionally, install newer NodeJS, npm, Sqitch, and more
-RUN wget -q -S -O - https://deb.nodesource.com/setup_8.x | sudo bash \
-      && apt-get install -y s3cmd sudo bzip2 python-software-properties software-properties-common \
-      && apt-get install -y build-essential cpanminus libdbd-pg-perl nginx netcat npm \
-      && npm install -g newman ava \
-      && sudo cpanm --quiet --notest App::Sqitch
+      && git clone https://github.com/darold/pgbadger.git /root/pgbadger && cd /root/pgbadger && git checkout "tags/v10.1"
+
+# additional software
+RUN apt-get install -y s3cmd sudo bzip2 python-software-properties software-properties-common
 
 # configure psql, configure postgres & check postgres start & stop & prepare start script
 RUN echo "\\set dba '\\\\\\\\i /root/postgres_dba/start.psql'" >> ~/.psqlrc \

docker/README.md

@@ -1,7 +1,13 @@
 How to build/rebuild:
 
 ```bash
-docker build --build-arg PG_SERVER_VERSION=9.6 -t postgresmen/postgres-with-stuff:pg9.6 .
+export NANCY_DOCKER_PG_VERSION=9.6
+
+docker build \
+  --build-arg "PG_SERVER_VERSION=${NANCY_DOCKER_PG_VERSION}" \
+  -t "postgresmen/postgres-nancy:${NANCY_DOCKER_PG_VERSION}" .
+
 docker login # you must be registered, go to hub.docker.com
-docker push postgresmen/postgres-with-stuff:pg9.6
+
+docker push "postgresmen/postgres-nancy:${NANCY_DOCKER_PG_VERSION}"
 ```

docker/postgresql_10_tweak.conf

@@ -1,18 +1,5 @@
-# Assume we have machine with 2CPU/4096MB/SSD (CircleCI default)
 # IMPORTANT: on faster systems, you need to use your own memory-related settings!
-statement_timeout = 300s
-work_mem = 32MB # warning: tune it if you expect *many* concurrent connections
-shared_buffers = 3GB
-effective_cache_size = 1GB
-maintenance_work_mem = 512MB
-checkpoint_completion_target = 0.7
-wal_buffers = 16MB
-random_page_cost = 1.1
-effective_io_concurrency = 200
-# do not use parallel execution to avoid issues with analysis
-max_worker_processes = 0
-max_parallel_workers_per_gather = 0
-max_parallel_workers = 0
+statement_timeout = 0
 
 log_destination = 'stderr,csvlog'
 logging_collector = on

docker/postgresql_11_tweak.conf

@@ -0,0 +1,34 @@
+# IMPORTANT: on faster systems, you need to use your own memory-related settings!
+statement_timeout = 0
+
+log_destination = 'stderr,csvlog'
+logging_collector = on
+log_directory = '/var/log/postgresql'
+# log_filename – to be set dynamically
+log_min_messages = notice
+log_min_error_statement = notice
+log_min_duration_statement = -1 # rely on "auto_explain.log_min_duration = 0", avoid duplicates
+log_checkpoints = on
+log_connections = on
+log_disconnections = on
+log_line_prefix = '%t [%p]: [%l-1] db=%d,user=%u (%a,%h) '
+log_lock_waits = on
+log_replication_commands = on
+log_temp_files = 0
+log_autovacuum_min_duration = 0
+
+shared_preload_libraries = 'pg_stat_statements,auto_explain'
+
+pg_stat_statements.max = 5000
+pg_stat_statements.track = all
+pg_stat_statements.track_utility = on
+pg_stat_statements.save = on
+
+auto_explain.log_min_duration = 0
+auto_explain.log_analyze = on
+auto_explain.log_verbose = on
+auto_explain.log_buffers = on
+auto_explain.log_format = 'json'
+auto_explain.log_timing = on
+auto_explain.log_triggers = on
+auto_explain.log_nested_statements = on

docker/postgresql_9.6_tweak.conf

@@ -1,17 +1,5 @@
-# Assume we have machine with 2CPU/4096MB/SSD (CircleCI default)
 # IMPORTANT: on faster systems, you need to use your own memory-related settings!
-statement_timeout = 300s
-work_mem = 32MB # warning: tune it if you expect *many* concurrent connections
-shared_buffers = 3GB
-effective_cache_size = 1GB
-maintenance_work_mem = 512MB
-checkpoint_completion_target = 0.7
-wal_buffers = 16MB
-random_page_cost = 1.1
-effective_io_concurrency = 200
-# do not use parallel execution to avoid issues with analysis
-max_worker_processes = 0
-max_parallel_workers_per_gather = 0
+statement_timeout = 0
 
 log_destination = 'stderr,csvlog'
 logging_collector = on

nancy_run.sh

@@ -931,13 +931,13 @@ if [[ "$RUN_ON" == "localhost" ]]; then
     if [[ -z ${DB_LOCAL_PGDATA+x} ]]; then
       CONTAINER_HASH=$(docker run --name="pg_nancy_${CURRENT_TS}" \
         -v $TMP_PATH:/machine_home \
-        -dit "postgresmen/postgres-with-stuff:pg${PG_VERSION}" \
+        -dit "postgresmen/postgres-nancy:${PG_VERSION}" \
       )
     else
       CONTAINER_HASH=$(docker run --name="pg_nancy_${CURRENT_TS}" \
         -v $TMP_PATH:/machine_home \
         -v $DB_LOCAL_PGDATA:/pgdata \
-        -dit "postgresmen/postgres-with-stuff:pg${PG_VERSION}" \
+        -dit "postgresmen/postgres-nancy:${PG_VERSION}" \
       )
     fi
   else
@@ -998,7 +998,7 @@ elif [[ "$RUN_ON" == "aws" ]]; then
       -v /home/ubuntu:/machine_home \
       -v /home/storage:/storage \
       -v /home/backup:/backup \
-      -dit "postgresmen/postgres-with-stuff:pg${PG_VERSION}"
+      -dit "postgresmen/postgres-nancy:${PG_VERSION}"
   )
   DOCKER_CONFIG=$(docker-machine config $DOCKER_MACHINE)
   msg "  To connect container machine use:"
@@ -1343,7 +1343,9 @@ function pg_config_init() {
     if [[ $CPU_CNT > 1 ]]; then # Only for postgres 9.6+!
       local max_worker_processes="$CPU_CNT"
       local max_parallel_workers_per_gather="$(echo "print round($CPU_CNT / 2)" | python | awk -F '.' '{print $1}')"
-      local max_parallel_workers="$CPU_CNT"
+      if [[ ! "$PG_VERSION" = "9.6" ]]; then # the following is only for 10+ (and we don't support 9.5 and older)
+        local max_parallel_workers="$CPU_CNT"
+      fi
     fi
 
     docker_exec bash -c "echo '# AUTO-TUNED KNOBS:' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
@@ -1354,7 +1356,9 @@ function pg_config_init() {
     docker_exec bash -c "echo 'effective_io_concurrency = $effective_io_concurrency' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
     docker_exec bash -c "echo 'max_worker_processes = $max_worker_processes' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
     docker_exec bash -c "echo 'max_parallel_workers_per_gather = $max_parallel_workers_per_gather' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
-    docker_exec bash -c "echo 'max_parallel_workers = $max_parallel_workers' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
+    if [[ ! "$PG_VERSION" = "9.6" ]]; then # the following is only for 10+ (and we don't support 9.5 and older)
+      docker_exec bash -c "echo 'max_parallel_workers = $max_parallel_workers' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
+    fi
     restart_needed=true
   fi
   if [[ ! -z ${DELTA_CONFIG+x} ]]; then # if DELTA_CONFIG is not empty, restart will be done later
@@ -1591,4 +1595,4 @@ echo -e "  Queries:            "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILEN
 echo -e "  Query groups:       "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.normalyzed_info | length')
 echo -e "  Errors:             "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.errors_number')
 echo -e "  Errors groups:      "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.error_info | length')
-echo -e "------------------------------------------------------------------------------"
+echo -e "------------------------------------------------------------------------------"