Merge branch 'postgres-12' into 'master'

Support PostgreSQL 12

See merge request postgres-ai/nancy!214

Author: NikolayS

README.md

@@ -41,7 +41,7 @@ Database experiments are needed when you:
 
 * Works anywhere where Docker can run (checked: Linux Ubuntu/Debian, macOS)
 * Experiments are conducted in a Docker container with extended Postgres setup
-* Supported Postgres versions: 11 (default), 10, 9.6
+* Supported Postgres versions: 12 (default), 11, 10, 9.6
 * Postgres config specified via options, may be partial
 * Supported locations for experimental runs:
   * Any machine with Docker installed

docker/Dockerfile

@@ -1,43 +1,44 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 ARG PG_SERVER_VERSION
 
-ENV PG_SERVER_VERSION=${PG_SERVER_VERSION:-11} \
+ENV PG_SERVER_VERSION=${PG_SERVER_VERSION:-12} \
     DEBIAN_FRONTEND=noninteractive
 
 # add custom FTS dictionaries
-ADD ./tsearch_data /usr/share/postgresql/$PG_SERVER_VERSION/tsearch_data
+ADD ./tsearch_data /usr/share/postgresql/${PG_SERVER_VERSION}/tsearch_data
 
 # logging ON; memory setting – for 2CPU/4096MB/SSD
 ADD ./postgresql_${PG_SERVER_VERSION}_tweak.conf /postgresql.tweak.conf
 
 # set up apt, add Postgres repo
 RUN apt-get update
-RUN apt-get install -y wget ca-certificates
+RUN apt-get install -y wget ca-certificates gnupg2
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main $PG_SERVER_VERSION" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" >> /etc/apt/sources.list.d/pgdg.list
 RUN apt-get update
 RUN apt-get install -y apt-utils
 
 # install additional utilites
 RUN apt-get install -y sudo git jq libjson-xs-perl vim
-RUN apt-get install -y sysbench s3cmd sudo bzip2 python-software-properties software-properties-common
+RUN apt-get install -y sysbench s3cmd sudo bzip2 software-properties-common
 RUN apt-get install -y sysstat iotop moreutils psmisc
 
 # install Postgres and postgres-specific packages
-RUN apt-get install -y postgresql-$PG_SERVER_VERSION
-RUN apt-get install -y postgresql-contrib-$PG_SERVER_VERSION
-RUN apt-get install -y postgresql-plpython-$PG_SERVER_VERSION
-RUN apt-get install -y postgresql-server-dev-$PG_SERVER_VERSION
-RUN apt-get install -y postgresql-$PG_SERVER_VERSION-dbg
-RUN apt-get install -y postgresql-$PG_SERVER_VERSION-pg-stat-kcache
-RUN apt-get install -y postgresql-client-11
+RUN apt-get install -y postgresql-${PG_SERVER_VERSION}
+RUN apt-get install -y postgresql-contrib-${PG_SERVER_VERSION}
+RUN apt-get install -y postgresql-server-dev-${PG_SERVER_VERSION}
+#RUN if [ "${PG_SERVER_VERSION}" != "12" ]; then apt-get install -y postgresql-${PG_SERVER_VERSION}-dbg; fi
+RUN apt-get install -y postgresql-${PG_SERVER_VERSION}-pg-stat-kcache
+RUN apt-get install -y postgresql-client-12
+RUN if [ "${PG_SERVER_VERSION}" == "12" ]; then apt-get install -y postgresql-plpython3-${PG_SERVER_VERSION}; fi
+RUN if [ "${PG_SERVER_VERSION}" != "12" ]; then apt-get install -y postgresql-plpython-${PG_SERVER_VERSION}; fi
 
 RUN apt-get install -y pgreplay
-RUN apt-get install -y pspg
+RUN apt-get update && apt-get install -y pspg
 RUN git clone https://github.com/NikolayS/postgres_dba.git /root/postgres_dba
-RUN apt-get install -y postgresql-$PG_SERVER_VERSION-repack
-RUN git clone https://github.com/darold/pgbadger.git /root/pgbadger && cd /root/pgbadger && git checkout "tags/v10.3"
+RUN apt-get install -y postgresql-${PG_SERVER_VERSION}-repack
+RUN git clone https://github.com/darold/pgbadger.git /root/pgbadger && cd /root/pgbadger && git checkout "tags/v11.1"
 
 # install FlameGraph and generic perf
 RUN git clone https://github.com/brendangregg/FlameGraph /root/FlameGraph
@@ -48,30 +49,30 @@ RUN path=$(ls /usr/lib/linux-tools/*generic/perf | head -n 1) && ln -s -f "$path
 # configure psql, configure Postgres
 RUN echo "\\set dba '\\\\\\\\i /root/postgres_dba/start.psql'" >> ~/.psqlrc
 RUN echo "\\setenv PAGER 'pspg -bX --no-mouse'" >> ~/.psqlrc
-RUN echo "local   all all trust" > /etc/postgresql/$PG_SERVER_VERSION/main/pg_hba.conf
-RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/$PG_SERVER_VERSION/main/pg_hba.conf
-RUN echo "listen_addresses='*'" >> /etc/postgresql/$PG_SERVER_VERSION/main/postgresql.conf
-RUN echo "log_filename='postgresql-$PG_SERVER_VERSION-main.log'" >> /etc/postgresql/$PG_SERVER_VERSION/main/postgresql.conf
+RUN echo "local   all all trust" > /etc/postgresql/${PG_SERVER_VERSION}/main/pg_hba.conf
+RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/${PG_SERVER_VERSION}/main/pg_hba.conf
+RUN echo "listen_addresses='*'" >> /etc/postgresql/${PG_SERVER_VERSION}/main/postgresql.conf
+RUN echo "log_filename='postgresql-${PG_SERVER_VERSION}-main.log'" >> /etc/postgresql/${PG_SERVER_VERSION}/main/postgresql.conf
 
 # prepare database 'test' with 'testuser'
 RUN /etc/init.d/postgresql start && psql -U postgres -c "create database test" \
     && psql -U postgres -d test -c "CREATE EXTENSION pg_repack" \
     && psql -U postgres -c "CREATE ROLE testuser LOGIN password 'testuser' superuser" && /etc/init.d/postgresql stop
 
 # apply 'tweaked' config
-RUN cat /postgresql.tweak.conf >> /etc/postgresql/$PG_SERVER_VERSION/main/postgresql.conf
+RUN cat /postgresql.tweak.conf >> /etc/postgresql/${PG_SERVER_VERSION}/main/postgresql.conf
 
 # prepare Postgres start script
 RUN echo "#!/bin/bash" > /pg_start.sh && chmod a+x /pg_start.sh
-RUN printf "sudo -u postgres /usr/lib/postgresql/$PG_SERVER_VERSION/bin/postgres -D /var/lib/postgresql/$PG_SERVER_VERSION/main -c config_file=/etc/postgresql/$PG_SERVER_VERSION/main/postgresql.conf \n" >> /pg_start.sh
+RUN printf "sudo -u postgres /usr/lib/postgresql/${PG_SERVER_VERSION}/bin/postgres -D /var/lib/postgresql/${PG_SERVER_VERSION}/main -c config_file=/etc/postgresql/${PG_SERVER_VERSION}/main/postgresql.conf \n" >> /pg_start.sh
 # infinite sleep to allow restarting Postgres
 RUN echo "/bin/bash -c \"trap : TERM INT; sleep infinity & wait\"" >> /pg_start.sh
 
 # generate english locale for iostat + iostat-tool
 RUN locale-gen en_US.UTF-8
 
 # install pip and iostat-tool
-RUN apt-get install curl
+RUN apt-get install -y curl
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
 RUN python3 get-pip.py
 RUN pip3 install iostat-tool

docker/postgresql_12_tweak.conf

@@ -0,0 +1,34 @@
+# IMPORTANT: on faster systems, you need to use your own memory-related settings!
+statement_timeout = 0
+
+log_destination = 'stderr,csvlog'
+logging_collector = on
+log_directory = '/var/log/postgresql'
+# log_filename – to be set dynamically
+log_min_messages = notice
+log_min_error_statement = notice
+log_min_duration_statement = -1 # rely on "auto_explain.log_min_duration = 0", avoid duplicates
+log_checkpoints = on
+log_connections = on
+log_disconnections = on
+log_line_prefix = '%t [%p]: [%l-1] db=%d,user=%u (%a,%h) '
+log_lock_waits = on
+log_replication_commands = on
+log_temp_files = 0
+log_autovacuum_min_duration = 0
+
+shared_preload_libraries = 'pg_stat_statements,auto_explain,pg_stat_kcache'
+
+pg_stat_statements.max = 5000
+pg_stat_statements.track = all
+pg_stat_statements.track_utility = on
+pg_stat_statements.save = on
+
+auto_explain.log_min_duration = 0
+auto_explain.log_analyze = on
+auto_explain.log_verbose = on
+auto_explain.log_buffers = on
+auto_explain.log_format = 'json'
+auto_explain.log_timing = on
+auto_explain.log_triggers = on
+auto_explain.log_nested_statements = on

docker/rebuild_push_images.sh

@@ -2,7 +2,7 @@
 # Rebuild all images and push to Docker Hub
 set -e -u -o pipefail
 
-VERSIONS="9.6 10 11"
+VERSIONS="9.6 10 11 12"
 
 if [[ ! -z ${1+x} ]]; then
   VERSIONS="$1"

help/nancy_run.md

@@ -82,7 +82,8 @@
 
   * '9.6'
   * '10'
-  * '11' (default)
+  * '11'
+  * '12' (default)
 
   Currently, there is no way to specify the minor version – it is always the
   most recent version, available in the official PostgreSQL APT repository (see

nancy_run.sh

@@ -19,7 +19,7 @@ DURATION_WRKLD=""
 VERBOSE_OUTPUT_REDIRECT=" > /dev/null"
 STDERR_DST="/dev/null"
 EBS_SIZE_MULTIPLIER=5
-POSTGRES_VERSION_DEFAULT=11
+POSTGRES_VERSION_DEFAULT=12
 AWS_BLOCK_DURATION=0
 MSG_PREFIX=""
 declare -a RUNS # i - delta_config  i+1 delta_ddl_do i+2 delta_ddl_undo