chore: Gather backtrace from core dump on CI (#736)

Add script that check health state of postgresql after run tests, because
some tests can lead to fail database with segmentation fault as it was
in #734. For easy debug this scenarios and provide helpfull information to
upstream we need check db state.

Right now health script extract backtrace from core dumps and print logs.
In the future list health checks can be increased.

Example build output https://travis-ci.org/Gordiychuk/pgjdbc/builds/194050719

Sample output:
The command "./.travis/travis_build.sh" exited with 1.
0.48s$ ./.travis/travis_check_postgres_health.sh
Detected core dump: /etc/postgresql/HEAD/main/core
Reading symbols from /usr/local/pgsql/bin/postgres...done.
[New LWP 16124]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `postgres: test test 127.0.0.1(54316) BIND                         '.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000000007e670d in exec_bind_message (input_message=0x7ffc29a05ee0)
    at postgres.c:1562
1562			(!IsTransactionExitStmt(psrc->raw_parse_tree->stmt) ||
(gdb) #0  0x00000000007e670d in exec_bind_message (input_message=0x7ffc29a05ee0)
    at postgres.c:1562
        portal_name = 0x1e47340 ""
        stmt_name = 0x1e47341 "S_3"

Closes #735

.travis.yml

@@ -31,6 +31,7 @@ script:
   - test -d "${JDK9_HOME}" || export JDK9_HOME=$(jdk_switcher home oraclejdk8)
   - envsubst < toolchains.xml > ~/.m2/toolchains.xml
   - ./.travis/travis_build.sh
+  - ./.travis/travis_check_postgres_health.sh
   # To avoid useless S3 cache updates (https://github.com/travis-ci/travis-ci/issues/1441#issuecomment-67607074)
   #- mkdir /tmp/cache-trick
   #- mv $HOME/.m2/repository/org/postgresql /tmp/cache-trick/

.travis/travis_check_postgres_health.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+check_core_dump() {
+    local status=0
+    while IFS= read -r core_dump_file
+    do
+        status=1
+        echo "Detected core dump: ${core_dump_file}"
+        echo bt full | sudo gdb --quiet /usr/local/pgsql/bin/postgres "${core_dump_file}"
+    done < <(sudo find "${PG_DATADIR}" -type f -iname "core*")
+    return ${status}
+}
+
+print_logs() {
+    if [[ "${PG_VERSION}" = "HEAD" ]]
+    then
+      sudo cat /tmp/postgres.log
+    else
+      cat "/var/log/postgresql/postgresql-${PG_VERSION}-main.log"
+    fi
+}
+
+check_core_dump
+if [[ $? -ne 0 ]]
+then
+    print_logs
+    exit 1
+fi

.travis/travis_configure_xa.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -x -e
 
-sudo sed -i -e 's/#max_prepared_transactions = 0/max_prepared_transactions = 64/g' /etc/postgresql/${PG_VERSION}/main/postgresql.conf
+sudo sed -i -e 's/#max_prepared_transactions = 0/max_prepared_transactions = 64/g' ${PG_DATADIR}/postgresql.conf

.travis/travis_install_head_postgres.sh

@@ -4,13 +4,29 @@ set -x -e
 sudo service postgresql stop
 sudo apt-get remove postgresql libpq-dev libpq5 postgresql-client-common postgresql-common -qq --purge
 sudo apt-get -y install libxml2
+sudo apt-get -y install gdb
 
-git clone --depth=1 https://github.com/postgres/postgres.git
-cd postgres
+if [[ -z ${POSTGRES_SOURCE_SHA} ]]
+then
+    git clone --depth=1 https://github.com/postgres/postgres.git
+    cd postgres
+else
+    git clone https://github.com/postgres/postgres.git
+    cd postgres
+    git checkout ${POSTGRES_SOURCE_SHA}
+fi
 
 # Build PostgreSQL from source
-sudo ./configure --with-libxml && sudo make && sudo make install
-sudo ln -s /usr/local/pgsql/bin/psql /usr/bin/psql
+if [[ "${COMPILE_PG_WITH_DEBUG_FLAG}" == "Y" ]]
+then
+    sudo ./configure --enable-debug --with-libxml CFLAGS="-ggdb"
+else
+    sudo ./configure --with-libxml CFLAGS="-ggdb"
+fi
+
+sudo make && sudo make install
+sudo ln -sf /usr/local/pgsql/bin/psql /usr/bin/psql
+
 # Build contrib from source
 cd contrib
 sudo make all && sudo make install
@@ -20,8 +36,9 @@ LD_LIBRARY_PATH=/usr/local/pgsql/lib
 export LD_LIBRARY_PATH
 sudo /sbin/ldconfig /usr/local/pgsql/lib
 
+sudo rm -rf ${PG_DATADIR}
 sudo mkdir -p ${PG_DATADIR}
 sudo chmod 777 ${PG_DATADIR}
 sudo chown -R postgres:postgres ${PG_DATADIR}
 
-sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -U postgres initdb"
+sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -U postgres initdb"

.travis/travis_start_postgres.sh

@@ -7,11 +7,11 @@ then
 elif [ "x${PG_VERSION}" = "xHEAD" ]
 then
     #Start head postgres
-    sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -w -t 300 -o '-p 5432' -l /tmp/postgres.log start"
+    sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -w -t 300 -c -o '-p 5432' -l /tmp/postgres.log start"
     sudo tail /tmp/postgres.log
 elif [ "$XA" = "true" ] || [ "${REPLICATION}" = "Y" ]
 then
     sudo service postgresql stop
     sudo service postgresql start ${PG_VERSION}
     sudo tail /var/log/postgresql/postgresql-${PG_VERSION}-main.log
-fi
+fi