chore: Gather backtrace from core dump on CI (#736)

Add script that check health state of postgresql after run tests, because
some tests can lead to fail database with segmentation fault as it was
in #734. For easy debug this scenarios and provide helpfull information to
upstream we need check db state.

Right now health script extract backtrace from core dumps and print logs.
In the future list health checks can be increased.

Example build output https://travis-ci.org/Gordiychuk/pgjdbc/builds/194050719

Sample output:
The command "./.travis/travis_build.sh" exited with 1.
0.48s$ ./.travis/travis_check_postgres_health.sh
Detected core dump: /etc/postgresql/HEAD/main/core
Reading symbols from /usr/local/pgsql/bin/postgres...done.
[New LWP 16124]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `postgres: test test 127.0.0.1(54316) BIND                         '.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000000007e670d in exec_bind_message (input_message=0x7ffc29a05ee0)
    at postgres.c:1562
1562			(!IsTransactionExitStmt(psrc->raw_parse_tree->stmt) ||
(gdb) #0  0x00000000007e670d in exec_bind_message (input_message=0x7ffc29a05ee0)
    at postgres.c:1562
        portal_name = 0x1e47340 ""
        stmt_name = 0x1e47341 "S_3"

Closes #735

.travis.yml

@@ -31,6 +31,7 @@ script:
  - test -d "${JDK9_HOME}" || export JDK9_HOME=$(jdk_switcher home oraclejdk8)
  - envsubst < toolchains.xml > ~/.m2/toolchains.xml
  - ./.travis/travis_build.sh
  - ./.travis/travis_check_postgres_health.sh
  # To avoid useless S3 cache updates (https://github.com/travis-ci/travis-ci/issues/1441#issuecomment-67607074)
  #- mkdir /tmp/cache-trick
  #- mv $HOME/.m2/repository/org/postgresql /tmp/cache-trick/

.travis/travis_check_postgres_health.sh

@@ -0,0 +1,28 @@
#!/usr/bin/env bash

check_core_dump() {
    local status=0
    while IFS= read -r core_dump_file
    do
        status=1
        echo "Detected core dump: ${core_dump_file}"
        echo bt full | sudo gdb --quiet /usr/local/pgsql/bin/postgres "${core_dump_file}"
    done < <(sudo find "${PG_DATADIR}" -type f -iname "core*")
    return ${status}
}

print_logs() {
    if [[ "${PG_VERSION}" = "HEAD" ]]
    then
      sudo cat /tmp/postgres.log
    else
      cat "/var/log/postgresql/postgresql-${PG_VERSION}-main.log"
    fi
}

check_core_dump
if [[ $? -ne 0 ]]
then
    print_logs
    exit 1
fi

.travis/travis_configure_xa.sh

@@ -1,4 +1,4 @@
#!/usr/bin/env bash
set -x -e

sudo sed -i -e 's/#max_prepared_transactions = 0/max_prepared_transactions = 64/g' /etc/postgresql/${PG_VERSION}/main/postgresql.conf
sudo sed -i -e 's/#max_prepared_transactions = 0/max_prepared_transactions = 64/g' ${PG_DATADIR}/postgresql.conf

.travis/travis_install_head_postgres.sh

@@ -4,13 +4,29 @@ set -x -e
sudo service postgresql stop
sudo apt-get remove postgresql libpq-dev libpq5 postgresql-client-common postgresql-common -qq --purge
sudo apt-get -y install libxml2
sudo apt-get -y install gdb

git clone --depth=1 https://github.com/postgres/postgres.git
cd postgres
if [[ -z ${POSTGRES_SOURCE_SHA} ]]
then
    git clone --depth=1 https://github.com/postgres/postgres.git
    cd postgres
else
    git clone https://github.com/postgres/postgres.git
    cd postgres
    git checkout ${POSTGRES_SOURCE_SHA}
fi

# Build PostgreSQL from source
sudo ./configure --with-libxml && sudo make && sudo make install
sudo ln -s /usr/local/pgsql/bin/psql /usr/bin/psql
if [[ "${COMPILE_PG_WITH_DEBUG_FLAG}" == "Y" ]]
then
    sudo ./configure --enable-debug --with-libxml CFLAGS="-ggdb"
else
    sudo ./configure --with-libxml CFLAGS="-ggdb"
fi

sudo make && sudo make install
sudo ln -sf /usr/local/pgsql/bin/psql /usr/bin/psql

# Build contrib from source
cd contrib
sudo make all && sudo make install
@@ -20,8 +36,9 @@ LD_LIBRARY_PATH=/usr/local/pgsql/lib
export LD_LIBRARY_PATH
sudo /sbin/ldconfig /usr/local/pgsql/lib

sudo rm -rf ${PG_DATADIR}
sudo mkdir -p ${PG_DATADIR}
sudo chmod 777 ${PG_DATADIR}
sudo chown -R postgres:postgres ${PG_DATADIR}

sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -U postgres initdb"
sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -U postgres initdb"

.travis/travis_start_postgres.sh

@@ -7,11 +7,11 @@ then
elif [ "x${PG_VERSION}" = "xHEAD" ]
then
    #Start head postgres
    sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -w -t 300 -o '-p 5432' -l /tmp/postgres.log start"
    sudo su postgres -c "/usr/local/pgsql/bin/pg_ctl -D ${PG_DATADIR} -w -t 300 -c -o '-p 5432' -l /tmp/postgres.log start"
    sudo tail /tmp/postgres.log
elif [ "$XA" = "true" ] || [ "${REPLICATION}" = "Y" ]
then
    sudo service postgresql stop
    sudo service postgresql start ${PG_VERSION}
    sudo tail /var/log/postgresql/postgresql-${PG_VERSION}-main.log
fi
fi