gdal_with_coverage changes

.travis.yml

@@ -1,175 +1,231 @@
 # This is the config file for building GDAL and running its autotest suite
 # with Travis-ci.org
 
-branches:
-  except:
-  - /^(cherry-pick-)?backport-\d+-to-/
-
 matrix:
-  fast_finish: true
   include:
-
     - os: linux
       compiler: gcc
       language: cpp
       sudo: required
       dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=ubuntu_1804
-        - DETAILS=""
 
-    - os: linux
-      compiler: clang
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=trusty_clang
-        - DETAILS="optimized build, no libtool"
-    - os: linux
-      compiler: clang
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=mingw_w64
-        - DETAILS="optimized build, libtool"
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=python3
-        - DETAILS=""
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: false
-      dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      apt:
-        packages:
-          - ccache
-      env:
-        - BUILD_NAME=android
-        - DETAILS=""
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: false
-      dist: precise
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      apt:
-        packages:
-          - ccache
-      env:
-        - BUILD_NAME=big_endian
-        - DETAILS=""
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: required
-      dist: precise
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=gcc52_stdcpp14_sanitize
-        - DETAILS=""
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=ubuntu_1604
-        - DETAILS=""
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=trusty_32bit
-        - DETAILS=""
-    - osx_image: xcode8
-      language: objective-c
-      compiler: clang
-      cache:
-        apt: true
-        directories:
-          - $HOME/.ccache
-      env:
-        - BUILD_NAME=osx
-        - DETAILS="debug build, libtool"
+before_install:
+  - sudo mv /etc/apt/sources.list.d/pgdg* /tmp
+  - sudo apt-get remove libpq* postgresql*
+  - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
+#  - sudo add-apt-repository -y ppa:marlam/gta
+  - sudo apt-get update -qq
 
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-      env:
-        - BUILD_NAME=csa_part_1
-        - DETAILS="CLang Static Analyzer part 1"
+# MSSQL: server side
+  - sudo apt-get install docker
+  - docker pull microsoft/mssql-server-linux:2017-latest
+  - sudo docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=DummyPassw0rd'  -p 1433:1433 --name sql1 -d microsoft/mssql-server-linux:2017-latest
+  - sleep 10
+  - docker exec -it sql1 /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -U SA -P DummyPassw0rd -Q "CREATE DATABASE TestDB;"
 
-    - os: linux
-      compiler: gcc
-      language: cpp
-      sudo: required
-      dist: trusty
-      cache:
-        apt: true
-      env:
-        - BUILD_NAME=csa_part_2
-        - DETAILS="CLang Static Analyzer part 2"
+  - sudo apt-get install python-numpy  libpng12-dev libjpeg-dev libgif-dev liblzma-dev libgeos-dev libcurl4-gnutls-dev libproj-dev libxml2-dev libexpat-dev libxerces-c-dev libnetcdf-dev netcdf-bin libpoppler-dev libspatialite-dev gpsbabel swig libhdf4-alt-dev libhdf5-serial-dev libpodofo-dev poppler-utils libfreexl-dev unixodbc-dev libwebp-dev openjdk-7-jdk libepsilon-dev liblcms2-2 libpcre3-dev mercurial cmake libcrypto++-dev postgresql-9.3-postgis-2.2 postgresql-9.3-postgis-scripts libpq-dev sqlite3
+# libgta-dev
+  - sudo apt-get install python-lxml
+  - sudo apt-get install python-pip
+  - sudo apt-get install libogdi3.2-dev
+# Boost for Mongo
+#  - sudo apt-get install libboost-regex-dev libboost-system-dev libboost-thread-dev
+#  - sudo pip install pyflakes
+#  - pyflakes autotest
+#  - pyflakes gdal/swig/python/scripts
+#  - pyflakes gdal/swig/python/samples
+# For cfchecks
+  - sudo apt-get install libudunits2-dev udunits-bin
+  - pip install --user cdat-lite
+  - mkdir -p autotest/gdrivers/tmp/cache
+  - wget http://cfconventions.org/Data/cf-standard-names/current/src/cf-standard-name-table.xml -O autotest/gdrivers/tmp/cache/cf-standard-name-table-v18.xml
+#
+#  - psql -c "drop database if exists autotest" -U postgres
+  - psql -c "create database autotest" -U postgres
+  - psql -c "create extension postgis" -d autotest -U postgres
+  - mysql -e "create database autotest;"
+  - mysql -e "GRANT ALL ON autotest.* TO 'root'@'localhost';" -u root
+  - mysql -e "GRANT ALL ON autotest.* TO 'travis'@'localhost';" -u root
+  - wget http://s3.amazonaws.com/etc-data.koordinates.com/gdal-travisci/FileGDB_API_1_2-64.tar.gz
+  - wget http://s3.amazonaws.com/etc-data.koordinates.com/gdal-travisci/MrSID_DSDK-8.5.0.3422-linux.x86-64.gcc44.tar.gz
+  - wget http://s3.amazonaws.com/etc-data.koordinates.com/gdal-travisci/install-libecwj2-ubuntu12.04-64bit.tar.gz
+  - wget http://s3.amazonaws.com/etc-data.koordinates.com/gdal-travisci/install-libkml-r864-64bit.tar.gz
+  - wget https://github.com/uclouvain/openjpeg/releases/download/v2.3.0/openjpeg-v2.3.0-linux-x86_64.tar.gz
+  - wget http://even.rouault.free.fr/mongo-cxx-1.0.2-install-ubuntu12.04-64bit.tar.gz
+  - tar xzf MrSID_DSDK-8.5.0.3422-linux.x86-64.gcc44.tar.gz
+  - sudo cp -r MrSID_DSDK-8.5.0.3422-linux.x86-64.gcc44/Raster_DSDK/include/* /usr/local/include
+  - sudo cp -r MrSID_DSDK-8.5.0.3422-linux.x86-64.gcc44/Raster_DSDK/lib/* /usr/local/lib
+  - sudo cp -r MrSID_DSDK-8.5.0.3422-linux.x86-64.gcc44/Lidar_DSDK/include/* /usr/local/include
+  - sudo cp -r MrSID_DSDK-8.5.0.3422-linux.x86-64.gcc44/Lidar_DSDK/lib/* /usr/local/lib
+  - tar xzf FileGDB_API_1_2-64.tar.gz
+  - sudo cp -r FileGDB_API/include/* /usr/local/include
+  - sudo cp -r FileGDB_API/lib/* /usr/local/lib
+  - tar xzf install-libecwj2-ubuntu12.04-64bit.tar.gz
+  - sudo cp -r install-libecwj2/include/* /usr/local/include
+  - sudo cp -r install-libecwj2/lib/* /usr/local/lib
+  - tar xzf install-libkml-r864-64bit.tar.gz
+  - sudo cp -r install-libkml/include/* /usr/local/include
+  - sudo cp -r install-libkml/lib/* /usr/local/lib
+  - tar xzf openjpeg-v2.3.0-linux-x86_64.tar.gz
+  - sudo cp -r openjpeg-v2.3.0-linux-x86_64/include/* /usr/local/include
+  - sudo cp -r openjpeg-v2.3.0-linux-x86_64/lib/* /usr/local/lib
+  - tar xzf mongo-cxx-1.0.2-install-ubuntu12.04-64bit.tar.gz
+  - sudo cp -r mongo-cxx-1.0.2-install/include/* /usr/local/include
+  - sudo cp -r mongo-cxx-1.0.2-install/lib/* /usr/local/lib
+# oracle stuff
+  - curl -L -o instantclient-sdk-linux.x64-11.2.0.4.0.zip "$INSTANT_SDK"
+  - curl -L -o instantclient-linux.x64-11.2.0.4.0.zip "$INSTANT_CLIENT"
+  - unzip instantclient-sdk-linux.x64-11.2.0.4.0.zip
+  - unzip instantclient-linux.x64-11.2.0.4.0.zip
+  - cd instantclient_11_2
+  - ln -s libclntsh.so.11.1 libclntsh.so
+  - ln -s libocci.so.11.1 libocci.so
+  - ln -s . Lib 
+  - ln -s . lib 
+  - cd .. 
+  - sudo mv instantclient_11_2 /usr/.
+  - sudo apt-get install libaio1
+  - export ORACLE_HOME=/usr/instantclient_11_2/
+  - export PATH=$PATH:/usr/instantclient_11_2/
+  - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/instantclient_11_2/
+# end of oracle stuff
+  - wget https://bitbucket.org/chchrsc/kealib/get/c6d36f3db5e4.zip
+  - unzip c6d36f3db5e4.zip
+  - cd chchrsc-kealib-c6d36f3db5e4/trunk
+  - cmake . -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DHDF5_INCLUDE_DIR=/usr/include -DHDF5_LIB_PATH=/usr/lib -DLIBKEA_WITH_GDAL=OFF
+  - make -j4
+  - sudo make install
+  - cd ../..
+# Build zstd
+  - wget https://github.com/facebook/zstd/archive/v1.3.3.tar.gz
+  - tar xvzf v1.3.3.tar.gz
+  - cd zstd-1.3.3/lib
+# Faster build
+  - make -j3 PREFIX=/usr ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1
+  - sudo make install PREFIX=/usr ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1
+  - cd ../..
 
-before_install:
-  - ./gdal/ci/travis/${BUILD_NAME}/before_install.sh
+# coverage
+  - sudo apt-get install -qq curlftpfs lcov
+
+# MSSQL: client side
+  - sudo bash -c "curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -"
+  - sudo bash -c "curl https://packages.microsoft.com/config/ubuntu/14.04/prod.list | tee /etc/apt/sources.list.d/msprod.list"
+  - sudo apt-get update
+  - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17
+
+# Nasty: force reinstallation of unixodbc-dev since the previous line installed unixodbc 2.3.1 from microsoft repo, which lacks the -dev package
+# DONT DO THAT ON YOUR PRODUCTION SERVER.
+  - wget http://mirrors.edge.kernel.org/ubuntu/pool/main/u/unixodbc/libodbc1_2.3.1-4.1_amd64.deb
+  - wget http://mirrors.edge.kernel.org/ubuntu/pool/main/u/unixodbc/odbcinst1debian2_2.3.1-4.1_amd64.deb
+  - wget http://mirrors.edge.kernel.org/ubuntu/pool/main/u/unixodbc/odbcinst_2.3.1-4.1_amd64.deb
+  - wget http://mirrors.edge.kernel.org/ubuntu/pool/main/u/unixodbc/unixodbc-dev_2.3.1-4.1_amd64.deb
+  - wget http://mirrors.edge.kernel.org/ubuntu/pool/main/u/unixodbc/unixodbc_2.3.1-4.1_amd64.deb
+  - sudo dpkg -i --force-all libodbc1_2.3.1-4.1_amd64.deb odbcinst1debian2_2.3.1-4.1_amd64.deb odbcinst_2.3.1-4.1_amd64.deb unixodbc-dev_2.3.1-4.1_amd64.deb unixodbc_2.3.1-4.1_amd64.deb
+
+#
+  - sudo ldconfig
+#  - sudo pip install cpp-coveralls --use-mirrors
+  - cd gdal
+  - wget http://trac.osgeo.org/gdal/raw-attachment/wiki/AutotestStatus/coverage_449b5ce7ecab7ea880dbcdb4667ac3de9aec870c.patch -O coverage.patch
+  - wget http://trac.osgeo.org/gdal/raw-attachment/wiki/AutotestStatus/filter_info.c
+  - patch -p0 < coverage.patch
+  - cd ..
+
+# installing PostGIS 2 from Ubuntu PPA draws libgdal1.9 dependency. We must install ourselves
+# in /usr too to avoid gdal version conflict. Might be fragile however
 
 install:
-  - ./gdal/ci/travis/${BUILD_NAME}/install.sh
+  - git clone --depth=1 https://github.com/OSGeo/proj.4 proj
+  - (cd proj/data && curl http://download.osgeo.org/proj/proj-datumgrid-1.8.tar.gz > proj-datumgrid-1.8.tar.gz && tar xvzf proj-datumgrid-1.8.tar.gz)
+  - (cd proj && ./autogen.sh && CFLAGS='-DPROJ_RENAME_SYMBOLS -O1' CXXFLAGS=$CFLAGS ./configure --prefix=/usr/local && make -j3 -s >/dev/null && sudo make -j3 install && sudo mv /usr/local/lib/libproj.so.15.0.0 /usr/local/lib/libinternalproj.so.15.0.0 && sudo rm /usr/local/lib/libproj.* && sudo ln -s libinternalproj.so.15.0.0  /usr/local/lib/libinternalproj.so.15 && sudo ln -s libinternalproj.so.15.0.0  /usr/local/lib/libinternalproj.so)
+  - cd gdal
+  - ./configure --prefix=/usr --without-libtool --enable-debug --with-oci=/usr/instantclient_11_2 --with-jpeg12 --with-python --with-poppler --with-podofo --with-spatialite --with-mysql --with-liblzma --with-webp --with-java --with-mdb --with-jvm-lib-add-rpath --with-epsilon --with-ecw=/usr/local --with-mrsid=/usr/local --with-mrsid-lidar=/usr/local --with-fgdb=/usr/local --with-libkml --with-libtiff=internal --with-rename-internal-libtiff-symbols --with-hide-internal-symbols  --with-gnm --with-null --with-proj=/usr/local
+# --with-mongocxx=/usr/local
+# --with-gta
+  - make USER_DEFS="-Wextra -Werror" -j3
+  - cd apps
+  - make USER_DEFS="-Wextra -Werror" test_ogrsf
+  - cd ..
+  - cd swig/java
+  - cat java.opt | sed "s/JAVA_HOME =.*/JAVA_HOME = \/usr\/lib\/jvm\/java-7-openjdk-amd64\//" > java.opt.tmp
+  - mv java.opt.tmp java.opt
+  - make 2>/dev/null >/dev/null
+  - cd ../..
+  - cd swig/perl
+  - make generate
+  - make
+  - cd ../..
+  - sudo rm -f /usr/lib/libgdal.so*
+  - sudo make install
+  - sudo ldconfig
+  - cd ../autotest/cpp
+  - make -j3
+  - cd ../../gdal
+  - wget https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/mdb-sqlite/mdb-sqlite-1.0.2.tar.bz2
+  - tar xjvf mdb-sqlite-1.0.2.tar.bz2
+  - sudo cp mdb-sqlite-1.0.2/lib/*.jar /usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext
 
+# When run in the same process after ogr_pgeo.py (with the MDB driver), FileGDB tests fail.
+# Run it in isolation
 script:
-  - ./gdal/ci/travis/${BUILD_NAME}/script.sh
+# Perl unit tests
+  - cd swig/perl
+  - make test
+  - cd ../..
+# Java unit tests
+#  - cd swig/java
+#  - make test
+#  - cd ../..
+# CPP unit tests
+  - cd ../autotest
+  - cd cpp
+  - GDAL_SKIP=JP2ECW make quick_test
+# Compile and test vsipreload
+  - make vsipreload.so
+  - LD_PRELOAD=./vsipreload.so gdalinfo /vsicurl/http://download.osgeo.org/gdal/data/ecw/spif83.ecw
+  - LD_PRELOAD=./vsipreload.so sqlite3  /vsicurl/http://download.osgeo.org/gdal/data/sqlite3/polygon.db "select * from polygon limit 10"
+  - cd ..
+# Download a sample file
+  - mkdir -p ogr/tmp/cache/
+  - cd ogr/tmp/cache/
+  - wget http://download.osgeo.org/gdal/data/pgeo/PGeoTest.zip
+  - unzip PGeoTest.zip
+  - cd ../../..
+  - pip install --user -r ./requirements.txt
+  - export PYTEST="pytest -vv -p no:sugar --color=no"
+# Run ogr_fgdb.py in isolation from the rest
+  - cd ogr
+  - $PYTEST ogr_fgdb.py
+  - rm ogr_fgdb.*
+  - cd ..
+# Run ogr_pgeo.py in isolation from the rest
+  - cd ogr
+  - $PYTEST ogr_pgeo.py
+  - rm ogr_pgeo.*
+  - cd ..
+# Run all the Python autotests
+  - $PYTEST
+# A bit messy, but force testing with libspatialite 4.0dev (that has been patched a bit to remove any hard-coded SRS definition so it is very small)
+  - cd ogr
+  - wget http://s3.amazonaws.com/etc-data.koordinates.com/gdal-travisci/libspatialite4.0dev_ubuntu12.04-64bit_srs_stripped.tar.gz
+  - tar xzf libspatialite4.0dev_ubuntu12.04-64bit_srs_stripped.tar.gz
+  - ln -s install-libspatialite-4.0dev/lib/libspatialite.so.5.0.1 libspatialite.so.3
+  - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD $PYTEST ogr_sqlite.py
+  - cd ..
+
+after_success: 
+- cd ../gdal
+- sudo cp ../coverage.py /usr/local/lib/python2.7/dist-packages/coveralls
+- echo "$TRAVIS_SECURE_ENV_VARS"
+- ./run_code_coverage_upload.sh
+#- sh -c 'if test "$TRAVIS_SECURE_ENV_VARS" = "true"; then echo "run coverage"; ./run_code_coverage_upload.sh; fi'
+#- coveralls --exclude swig
 
 notifications:
-  email:
-    recipients:
-      - gdal-commits@lists.osgeo.org
+#  email:
+#    recipients:
+#      - gdal-commits@lists.osgeo.org
 
   irc:
     channels: