From 2b857bbe51560c0785a64650bf4e36ec748fef6a Mon Sep 17 00:00:00 2001 From: Peter Petrik Date: Tue, 22 Sep 2020 16:01:41 +0200 Subject: [PATCH] fix #85 and #84 and #82 gdal problems --- qgis_bundle/recipes/gdal/recipe.sh | 7 +- qgis_deps/distribute.sh | 17 ++++ qgis_deps/recipes/freetds/recipe.sh | 95 ++++++++++++++++++++++ qgis_deps/recipes/gdal/recipe.sh | 47 +++++++---- qgis_deps/recipes/python/recipe.sh | 5 +- qgis_deps/recipes/python_pymssql/recipe.sh | 66 +++++++++++++++ qgis_deps/recipes/qgis_deps/recipe.sh | 1 + qgis_deps/recipes/unixodbc/recipe.sh | 17 ++-- 8 files changed, 228 insertions(+), 27 deletions(-) create mode 100644 qgis_deps/recipes/freetds/recipe.sh create mode 100644 qgis_deps/recipes/python_pymssql/recipe.sh diff --git a/qgis_bundle/recipes/gdal/recipe.sh b/qgis_bundle/recipes/gdal/recipe.sh index 81c1f6a7..5b05b1ed 100644 --- a/qgis_bundle/recipes/gdal/recipe.sh +++ b/qgis_bundle/recipes/gdal/recipe.sh @@ -47,17 +47,18 @@ function bundle_gdal() { # GDAL plugins # https://github.com/qgis/QGIS/blob/518cc16e87aba6798658acf75c86f27a0f4d99b3/src/app/main.cpp#L1198 # should be in Contents/MacOS/lib/gdalplugins + DEPS_GDAL_NOFOSS_PLUGINS_DIR=$DEPS_LIB_DIR/../3rdParty/gdalplugins try mkdir $BUNDLE_GDAL_PLUGINS_DIR # folder for some 3rdparty SDKs and libs try mkdir $BUNDLE_GDAL_PLUGINS_DIR/3rdparty if [[ "$WITH_ECW" == "true" ]]; then - try cp -av $DEPS_LIB_DIR/gdalplugins/$LINK_gdal_ecw $BUNDLE_GDAL_PLUGINS_DIR/ + try cp -av $DEPS_GDAL_NOFOSS_PLUGINS_DIR/$LINK_gdal_ecw $BUNDLE_GDAL_PLUGINS_DIR/ fi if [[ "$WITH_MRSID" == "true" ]]; then - try cp -av $DEPS_LIB_DIR/gdalplugins/$LINK_gdal_mrsid_raster $BUNDLE_GDAL_PLUGINS_DIR/ - try cp -av $DEPS_LIB_DIR/gdalplugins/$LINK_gdal_mrsid_lidar $BUNDLE_GDAL_PLUGINS_DIR/ + try cp -av $DEPS_GDAL_NOFOSS_PLUGINS_DIR/$LINK_gdal_mrsid_raster $BUNDLE_GDAL_PLUGINS_DIR/ + try cp -av $DEPS_GDAL_NOFOSS_PLUGINS_DIR/$LINK_gdal_mrsid_lidar $BUNDLE_GDAL_PLUGINS_DIR/ MRSID_SDK="$QGIS_BUNDLE_SCRIPT_DIR/../../external/$MRSID_SDK_VER" if [ ! -d "$MRSID_SDK" ]; then diff --git a/qgis_deps/distribute.sh b/qgis_deps/distribute.sh index afb2d328..d872e57d 100755 --- a/qgis_deps/distribute.sh +++ b/qgis_deps/distribute.sh @@ -393,6 +393,23 @@ function verify_binary() { check_linked_rpath $BINARY } +function test_binary_output() { + BINARY=$1 + OUTPUT=$2 + cd ${STAGE_PATH}/ + + BINOUT=`$BINARY` + if [[ $BINOUT == *"$OUTPUT"* ]]; then + # OK found expected string in binary output + : + else + echo "-raw-output-of-$BINARY--" + echo $BINOUT + echo "------" + error "Output of $BINARY does not contain $OUTPUT" + fi +} + run_final_check() { info "Running final check" } diff --git a/qgis_deps/recipes/freetds/recipe.sh b/qgis_deps/recipes/freetds/recipe.sh new file mode 100644 index 00000000..6bae6f90 --- /dev/null +++ b/qgis_deps/recipes/freetds/recipe.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +DESC_freetds="Libraries to talk to Microsoft SQL Server and Sybase databases" + +# version of your package +VERSION_freetds=1.2.5 +LINK_ct=libct.4.dylib +LINK_sybdb=libsybdb.5.dylib +LINK_tdsodbc=libtdsodbc.0.so + +# dependencies of this recipe +DEPS_freetds=(gettext libtool openssl unixodbc) + +# url of the package +URL_freetds=https://www.freetds.org/files/stable/freetds-${VERSION_freetds}.tar.gz + +# md5 of the package +MD5_freetds=5de473ef22bfcc6a415398a5a145c53b + +# default build path +BUILD_freetds=$BUILD_PATH/freetds/$(get_directory $URL_freetds) + +# default recipe path +RECIPE_freetds=$RECIPES_PATH/freetds + +# function called for preparing source code if needed +# (you can apply patch etc here.) +function prebuild_freetds() { + cd $BUILD_freetds + + # check marker + if [ -f .patched ]; then + return + fi + + patch_configure_file configure + + touch .patched +} + +function shouldbuild_freetds() { + # If lib is newer than the sourcecode skip build + if [ ${STAGE_PATH}/lib/$LINK_ct -nt $BUILD_freetds/.patched ]; then + DO_BUILD=0 + fi +} + +# function called to build the source code +function build_freetds() { + try rsync -a $BUILD_freetds/ $BUILD_PATH/freetds/build-$ARCH/ + try cd $BUILD_PATH/freetds/build-$ARCH + push_env + + # add unixodbc + export CFLAGS="$CFLAGS -I$STAGE_PATH/unixodbc/include" + export LDFLAGS="$LDFLAGS -L$STAGE_PATH/unixodbc/lib" + + try ${CONFIGURE} \ + --with-tdsver=7.3 \ + --mandir=${STAGE_PATH}/man \ + --sysconfdir=${STAGE_PATH}/etc \ + --with-unixodbc-includes=${STAGE_PATH}/include/unixodbc \ + --with-openssl=${STAGE_PATH} \ + --enable-sybase-compat \ + --enable-krb5 \ + --enable-odbc-wide + + check_file_configuration config.status + try $MAKE + try $MAKE install + + pop_env +} + +# function called after all the compile have been done +function postbuild_freetds() { + verify_binary bin/tsql + verify_binary bin/freebcp + verify_binary bin/bsqldb + verify_binary bin/defncopy + verify_binary bin/datacopy + verify_binary bin/bsqlodbc + verify_binary lib/$LINK_ct + verify_binary lib/$LINK_sybdb + verify_binary lib/$LINK_tdsodbc +} + +# function to append information to config file +function add_config_info_freetds() { + append_to_config_file "# freetds-${VERSION_freetds}: ${DESC_freetds}" + append_to_config_file "export VERSION_freetds=${VERSION_freetds}" + append_to_config_file "export LINK_ct=${LINK_ct}" + append_to_config_file "export LINK_sybdb=${LINK_sybdb}" + append_to_config_file "export LINK_tdsodbc=${LINK_tdsodbc}" +} diff --git a/qgis_deps/recipes/gdal/recipe.sh b/qgis_deps/recipes/gdal/recipe.sh index 77518f16..40ed7fc8 100644 --- a/qgis_deps/recipes/gdal/recipe.sh +++ b/qgis_deps/recipes/gdal/recipe.sh @@ -33,6 +33,7 @@ DEPS_gdal=( freexl libkml pcre + unixodbc ) # url of the package @@ -47,8 +48,15 @@ BUILD_gdal=$BUILD_PATH/gdal/$(get_directory $URL_gdal) # default recipe path RECIPE_gdal=$RECIPES_PATH/gdal -# 3rd Party +# This directory is automatically picked by GDAL on load to search for drivers GDAL_PLUGINS_DIR=${STAGE_PATH}/lib/gdalplugins + +# 3rd Party +# This directory is NOT automatically picked by GDAL. Compile driver +# that required some external 3rdParty licensed SDK/library, so +# normally you are not able to load them from qgis-deps +GDAL_NOFOSS_PLUGINS_DIR=${STAGE_PATH}/3rdParty/gdalplugins + ECW_SDK_VER="ERDASEcwJpeg2000SDK5.4.0" ECW_SDK="$RECIPES_PATH/../../../external/$ECW_SDK_VER/Desktop_Read-Only/" LINK_gdal_ecw=gdal_ECW_JP2ECW.dylib @@ -78,13 +86,14 @@ function shouldbuild_gdal() { if [ ${STAGE_PATH}/lib/$LINK_gdal -nt $BUILD_gdal/.patched ]; then DO_BUILD=0 fi + # DO_BUILD=1 } function build_ecw() { try cd $BUILD_PATH/gdal/build-$ARCH if [[ "$WITH_ECW" == "true" ]]; then - info "building GDAL ECW driver to $GDAL_PLUGINS_DIR" + info "building GDAL ECW driver to $GDAL_NOFOSS_PLUGINS_DIR" if [ ! -d "$ECW_SDK" ]; then echo "Missing ECW SDK in $ECW_SDK" @@ -99,10 +108,10 @@ function build_ecw() { -I$ECW_SDK/include/NCSEcw/ECW -I$ECW_SDK/include/NCSEcw/JP2 \ ${SRC} \ -dynamiclib \ - -install_name $GDAL_PLUGINS_DIR/${LINK_gdal_ecw} \ + -install_name $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_ecw} \ -current_version ${LINK_libgdal_version} \ -compatibility_version ${LINK_libgdal_version}.0 \ - -o $GDAL_PLUGINS_DIR/${LINK_gdal_ecw} \ + -o $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_ecw} \ -undefined dynamic_lookup \ -L$ECW_SDK/lib -lNCSEcw fi @@ -118,22 +127,22 @@ function build_mrsid() { fi # LIDAR - info "building GDAL MrSID Lidar driver to $GDAL_PLUGINS_DIR" + info "building GDAL MrSID Lidar driver to $GDAL_NOFOSS_PLUGINS_DIR" SRC=$(find frmts/mrsid_lidar -name *.c*) try $CXX -std=c++11 \ -Iport -Igcore -Ifrmts -Iogr -Ifrmts/mrsid_lidar \ -I$MRSID_SDK/Lidar_DSDK/include \ ${SRC} \ -dynamiclib \ - -install_name $GDAL_PLUGINS_DIR/${LINK_gdal_mrsid_lidar} \ + -install_name $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_mrsid_lidar} \ -current_version ${LINK_libgdal_version} \ -compatibility_version ${LINK_libgdal_version}.0 \ - -o $GDAL_PLUGINS_DIR/${LINK_gdal_mrsid_lidar} \ + -o $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_mrsid_lidar} \ -undefined dynamic_lookup \ -L$MRSID_SDK/Lidar_DSDK/lib -llti_lidar_dsdk # RASTER - info "building GDAL MRSID driver to $GDAL_PLUGINS_DIR" + info "building GDAL MRSID driver to $GDAL_NOFOSS_PLUGINS_DIR" SRC=$(find frmts/mrsid -name *.c*) try $CXX -std=c++11 \ -DMRSID_J2K=1 \ @@ -141,13 +150,12 @@ function build_mrsid() { -I$MRSID_SDK/Raster_DSDK/include \ ${SRC} \ -dynamiclib \ - -install_name $GDAL_PLUGINS_DIR/${LINK_gdal_mrsid_raster} \ + -install_name $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_mrsid_raster} \ -current_version ${LINK_libgdal_version} \ -compatibility_version ${LINK_libgdal_version}.0 \ - -o $GDAL_PLUGINS_DIR/${LINK_gdal_mrsid_raster} \ + -o $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_mrsid_raster} \ -undefined dynamic_lookup \ -L$MRSID_SDK/Raster_DSDK/lib -lltidsdk - fi } @@ -157,8 +165,14 @@ function build_gdal() { try cd $BUILD_PATH/gdal/build-$ARCH push_env + try mkdir -p $GDAL_NOFOSS_PLUGINS_DIR try mkdir -p $GDAL_PLUGINS_DIR + # add unixodbc + export CFLAGS="$CFLAGS -I$STAGE_PATH/unixodbc/include" + export LDFLAGS="$LDFLAGS -L$STAGE_PATH/unixodbc/lib" + export CXXFLAGS="${CFLAGS}" + try ${CONFIGURE} \ --with-ecw=no \ --with-mrsid=no \ @@ -168,6 +182,8 @@ function build_gdal() { --enable-driver-gml \ --enable-driver-mvt \ --enable-driver-xlsx \ + --enable-driver-mssqlspatial \ + --with-odbc=yes \ --with-liblzma=$STAGE_PATH \ --with-zstd=$STAGE_PATH \ --with-libtiff=$STAGE_PATH \ @@ -206,13 +222,16 @@ function postbuild_gdal() { verify_binary bin/gdalinfo if [[ "$WITH_ECW" == "true" ]]; then - verify_binary $GDAL_PLUGINS_DIR/${LINK_gdal_ecw} + verify_binary $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_ecw} fi if [[ "$MRSID_SDK" == "true" ]]; then - verify_binary $GDAL_PLUGINS_DIR/${LINK_gdal_mrsid_lidar} - verify_binary $GDAL_PLUGINS_DIR/${LINK_gdal_mrsid_raster} + verify_binary $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_mrsid_lidar} + verify_binary $GDAL_NOFOSS_PLUGINS_DIR/${LINK_gdal_mrsid_raster} fi + + test_binary_output "./bin/ogrinfo --formats" OSM + test_binary_output "./bin/ogrinfo --formats" MSSQLSpatial } # function to append information to config file diff --git a/qgis_deps/recipes/python/recipe.sh b/qgis_deps/recipes/python/recipe.sh index c38dde10..48b02f69 100644 --- a/qgis_deps/recipes/python/recipe.sh +++ b/qgis_deps/recipes/python/recipe.sh @@ -107,8 +107,9 @@ function install_python() { # dlopen some libraries (e.g. _ssl -> libcrypto.dylib) export DYLD_LIBRARY_PATH=$STAGE_PATH/lib - # add unixodbc includes too - export CFLAGS="$CFLAGS -I$STAGE_PATH/include/unixodbc" + # add unixodbc + export CFLAGS="$CFLAGS -I$STAGE_PATH/unixodbc/include" + export LDFLAGS="$LDFLAGS -L$STAGE_PATH/unixodbc/lib" export CXXFLAGS="${CFLAGS}" try ${CONFIGURE} \ diff --git a/qgis_deps/recipes/python_pymssql/recipe.sh b/qgis_deps/recipes/python_pymssql/recipe.sh new file mode 100644 index 00000000..73478f48 --- /dev/null +++ b/qgis_deps/recipes/python_pymssql/recipe.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +DESC_python_pymssql="Python binding of MSSQL" + +# version of your package +# need to keep in sync with hdf5 +VERSION_python_pymssql=2.1.5 + +# dependencies of this recipe +DEPS_python_pymssql=(python freetds python_packages) + +# url of the package +URL_python_pymssql= + +# md5 of the package +MD5_python_pymssql= + +# default build path +BUILD_python_pymssql=$BUILD_PATH/python_pymssql/v${VERSION_python_pymssql} + +# default recipe path +RECIPE_python_pymssql=$RECIPES_PATH/python_pymssql + +# function called for preparing source code if needed +# (you can apply patch etc here.) +function prebuild_python_pymssql() { + mkdir -p $BUILD_python_pymssql + cd $BUILD_python_pymssql + + # check marker + if [ -f .patched ]; then + return + fi + + touch .patched +} + +function shouldbuild_python_pymssql() { + # If lib is newer than the sourcecode skip build + if python_package_installed pymssql; then + DO_BUILD=0 + fi +} + +# function called to build the source code +function build_python_pymssql() { + try cd $BUILD_python_pymssql + push_env + + DYLD_LIBRARY_PATH=$STAGE_PATH/lib try $PIP_NO_BINARY pymssql==${VERSION_python_pymssql} + + pop_env +} + +# function called after all the compile have been done +function postbuild_python_pymssql() { + if ! python_package_installed_verbose pymssql; then + error "Missing python package pymssql" + fi +} + +# function to append information to config file +function add_config_info_python_pymssql() { + append_to_config_file "# python_pymssql-${VERSION_python_pymssql}: ${DESC_python_pymssql}" + append_to_config_file "export VERSION_python_pymssql=${VERSION_python_pymssql}" +} \ No newline at end of file diff --git a/qgis_deps/recipes/qgis_deps/recipe.sh b/qgis_deps/recipes/qgis_deps/recipe.sh index 991d8491..cdf5d5b7 100644 --- a/qgis_deps/recipes/qgis_deps/recipe.sh +++ b/qgis_deps/recipes/qgis_deps/recipe.sh @@ -27,6 +27,7 @@ DEPS_qgis_deps=( python_packages python_matplotlib python_llvmlite + python_pymssql python_statsmodels python_gdal python_fiona diff --git a/qgis_deps/recipes/unixodbc/recipe.sh b/qgis_deps/recipes/unixodbc/recipe.sh index 0b0dfa79..055298b2 100644 --- a/qgis_deps/recipes/unixodbc/recipe.sh +++ b/qgis_deps/recipes/unixodbc/recipe.sh @@ -3,7 +3,7 @@ DESC_unixodbc="ODBC 3 connectivity for UNIX" # version of your package -VERSION_unixodbc=2.3.7 +VERSION_unixodbc=2.3.9 LINK_unixodbc=libodbc.2.dylib # dependencies of this recipe @@ -13,7 +13,7 @@ DEPS_unixodbc=(libtool) URL_unixodbc=http://www.unixodbc.org/unixODBC-${VERSION_unixodbc}.tar.gz # md5 of the package -MD5_unixodbc=274a711b0c77394e052db6493840c6f9 +MD5_unixodbc=06f76e034bb41df5233554abe961a16f # default build path BUILD_unixodbc=$BUILD_PATH/unixodbc/$(get_directory $URL_unixodbc) @@ -38,7 +38,7 @@ function prebuild_unixodbc() { function shouldbuild_unixodbc() { # If lib is newer than the sourcecode skip build - if [ ${STAGE_PATH}/lib/$LINK_unixodbc -nt $BUILD_unixodbc/.patched ]; then + if [ ${STAGE_PATH}/unixodbc/lib/$LINK_unixodbc -nt $BUILD_unixodbc/.patched ]; then DO_BUILD=0 fi } @@ -51,12 +51,12 @@ function build_unixodbc() { # includedir must be specific, since posgresql has # also similar include file names - try ${CONFIGURE} \ + try ./configure \ + --prefix=$STAGE_PATH/unixodbc \ + --sysconfdir=$STAGE_PATH/unixodbc/etc \ --disable-debug \ --disable-dependency-tracking \ - --enable-gui=no \ - --includedir=$STAGE_PATH/include/unixodbc \ - --sysconfdir=$STAGE_PATH/etc + --enable-gui=no check_file_configuration config.status try $MAKESMP @@ -67,7 +67,8 @@ function build_unixodbc() { # function called after all the compile have been done function postbuild_unixodbc() { - verify_binary bin/odbcinst + verify_binary unixodbc/bin/odbcinst + verify_binary unixodbc/lib/${LINK_unixodbc} } # function to append information to config file