GTiff: add support for ZSTD compression/decompression; add --with-zst…

…d to configure and corresponding nmake.opt options

git-svn-id: https://svn.osgeo.org/gdal/trunk@41485 f0d54148-0727-0410-94bb-9a71ac55c965

autotest/gcore/tiff_write.py

@@ -7831,6 +7831,19 @@ def tiff_write_170_invalid_compresion():
     gdal.Unlink('/vsimem/out.tif')
     return 'success'
 
+###############################################################################
+# Test ZSTD compression
+
+def tiff_write_171_zstd():
+
+    md = gdaltest.tiff_drv.GetMetadata()
+    if md['DMD_CREATIONOPTIONLIST'].find('ZSTD') == -1:
+        return 'skip'
+
+    ut = gdaltest.GDALTest( 'GTiff', 'byte.tif', 1, 4672,
+                            options = [ 'COMPRESS=ZSTD', 'ZSTD_LEVEL=1' ] )
+    return ut.testCreateCopy()
+
 ###############################################################################
 # Ask to run again tests with GDAL_API_PROXY=YES
 
@@ -8031,6 +8044,7 @@ def tiff_write_cleanup():
     tiff_write_168_ccitfax3,
     tiff_write_169_ccitrle,
     tiff_write_170_invalid_compresion,
+    tiff_write_171_zstd,
     #tiff_write_api_proxy,
     tiff_write_cleanup ]
 

gdal/GDALmake.opt.in

@@ -443,6 +443,7 @@ GEOMEDIA_SETTING    =       @ODBC_SETTING@
 NETCDF_SETTING  =       @NETCDF_SETTING@
 LIBZ_SETTING	=	@LIBZ_SETTING@
 LIBLZMA_SETTING	=	@LIBLZMA_SETTING@
+ZSTD_SETTING	=	@ZSTD_SETTING@
 
 #
 # DDS via Crunch Support.

gdal/configure

@@ -826,6 +826,7 @@ PG_LIB
 PG_INC
 HAVE_PG
 PG_CONFIG
+ZSTD_SETTING
 LIBLZMA_SETTING
 LTLIBICONV
 LIBICONV
@@ -967,6 +968,7 @@ with_libz
 enable_rpath
 with_libiconv_prefix
 with_liblzma
+with_zstd
 with_pg
 with_grass
 with_libgrass
@@ -1748,6 +1750,7 @@ Optional Packages:
   --with-libiconv-prefix[=DIR]  search for libiconv in DIR/include and DIR/lib
   --without-libiconv-prefix     don't search for libiconv in includedir and libdir
   --with-liblzma=ARG       Include liblzma support (ARG=yes/no)
+  --with-zstd=ARG       Include zstd support (ARG=yes/no/installation_prefix)
   --with-pg=ARG           Include PostgreSQL GDAL/OGR Support (ARG=path to
                           pg_config)
   --with-grass=ARG      Include GRASS support (GRASS 5.7+, ARG=GRASS install tree dir)
@@ -22669,6 +22672,121 @@ LIBLZMA_SETTING=$LIBLZMA_SETTING
 
 
 
+
+
+# Check whether --with-zstd was given.
+if test "${with_zstd+set}" = set; then :
+  withval=$with_zstd;
+fi
+
+
+if test "$with_zstd" = "yes" ; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_decompressStream in -lzstd" >&5
+$as_echo_n "checking for ZSTD_decompressStream in -lzstd... " >&6; }
+if ${ac_cv_lib_zstd_ZSTD_decompressStream+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lzstd  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ZSTD_decompressStream ();
+int
+main ()
+{
+return ZSTD_decompressStream ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_zstd_ZSTD_decompressStream=yes
+else
+  ac_cv_lib_zstd_ZSTD_decompressStream=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_decompressStream" >&5
+$as_echo "$ac_cv_lib_zstd_ZSTD_decompressStream" >&6; }
+if test "x$ac_cv_lib_zstd_ZSTD_decompressStream" = xyes; then :
+  ZSTD_SETTING=yes
+else
+  ZSTD_SETTING=no
+fi
+
+
+  if test "$zstd_SETTING" = "yes" ; then
+    LIBS="-lzstd $LIBS"
+  fi
+elif test "$with_zstd" != "" -a "$with_zstd" != "no"; then
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_decompressStream in -lzstd" >&5
+$as_echo_n "checking for ZSTD_decompressStream in -lzstd... " >&6; }
+if ${ac_cv_lib_zstd_ZSTD_decompressStream+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lzstd -L$with_zstd/lib $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ZSTD_decompressStream ();
+int
+main ()
+{
+return ZSTD_decompressStream ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_zstd_ZSTD_decompressStream=yes
+else
+  ac_cv_lib_zstd_ZSTD_decompressStream=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_decompressStream" >&5
+$as_echo "$ac_cv_lib_zstd_ZSTD_decompressStream" >&6; }
+if test "x$ac_cv_lib_zstd_ZSTD_decompressStream" = xyes; then :
+  ZSTD_SETTING=yes
+else
+  ZSTD_SETTING=no
+fi
+
+
+  if test "$ZSTD_SETTING" = "yes" -a -f "$with_zstd/include/zstd.h" ; then
+    LIBS="-L$with_zstd/lib -lzstd $LIBS"
+    EXTRA_INCLUDES="-I$with_zstd/include $EXTRA_INCLUDES"
+  else
+    echo "libzstd not found - ZSTD support disabled"
+  fi
+
+else
+    ZSTD_SETTING=no
+fi
+
+ZSTD_SETTING=$ZSTD_SETTING
+
+
+
 PG_CONFIG=no
 
 
@@ -37828,6 +37946,9 @@ echo "  LIBZ support:              ${LIBZ_SETTING}"
 echo "  LIBLZMA support:           ${LIBLZMA_SETTING}"
 
 
+echo "  ZSTD support:              ${ZSTD_SETTING}"
+
+
 echo "  cryptopp support:          ${HAVE_CRYPTOPP}"
 
 

gdal/configure.ac

@@ -1148,6 +1148,36 @@ fi
 
 AC_SUBST(LIBLZMA_SETTING,$LIBLZMA_SETTING)
 
+
+dnl ---------------------------------------------------------------------------
+dnl Check if zstd is available.
+dnl ---------------------------------------------------------------------------
+
+AC_ARG_WITH(zstd,[  --with-zstd[=ARG]       Include zstd support (ARG=yes/no/installation_prefix)],,)
+
+if test "$with_zstd" = "yes" ; then
+  AC_CHECK_LIB(zstd,ZSTD_decompressStream,ZSTD_SETTING=yes,ZSTD_SETTING=no,)
+
+  if test "$zstd_SETTING" = "yes" ; then
+    LIBS="-lzstd $LIBS"
+  fi
+elif test "$with_zstd" != "" -a "$with_zstd" != "no"; then
+
+  AC_CHECK_LIB(zstd,ZSTD_decompressStream,ZSTD_SETTING=yes,ZSTD_SETTING=no,-L$with_zstd/lib)
+
+  if test "$ZSTD_SETTING" = "yes" -a -f "$with_zstd/include/zstd.h" ; then
+    LIBS="-L$with_zstd/lib -lzstd $LIBS"
+    EXTRA_INCLUDES="-I$with_zstd/include $EXTRA_INCLUDES"
+  else
+    echo "libzstd not found - ZSTD support disabled"
+  fi
+
+else
+    ZSTD_SETTING=no
+fi
+
+AC_SUBST(ZSTD_SETTING,$ZSTD_SETTING)
+
 dnl ---------------------------------------------------------------------------
 dnl Select an PostgreSQL Library to use, or disable driver.
 dnl ---------------------------------------------------------------------------
@@ -5594,6 +5624,7 @@ LOC_MSG([  LIBTOOL support:           ${with_libtool}])
 LOC_MSG()
 LOC_MSG([  LIBZ support:              ${LIBZ_SETTING}])
 LOC_MSG([  LIBLZMA support:           ${LIBLZMA_SETTING}])
+LOC_MSG([  ZSTD support:              ${ZSTD_SETTING}])
 LOC_MSG([  cryptopp support:          ${HAVE_CRYPTOPP}])
 LOC_MSG([  crypto/openssl support:    ${HAVE_OPENSSL_CRYPTO}])
 LOC_MSG([  GRASS support:             ${GRASS_SETTING}])

gdal/frmts/gtiff/frmt_gtiff.html

@@ -298,13 +298,15 @@ <h3>Creation Options</h3>
 From GDAL 1.6.0, values of n=9...15 (UInt16 type) and n=17...31 (UInt32 type) are also accepted.
 From GDAL 2.2, n=16 is accepted for Float32 type to generate half-precision floating point values.</p></li>
 
-<li><p><b>COMPRESS=[JPEG/LZW/PACKBITS/DEFLATE/CCITTRLE/CCITTFAX3/CCITTFAX4/LZMA/NONE]</b>:
+<li><p><b>COMPRESS=[JPEG/LZW/PACKBITS/DEFLATE/CCITTRLE/CCITTFAX3/CCITTFAX4/LZMA/ZSTD/NONE]</b>:
 Set the compression to use.  JPEG should generally only be used with Byte data (8 bit per channel).
 But starting with GDAL 1.7.0 and provided that GDAL is built with internal libtiff and libjpeg,
 it is possible to read and write TIFF files with 12bit JPEG compressed TIFF files (seen as UInt16 bands with NBITS=12).
 See the <a href="http://trac.osgeo.org/gdal/wiki/TIFF12BitJPEG">"8 and 12 bit JPEG in TIFF"</a> wiki page for more details.
 The CCITT compression should only be used with 1bit (NBITS=1) data.
 LZW and DEFLATE compressions can be used with the PREDICTOR creation option.
+ZSTD is available since GDAL 2.3 when using internal libtiff and if GDAL built
+against libzstd &gt;=1.0, or if built against external libtiff with zstd support.
 None is the default.</p></li>
 
 <li><p><b>NUM_THREADS=number_of_threads/ALL_CPUS</b>: (From GDAL 2.1)
@@ -344,6 +346,8 @@ <h3>Creation Options</h3>
 
 <li><p><b>ZLEVEL=[1-9]</b>:  Set the level of compression when using DEFLATE compression. A value of 9 is best, and 1 is least compression. The default is 6.</p></li>
 
+<li><p><b>ZSTD_LEVEL=[1-22]</b>:  Set the level of compression when using ZSTD compression. A value of 22 is best (very slow), and 1 is least compression. The default is 9.</p></li>
+
 <li><p><b>PHOTOMETRIC=[MINISBLACK/MINISWHITE/RGB/CMYK/YCBCR/CIELAB/ICCLAB/ITULAB]</b>:
 Set the photometric interpretation tag. Default is MINISBLACK, but if the
 input image has 3 or 4 bands of Byte type, then RGB will be selected. You can