Merge pull request #1903 from rouault/add_proj_download

Add projsync utility

configure.ac

@@ -321,6 +321,7 @@ fi
 AC_SUBST(CURL_CFLAGS,$CURL_CFLAGS)
 AC_SUBST(CURL_LIBS,$CURL_LIBS)
 AC_SUBST(CURL_ENABLED_FLAGS,$CURL_ENABLED_FLAGS)
+AM_CONDITIONAL(HAVE_CURL, [test "x$FOUND_CURL" = "xyes"])
 
 dnl ---------------------------------------------------------------------------
 dnl Check for external Google Test

docs/source/apps/cct.rst

@@ -190,7 +190,7 @@ Hence, in honour of *cct* (the geodesist) this is :program:`cct` (the program).
     See also
     ********
 
-    **proj(1)**, **cs2cs(1)**, **geod(1)**, **gie(1)**, **projinfo(1)**
+    **proj(1)**, **cs2cs(1)**, **geod(1)**, **gie(1)**, **projinfo(1)**, **projsync(1)**
 
     Bugs
     ****

docs/source/apps/cs2cs.rst

@@ -230,7 +230,7 @@ outputs
     See also
     ********
 
-    **proj(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **projinfo(1)**
+    **proj(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **projinfo(1)**, **projsync(1)**
 
     Bugs
     ****

docs/source/apps/geod.rst

@@ -200,7 +200,7 @@ Further reading
     See also
     ********
 
-    **proj(1)**, **cs2cs(1)**, **cct(1)**, **geod(1)**, **gie(1)**
+    **proj(1)**, **cs2cs(1)**, **cct(1)**, **gie(1)**, **projinfo(1)**, **projsync(1)**
 
     Bugs
     ****

docs/source/apps/gie.rst

@@ -366,7 +366,7 @@ So in honour, and hopefully also in the spirit, of Gerald Ian Evenden
     See also
     ********
 
-    **proj(1)**, **cs2cs(1)**, **cct(1)**, **geod(1)**
+    **proj(1)**, **cs2cs(1)**, **cct(1)**, **geod(1)**, **projinfo(1)**, **projsync(1)**
 
     Bugs
     ****

docs/source/apps/index.rst

@@ -24,3 +24,4 @@ operations.
    gie
    proj
    projinfo
+   projsync

docs/source/apps/proj.rst

@@ -214,7 +214,7 @@ data will appear as three lines of::
     See also
     ********
 
-    **cs2cs(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **projinfo(1)**
+    **cs2cs(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **projinfo(1)**, **projsync(1)**
 
     Bugs
     ****

docs/source/apps/projinfo.rst

@@ -416,7 +416,7 @@ Output:
     See also
     ********
 
-    **cs2cs(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **proj(1)**
+    **cs2cs(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **proj(1)**, **projsync(1)**
 
     Bugs
     ****

docs/source/apps/projsync.rst

@@ -0,0 +1,160 @@
+.. _projsync:
+
+================================================================================
+projsync
+================================================================================
+
+.. Index:: projsync
+
+.. only:: html
+
+    .. versionadded:: 7.0.0
+
+    Tool for synchronizing PROJ datum and transformation support data.
+
+Synopsis
+********
+
+    | **projsync**
+    |      [--endpoint URL]
+    |      [--local-geojson-file FILENAME]
+    |      ([--user-writable-directory] | [--system-directory] | [--target-dir DIRNAME])
+    |      [--bbox west_long,south_lat,east_long,north_lat]
+    |      [--spatial-test contains|intersects]
+    |      [--source-id ID] [--area-of-use NAME]
+    |      [--file NAME]
+    |      [--all] [--exclude-world-coverage]
+    |      [--quiet] [--dry-run] [--list-files]
+
+Description
+***********
+
+:program:`projsync` is a program that downloads remote resource files
+into a local directory. This is an alternative to downloading a proj-data-X.Y.Z
+archive file, or using the on-demand :ref:`networking capabilities <network>` of PROJ.
+
+The following control parameters can appear in any order:
+
+.. program:: projsync
+
+.. option:: --endpoint URL
+
+    Defines the URL where to download the master ``files.geojson`` file and then
+    the resource files. Defaults to the value set in :ref:`proj-ini`
+
+.. option:: --local-geojson-file FILENAME
+
+    Defines the filename for the master GeoJSON files that references resources.
+    Defaults to ${endpoint}/files.geojson
+
+.. option:: --user-writable-directory
+
+    Specifies that resource files must be downloaded in the
+    :ref:`user writable directory <user_writable_directory>`. This is the default.
+
+.. option:: --system-directory
+
+    Specifies that resource files must be downloaded in the
+    ${installation_prefix}/share/proj directory. The user launching projsync
+    should make sure it has writing rights in that directory.
+
+.. option:: --target-dir DIRNAME
+
+    Directory into which resource files must be downloaded.
+
+.. option:: --bbox west_long,south_lat,east_long,north_lat
+
+    Specify an area of interest to restrict the resources to download.
+    The area of interest is specified as a
+    bounding box with geographic coordinates, expressed in degrees in a
+    unspecified geographic CRS.
+    `west_long` and `east_long` should be in the [-180,180] range, and
+    `south_lat` and `north_lat` in the [-90,90]. `west_long` is generally lower than
+    `east_long`, except in the case where the area of interest crosses the antimeridian.
+
+.. option:: --spatial-test contains|intersects
+
+    Specify how the extent of the resource files
+    are compared to the area of use specified explicitly with :option:`--bbox`.
+    By default, any resource files whose extent intersects the value specified
+    by :option:`--bbox` will be selected.
+    If using the ``contains`` strategy, only resource files whose extent is
+    contained in the value specified by :option:`--bbox` will be selected.
+
+.. option:: --source-id ID
+
+    Restrict resource files to be downloaded to those whose source_id property
+    contains the ID value. Specifying ``?`` as ID will list all possible values.
+
+.. option:: --area-of-use NAME
+
+    Restrict resource files to be downloaded to those whose area_of_use property
+    contains the NAME value. Specifying ``?`` as NAME will list all possible values.
+
+.. option:: --file NAME
+
+    Restrict resource files to be downloaded to those whose name property
+    contains the NAME value. Specifying ``?`` as NAME will list all possible values.
+
+.. option:: --all
+
+    Ask to download all files.
+
+.. option:: --exclude-world-coverage
+
+    Exclude files which have world coverage.
+
+.. option:: -q / --quiet
+
+    Quiet mode
+
+.. option:: --dry-run
+
+    Simulate the behaviour of the tool without downloading resource files.
+
+.. option:: --list-files
+
+    List file names, with the source_id and area_of_use properties.
+
+
+At least one of  :option:`--list-files`,  :option:`--file`,  :option:`--source-id`,
+:option:`--area-of-use`,  :option:`--bbox` or  :option:`--all` must be specified.
+
+Options :option:`--file`,  :option:`--source-id`, :option:`--area-of-use` and
+:option:`--bbox` are combined with a AND logic.
+
+Examples
+********
+
+1. Download all resource files
+
+.. code-block:: console
+
+      projsync --all
+
+2. Download resource files covering specified point and attributing to an agency
+
+.. code-block:: console
+
+      projsync --source-id fr_ign --bbox 2,49,2,49
+
+
+.. only:: man
+
+    See also
+    ********
+
+    **cs2cs(1)**, **cct(1)**, **geod(1)**, **gie(1)**, **proj(1)**, **projinfo(1)**
+
+    Bugs
+    ****
+
+    A list of know bugs can be found at https://github.com/OSGeo/PROJ/issues
+    where new bug reports can be submitted to.
+    Bugs specific to resource files should be submitted to
+    https://github.com/OSGeo/PROJ-data/issues
+
+    Home page
+    *********
+
+    https://proj.org/

docs/source/conf.py

@@ -346,6 +346,13 @@
         ['Even Rouault'],
         1
     ),
+    (
+        'apps/projsync',
+        'projsync',
+        u'Downloading tool of resource files',
+        ['Even Rouault'],
+        1
+    ),
 ]
 
 # If true, show URL addresses after external links.

docs/source/usage/network.rst

@@ -76,6 +76,11 @@ capability to download whole grids in the
 :ref:`PROJ user writable directory <user_writable_directory>` by using the
 :cpp:func:`proj_is_download_needed` and :cpp:func:`proj_download_file` functions.
 
+Download utility
+----------------
+
+:ref:`projsync` is a tool for downloading resource files.
+
 
 Mirroring
 ---------

scripts/reference_exported_symbols.txt

@@ -281,6 +281,9 @@ osgeo::proj::datum::TemporalDatum::temporalOrigin() const
 osgeo::proj::datum::VerticalReferenceFrame::create(osgeo::proj::util::PropertyMap const&, osgeo::proj::util::optional<std::string> const&, osgeo::proj::util::optional<osgeo::proj::datum::RealizationMethod> const&)
 osgeo::proj::datum::VerticalReferenceFrame::realizationMethod() const
 osgeo::proj::datum::VerticalReferenceFrame::~VerticalReferenceFrame()
+osgeo::proj::File::~File()
+osgeo::proj::FileManager::open(projCtx_t*, char const*, osgeo::proj::FileAccess)
+osgeo::proj::File::read_line(unsigned long, bool&, bool&)
 osgeo::proj::GenericShiftGrid::~GenericShiftGrid()
 osgeo::proj::GenericShiftGrid::GenericShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&)
 osgeo::proj::GenericShiftGrid::gridAt(double, double) const
@@ -742,6 +745,7 @@ pj_context_get_grid_cache_filename(projCtx_t*)
 pj_context_get_url_endpoint(projCtx_t*)
 pj_context_get_user_writable_directory(projCtx_t*, bool)
 pj_context_is_network_enabled(projCtx_t*)
+pj_context_set_user_writable_directory(projCtx_t*, std::string const&)
 pj_ctx_alloc
 pj_ctx_fclose
 pj_ctx_fgets
@@ -774,6 +778,7 @@ pj_get_datums_ref()
 pj_get_def
 pj_get_default_ctx
 pj_get_default_fileapi
+pj_get_default_searchpaths(projCtx_t*)
 pj_get_errno_ref
 pj_get_release
 pj_get_spheroid_defn

scripts/reformat_cpp.sh

@@ -16,9 +16,11 @@ esac
 TOPDIR="$SCRIPT_DIR/.."
 
 for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp \
-         "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp \
+         "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp \
+         "$TOPDIR"/src/apps/projinfo.cpp "$TOPDIR"/src/apps/projsync.cpp \
          "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp \
          "$TOPDIR"/src/filemanager.hpp "$TOPDIR"/src/filemanager.cpp \
+         "$TOPDIR"/src/networkfilemanager.cpp \
          "$TOPDIR"/src/sqlite3_utils.hpp "$TOPDIR"/src/sqlite3_utils.cpp ; do
     if ! echo "$i" | grep -q "lru_cache.hpp"; then
         "$SCRIPT_DIR"/reformat.sh "$i";

src/CMakeLists.txt

@@ -15,6 +15,8 @@ option(BUILD_PROJ
   "Build proj (cartographic projection tool)" ON)
 option(BUILD_PROJINFO
   "Build projinfo (SRS and coordinate operation metadata/query tool)" ON)
+option(BUILD_PROJSYNC
+  "Build projsync (synchronize transformation support data)" ON)
 
 if(NOT MSVC)
 
@@ -72,6 +74,15 @@ if(BUILD_GIE)
   set(BIN_TARGETS ${BIN_TARGETS} gie)
 endif()
 
+if(BUILD_PROJSYNC)
+  if(NOT ENABLE_CURL)
+    message(SEND_ERROR "projsync requires Curl")
+  endif()
+  include(bin_projsync.cmake)
+  set(BIN_TARGETS ${BIN_TARGETS} bin_projsync)
+endif()
+
+
 if(MSVC OR CMAKE_CONFIGURATION_TYPES)
   if(BIN_TARGETS)
     # Add _d suffix for your debug versions of the tools

src/Makefile.am

@@ -1,6 +1,5 @@
 AM_CFLAGS = @C_WFLAGS@
 
-bin_PROGRAMS =	proj geod cs2cs gie cct projinfo
 EXTRA_PROGRAMS = multistresstest test228
 
 TESTS = geodtest
@@ -15,15 +14,26 @@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesi
 
 EXTRA_DIST = bin_cct.cmake bin_gie.cmake bin_cs2cs.cmake \
 	bin_geod.cmake bin_proj.cmake bin_projinfo.cmake \
-	lib_proj.cmake CMakeLists.txt bin_geodtest.cmake tests/geodtest.cpp \
-	wkt1_grammar.y wkt2_grammar.y apps/emess.h apps/utils.h
+	lib_proj.cmake CMakeLists.txt bin_geodtest.cmake \
+	bin_projsync.cmake \
+	tests/geodtest.cpp \
+	wkt1_grammar.y wkt2_grammar.y apps/emess.h apps/utils.h \
+	apps/projsync.cpp
 
 proj_SOURCES = apps/proj.cpp apps/emess.cpp apps/utils.cpp
 projinfo_SOURCES = apps/projinfo.cpp
 cs2cs_SOURCES = apps/cs2cs.cpp apps/emess.cpp apps/utils.cpp
 cct_SOURCES = apps/cct.cpp apps/proj_strtod.cpp apps/proj_strtod.h apps/optargpm.h
 geod_SOURCES = apps/geod.cpp apps/geod_set.cpp apps/geod_interface.cpp apps/geod_interface.h apps/emess.cpp
 
+if HAVE_CURL
+projsync_SOURCES = apps/projsync.cpp
+projsync_LDADD = libproj.la
+PROJSYNC_BIN = projsync
+endif
+
+bin_PROGRAMS =	proj geod cs2cs gie cct projinfo $(PROJSYNC_BIN)
+
 gie_SOURCES = apps/gie.cpp apps/proj_strtod.cpp apps/proj_strtod.h apps/optargpm.h
 multistresstest_SOURCES = tests/multistresstest.cpp
 test228_SOURCES = tests/test228.cpp