Skip to content

Commit

Permalink
Download server (ruslo#1084)
Browse files Browse the repository at this point in the history
* add script to download archives for DOWNLOAD_SERVER

script synopsis:

download_package_for_server.sh [options]
    -h, --help       print this help message and exit.
    --URL            original URL to download from"
    --SHA1           SHA1 of the archive to download"
    --PACKAGE        name of the package to download"
    --VERSION        version of the package to download"
    -o,--output-dir  output directory to create structure and download
                     file into

* update module: accept list of download server

- only active if global variable HUNTER_DOWNLOAD_SERVER is set

- for cmake << 3.7
  - return one URL to try to download from
  - if the original URL matches one of the download server try to
    download from the original URL
  - otherwise download from the first download server
- for cmake >= 3.7
  - return list of URLs to try to download from
  - if the original URL matches one of the download server try to
    download from the original URL first, then from the other listed
    download server
  - try to download from the listed download server, one after another
  • Loading branch information
NeroBurner authored and ruslo committed Nov 20, 2017
1 parent 18bab8e commit b818355
Show file tree
Hide file tree
Showing 6 changed files with 371 additions and 1 deletion.
15 changes: 14 additions & 1 deletion cmake/modules/hunter_download.cmake
Expand Up @@ -5,6 +5,7 @@
include(CMakeParseArguments) # cmake_parse_arguments

include(hunter_create_args_file)
include(hunter_download_server_url)
include(hunter_find_licenses)
include(hunter_find_stamps)
include(hunter_internal_error)
Expand Down Expand Up @@ -78,8 +79,15 @@ function(hunter_download)

set(HUNTER_PACKAGE_VERSION "${HUNTER_${h_name}_VERSION}")
set(ver "${HUNTER_PACKAGE_VERSION}")
set(HUNTER_PACKAGE_URL "${HUNTER_${h_name}_URL}")
set(HUNTER_PACKAGE_SHA1 "${HUNTER_${h_name}_SHA1}")
# set download URL, either direct download or redirected if HUNTER_DOWNLOAD_SERVER is set
hunter_download_server_url(
PACKAGE "${HUNTER_PACKAGE_NAME}"
VERSION "${HUNTER_PACKAGE_VERSION}"
SHA1 "${HUNTER_PACKAGE_SHA1}"
URL "${HUNTER_${h_name}_URL}"
OUTPUT HUNTER_PACKAGE_URL
)
set(
HUNTER_PACKAGE_CONFIGURATION_TYPES
"${HUNTER_${h_name}_CONFIGURATION_TYPES}"
Expand Down Expand Up @@ -450,6 +458,11 @@ function(hunter_download)
"${HUNTER_DOWNLOAD_TOOLCHAIN}"
"set(HUNTER_SUPPRESS_LIST_OF_FILES \"${HUNTER_SUPPRESS_LIST_OF_FILES}\" CACHE INTERNAL \"\")\n"
)
file(
APPEND
"${HUNTER_DOWNLOAD_TOOLCHAIN}"
"set(HUNTER_DOWNLOAD_SERVER \"${HUNTER_DOWNLOAD_SERVER}\" CACHE INTERNAL \"\")\n"
)

string(COMPARE NOTEQUAL "${CMAKE_MAKE_PROGRAM}" "" has_make)
if(has_make)
Expand Down
135 changes: 135 additions & 0 deletions cmake/modules/hunter_download_server_url.cmake
@@ -0,0 +1,135 @@
# Copyright (c) 2017 NeroBurner
# All rights reserved.

include(CMakeParseArguments) # cmake_parse_arguments

include(hunter_status_print)
include(hunter_internal_error)
include(hunter_test_string_not_empty)

# used gobal variables
# - HUNTER_DOWNLOAD_SERVER
# - if given, a list of servers to download from

# update download url to point to HUNTER_DOWNLOAD_SERVER
# - PACKAGE: name of the package to update the url for
# - VERSION: version of the package
# - SHA1: SHA1 of the package to download, only first 7 digits are used as archive-ID
# - URL: original URL for the package
# - OUTPUT: variable to output the updated URL
function(hunter_download_server_url)
set(function_name "hunter_download_server_url")
set(function_synopsis "${function_name}(PACKAGE foo VERSION 0.1.2-p3 SHA1 somelongsha URL http://original_url.example.com OUTPUT variable)")

# parse arguments
set(one_value_args PACKAGE VERSION SHA1 URL OUTPUT)
cmake_parse_arguments(x "" "${one_value_args}" "" ${ARGV})

# No free arguments allowed
list(LENGTH x_UNPARSED_ARGUMENTS x_len)
if(NOT x_len EQUAL "0")
hunter_internal_error(
"'${function_name}' incorrect usage,"
" expected no free arguments '${x_UNPARSED_ARGUMENTS}'."
" Synopsis: ${function_synopsis}"
)
endif()

# check mandatory arguments
foreach(arg PACKAGE VERSION SHA1 URL OUTPUT)
string(COMPARE EQUAL "${x_${arg}}" "" is_empty)
if(is_empty)
hunter_internal_error(
"'${function_name}' incorrect usage,"
" option '${arg}' with one argument is mandatory."
" Synopsis: ${function_synopsis}"
)
endif()
endforeach()

# default to the given URL
set(hunter_package_new_url "${x_URL}")

# check if download server is given
string(COMPARE NOTEQUAL "${HUNTER_DOWNLOAD_SERVER}" "" use_download_server)
if(NOT use_download_server)
# nothing to be done, set variable and exit
set(${x_OUTPUT} ${hunter_package_new_url} PARENT_SCOPE)
return()
endif()

# update download url to point to HUNTER_DOWNLOAD_SERVER
set(hunter_main_url) # variable for main_url (x_URL if it matches one of the download server)
set(hunter_url_list) # list of URLs to try downloading from

# extract archive-ID from archive SHA1
string(SUBSTRING "${x_SHA1}" 0 7 archive_id)

# get filename from download URL
get_filename_component(package_filename_raw "${x_URL}" NAME)
# replace special characters from filename
string(REGEX REPLACE "[!@#$%^&*?]" "_" package_filename "${package_filename_raw}")

foreach(list_item ${HUNTER_DOWNLOAD_SERVER})
# create new URL
set(download_server_url "${list_item}/${x_PACKAGE}/${x_VERSION}/${archive_id}/${package_filename}")

# check if package URL is in the download server list
string(FIND "${x_URL}" "${list_item}" found)
if(NOT (found EQUAL "-1"))
# if in list use the original URL instead of name mangling
hunter_status_print("DOWNLOAD_SERVER: \"${x_PACKAGE}\": URL \"${x_URL}\" matches list item \"${list_item}\".")
set(hunter_main_url "${x_URL}")
else()
# append URL to list of download URLs
list(APPEND hunter_url_list "${download_server_url}")
endif()
endforeach()

# check version to determine if external_project_add can handle multiple download URLs
set(multiple_urls_allowed TRUE)
if(CMAKE_VERSION VERSION_LESS "3.7")
set(msg "DOWNLOAD_SERVER: Multiple URLs are not supported,")
set(msg "${msg} only one URL will be used as download server.")
set(msg "${msg} Use CMake 3.7+ for multiple URLs (current version: ${CMAKE_VERSION})")
hunter_status_debug("${msg}")
set(multiple_urls_allowed FALSE)
endif()

string(COMPARE NOTEQUAL "${hunter_main_url}" "" has_main_url)
if(has_main_url)
if(multiple_urls_allowed)
# highest priority for original URL
set(hunter_package_new_url "${hunter_main_url}" ${hunter_url_list})
else()
# use only original URL
set(hunter_package_new_url "${hunter_main_url}")
endif()
else()
if(multiple_urls_allowed)
# try all download server one after another
set(hunter_package_new_url ${hunter_url_list})
else()
# use only first server
list(GET hunter_url_list 0 hunter_package_new_url)
endif()
endif()

hunter_status_print("DOWNLOAD_SERVER: replacing URL
PACKAGE: \"${x_PACKAGE}\"
VERSION: \"${x_VERSION}\"
SHA1: \"${x_SHA1}\"
old URL: \"${x_URL}\"
new URL: \"${hunter_package_new_url}\"")

set(download_command "./download_package_for_server.sh")
set(download_command "${download_command} --PACKAGE \"${x_PACKAGE}\"")
set(download_command "${download_command} --VERSION \"${x_VERSION}\" ")
set(download_command "${download_command} --SHA1 \"${x_SHA1}\" ")
set(download_command "${download_command} --URL \"${x_URL}\"")
hunter_status_print("DOWNLOAD_SERVER: download with maintenance-script:
${download_command}")

# set output_var to found definition
set(${x_OUTPUT} ${hunter_package_new_url} PARENT_SCOPE)
endfunction()
2 changes: 2 additions & 0 deletions docs/quick-start/cmake.rst
Expand Up @@ -37,6 +37,8 @@ Notes about version of CMake
:doc:`protected sources </user-guides/cmake-user/protected-sources>`
* ``USERPWD`` sub-option for ``file(DOWNLOAD|UPLOAD ...)`` (`change <https://github.com/Kitware/CMake/commit/e5ba1041be862212a3ad66bd51930fc7beeb8140>`__)
* ``HTTP_{USERNAME|PASSWORD}`` sub-options for ``ExternalProject_Add`` (`change <https://github.com/Kitware/CMake/commit/e1ca117332fbf6adf3a467a420804e9cb1891582>`__)
* List of URLs can be passed to ``ExternalProject_Add``.
Used by :ref:`hunter download server`.

* `3.7.1`_ **Minimum for Android projects**

Expand Down
14 changes: 14 additions & 0 deletions docs/reference/layouts/deployed.rst
@@ -1,9 +1,14 @@
.. Copyright (c) 2016-2017, Ruslan Baratov
.. All rights reserved.
.. _layout deployed:

Deployed
--------

.. _layout deployed common:

Common
======

Expand Down Expand Up @@ -51,6 +56,8 @@ For example we have file ``toolchain.info`` and we want to save it in

* Unlock ``<Toolchain-ID>/cmake.lock``

.. _layout deployed base:

Base
====

Expand Down Expand Up @@ -116,6 +123,8 @@ for development.
├── Cellar/ # see below
└── Cache/ # see below
.. _layout deployed download:

Download
========

Expand Down Expand Up @@ -148,6 +157,8 @@ Hunter code).
└── Unpacked/ # Unpacked Hunter archive (HUNTER_SELF)
.. _layout deployed cache:

Cache
=====

Expand Down Expand Up @@ -202,6 +213,9 @@ arguments, dependencies, etc.).
├─ CACHE.DONE # stamp: deps.info and cache.sha1 created and ready to be used
└─ from.server # info downloaded from server, no need to upload this entry
.. _layout deployed cellar:

Cellar
======

Expand Down
61 changes: 61 additions & 0 deletions docs/reference/user-variables.rst
Expand Up @@ -208,6 +208,67 @@ and have some usage peculiarities:
for build. Combining with previous peculiarity it's expected that much
more disk space will be used than usually.

.. _hunter download server:

HUNTER_DOWNLOAD_SERVER
======================

Define a list of servers to download from.

We define the following packages for the examples:

- Package 1 name: ``foo``
- Package 1 SHA1: ``49dee30c5fedd8613a144f9bf6551fb46bb69e92``
- Package 1 URL: ``https://foo.com/downloads/foo-1.0.tar.gz``

- Package 2 name: ``boo``
- Package 2 SHA1: ``b1ec7331baf4c9996497851bfa2c847a73cd6085``
- Package 2 URL: ``https://server-2.com/downloads/boo-3.0.tar.gz``

If ``HUNTER_DOWNLOAD_SERVER`` is empty nothing changes and the following URLs
are used to download the sources:

- ``foo``: ``https://foo.com/downloads/foo-1.0.tar.gz``
- ``boo``: ``https://server-2.com/downloads/boo-3.0.tar.gz``

If ``HUNTER_DOWNLOAD_SERVER`` is a list of servers like
``https://server-1.com;https://server-2.com;https://server-3.com``
then the original package URL is analyzed. If the original URL matches one of the
defined servers we leave it untouched and set as a server with high priority.

For package ``foo`` the following URLs are passed to ``ExternalProject_Add``
(the original URL is not used):

- ``https://server-1.com/foo/1.0/SHASUM/foo-1.0.tar.gz``
- ``https://server-2.com/foo/1.0/SHASUM/foo-1.0.tar.gz``
- ``https://server-3.com/foo/1.0/SHASUM/foo-1.0.tar.gz``

For package ``boo`` the following URLs are passed to ``ExternalProject_Add``
(the original URL has the highest priority):

- ``https://server-2.com/downloads/boo-3.0.tar.gz`` (take priority, original URL used)
- ``https://server-1.com/boo/3.0/SHASUM/boo-3.0.tar.gz``
- ``https://server-3.com/boo/3.0/SHASUM/boo-3.0.tar.gz``

.. note::

Multiple URLs are supported only with CMake 3.7+. For earlier versions
the first listed URL is passed to ``ExternalProject_Add``.

The retry logic is implemented in the CMake function ``ExternalProject_Add``.

To create new URLs the following template is used:

``${HUNTER_DOWNLOAD_SERVER}/${PACKAGE_NAME}/${PACKAGE_VERSION}/${ARCHIVE_ID}/${filename}``

- The characters ``!@#$%^&*?`` occurring in ``${filename}`` are replaced with ``_``.
- ``${ARCHIVE_ID}`` is the first 7 characters of the package archive ``SHA1`` sum.

.. note::

This is the same structure as Hunter uses for its own :ref:`Download <layout deployed download>` directory.


Environment
~~~~~~~~~~~

Expand Down

0 comments on commit b818355

Please sign in to comment.