Merge remote-tracking branch 'upstream/develop' into develop

* upstream/develop: (32 commits)
  Libcloud DNS module fix (saltstack#34628)
  Properly pass in userdata as string (saltstack#34623)
  Add NILinuxRT support to timezone execution module (saltstack#34556)
  Properly gate makostack (saltstack#34614)
  Add support for edge case when Cmd and Entrypoint can't be blanked (saltstack#34593)
  Update github IP for ssh state integration tests (saltstack#34592)
  When sorting list actual_data, make it a list (saltstack#34590)
  Avoid circular imports when calling salt.utils functions (saltstack#34586)
  [2016.3] Avoid salt.utils circular imports when using "from" (saltstack#34585)
  Adds a mock required for the network settings beacon
  Avoid circular imports when calling salt.utils functions (saltstack#34584)
  Gate docker unit test to check for docker (saltstack#34591)
  Typo (`defaul` → `default`)
  Add code formatting
  Fix typo (`seens` → `seen`)
  Add a bunch of documentation on getting files from other environments (saltstack#34560)
  Rename unit.states.boto_secgroup to unit.states.boto_secgroup_test
  Handle cases where Docker Remote API returns an empty ExecutionDriver
  Rename tests.unit.simple to tests.unit.simple_test
  salt/modules/zypper.py: accept ignore_epoch argument
  ...

doc/conf.py

@@ -144,6 +144,8 @@ def __getattr__(cls, name):
     'netaddr.IPAddress',
     'netaddr.core',
     'netaddr.core.AddrFormatError',
+    'pyroute2',
+    'pyroute2.ipdb',
 ]
 
 for mod_name in MOCK_MODULES:

doc/ref/configuration/master.rst

@@ -1264,6 +1264,10 @@ Example:
       - roots
       - git
 
+.. note::
+    For masterless Salt, this parameter must be specified in the minion config
+    file.
+
 .. conf_master:: fileserver_followsymlinks
 
 ``fileserver_followsymlinks``
@@ -1431,6 +1435,10 @@ Example:
         - /srv/salt/prod/services
         - /srv/salt/prod/states
 
+.. note::
+    For masterless Salt, this parameter must be specified in the minion config
+    file.
+
 git: Git Remote File Server Backend
 -----------------------------------
 

doc/ref/file_server/all/salt.fileserver.azurefs.rst

@@ -3,4 +3,3 @@ salt.fileserver.azurefs
 =======================
 
 .. automodule:: salt.fileserver.azurefs
-    :members:

doc/ref/file_server/all/salt.fileserver.gitfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.gitfs
 =====================
 
 .. automodule:: salt.fileserver.gitfs
-    :members:

doc/ref/file_server/all/salt.fileserver.hgfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.hgfs
 ====================
 
 .. automodule:: salt.fileserver.hgfs
-    :members:

doc/ref/file_server/all/salt.fileserver.minionfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.minionfs
 ========================
 
 .. automodule:: salt.fileserver.minionfs
-    :members:

doc/ref/file_server/all/salt.fileserver.roots.rst

@@ -3,4 +3,3 @@ salt.fileserver.roots
 =====================
 
 .. automodule:: salt.fileserver.roots
-    :members:

doc/ref/file_server/all/salt.fileserver.s3fs.rst

@@ -3,4 +3,3 @@ salt.fileserver.s3fs
 ====================
 
 .. automodule:: salt.fileserver.s3fs
-    :members:

doc/ref/file_server/all/salt.fileserver.svnfs.rst

@@ -3,4 +3,3 @@ salt.fileserver.svnfs
 =====================
 
 .. automodule:: salt.fileserver.svnfs
-    :members:

doc/ref/file_server/backends.rst

@@ -50,8 +50,8 @@ With this configuration, the environments and files defined in the
 not found then the git repositories defined in :conf_master:`gitfs_remotes`
 will be searched.
 
-Environments
-------------
+Defining Environments
+---------------------
 
 Just as the order of the values in :conf_master:`fileserver_backend` matters,
 so too does the order in which different sources are defined within a

doc/ref/file_server/environments.rst

@@ -0,0 +1,89 @@
+.. _file-server-environments:
+
+===========================================
+Requesting Files from Specific Environments
+===========================================
+
+The Salt fileserver supports multiple environments, allowing for SLS files and
+other files to be isolated for better organization.
+
+For the default backend (called :py:mod:`roots <salt.fileserver.roots>`),
+environments are defined using the :conf_master:`roots <file_roots>` option.
+Other backends (such as :py:mod:`gitfs <salt.fileserver.gitfs>`) define
+environments in their own ways. For a list of available fileserver backends,
+see :ref:`here <all-salt.fileserver>`.
+
+.. _querystring-syntax:
+
+Querystring Syntax
+==================
+
+Any ``salt://`` file URL can specify its fileserver environment using a
+querystring syntax, like so:
+
+.. code-block:: bash
+
+    salt://path/to/file?saltenv=foo
+
+In :ref:`Reactor <reactor>` configurations, this method must be used to pull
+files from an environment other than ``base``.
+
+In States
+=========
+
+Minions can be instructed which environment to use both globally, and for a
+single state, and multiple methods for each are available:
+
+Globally
+--------
+
+A minion can be pinned to an environment using the :conf_minion:`environment`
+option in the minion config file.
+
+Additionally, the environment can be set for a single call to the following
+functions:
+
+- :py:mod:`state.apply <salt.modules.state.apply>`
+- :py:mod:`state.highstate <salt.modules.state.highstate>`
+- :py:mod:`state.sls <salt.modules.state.sls>`
+- :py:mod:`state.top <salt.modules.state.top>`
+
+.. note::
+    When the ``saltenv`` parameter is used to trigger a :ref:`highstate
+    <running-highstate>` using either :py:mod:`state.apply
+    <salt.modules.state.apply>` or :py:mod:`state.highstate
+    <salt.modules.state.highstate>`, only states from that environment will be
+    applied.
+
+On a Per-State Basis
+--------------------
+
+Within an individual state, there are two ways of specifying the environment.
+The first is to add a ``saltenv`` argument to the state. This example will pull
+the file from the ``config`` environment:
+
+.. code-block:: yaml
+
+    /etc/foo/bar.conf:
+      file.managed:
+        - source: salt://foo/bar.conf
+        - user: foo
+        - mode: 600
+        - saltenv: config
+
+Another way of doing the same thing is to use the :ref:`querystring syntax
+<querystring-syntax>` described above:
+
+.. code-block:: yaml
+
+    /etc/foo/bar.conf:
+      file.managed:
+        - source: salt://foo/bar.conf?saltenv=config
+        - user: foo
+        - mode: 600
+
+.. note::
+    Specifying the environment using either of the above methods is only
+    necessary in cases where a state from one environment needs to access files
+    from another environment. If the SLS file containing this state was in the
+    ``config`` environment, then it would look in that environment by default.

doc/topics/reactor/index.rst

@@ -57,10 +57,13 @@ and each event tag has a list of reactor SLS files to be run.
         - /srv/reactor/destroy/*.sls    # Globs can be used to match file names
 
       - 'myco/custom/event/tag':        # React to custom event tags
-        - salt://_reactor/mycustom.sls  # Put reactor files under file_roots. Can be served from any filserver backend (GitFS, etc.)
-
-
+        - salt://reactor/mycustom.sls   # Reactor files can come from the salt fileserver
 
+.. note::
+    In the above example, ``salt://reactor/mycustom.sls`` refers to the
+    ``base`` environment. To pull this file from a different environment, use
+    the :ref:`querystring syntax <querystring-syntax>` (e.g.
+    ``salt://reactor/mycustom.sls?saltenv=reactor``).
 
 Reactor sls files are similar to state and pillar sls files.  They are
 by default yaml + Jinja templates and are passed familiar context variables.

doc/topics/tutorials/states_pt4.rst

@@ -78,6 +78,12 @@ environments, allowing them to be pushed to QA hosts and tested.
 Finally, if moved to the same relative path within ``/srv/salt/prod``, the
 files are now available in all three environments.
 
+Requesting files from specific fileserver environments
+======================================================
+
+See :ref:`here <file-server-environments>` for documentation on how to request
+files from specific environments.
+
 Practical Example
 =================
 

salt/cloud/clouds/ec2.py

@@ -1700,7 +1700,7 @@ def request_instance(vm_=None, call=None):
         blacklist = __opts__['renderer_blacklist']
         whitelist = __opts__['renderer_whitelist']
         userdata = compile_template(
-            userdata, rend, renderer, blacklist, whitelist
+            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
         )
 
         params[spot_prefix + 'UserData'] = base64.b64encode(userdata)

salt/cloud/deploy/bootstrap-salt.sh

@@ -18,11 +18,11 @@
 #======================================================================================================================
 set -o nounset                              # Treat unset variables as an error
 
-__ScriptVersion="2016.06.27"
+__ScriptVersion="2016.07.07"
 __ScriptName="bootstrap-salt.sh"
 
-__ScriptFullName="${0}"
-__ScriptArgs="${*}"
+__ScriptFullName="$0"
+__ScriptArgs="$*"
 
 #======================================================================================================================
 #  Environment variables taken into account.
@@ -630,7 +630,7 @@ fi
 
 echoinfo "Running version: ${__ScriptVersion}"
 echoinfo "Executed by: ${CALLER}"
-echoinfo "Command line: \"${__ScriptFullName} ${__ScriptArgs}\""
+echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'"
 #echowarn "Running the unstable version of ${__ScriptName}"
 
 #---  FUNCTION  -------------------------------------------------------------------------------------------------------
@@ -1180,6 +1180,7 @@ __ubuntu_derivatives_translation() {
 
     linuxmint_16_ubuntu_base="13.10"
     linuxmint_17_ubuntu_base="14.04"
+    linuxmint_18_ubuntu_base="16.04"
     linaro_12_ubuntu_base="12.04"
     elementary_os_02_ubuntu_base="12.04"
 
@@ -2366,6 +2367,11 @@ install_ubuntu_stable_deps() {
                 set -o nounset
             fi
 
+            # Make sure https transport is available
+            if [ "$HTTP_VAL" = "https" ] ; then
+                __apt_get_install_noinput ca-certificates apt-transport-https || return 1
+            fi
+
             # Make sure wget is available
             __apt_get_install_noinput wget
 
@@ -2714,6 +2720,8 @@ install_debian_7_deps() {
     if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then
         if [ "$CPU_ARCH_L" = "amd64" ] || [ "$CPU_ARCH_L" = "x86_64" ]; then
             repo_arch="amd64"
+        elif [ "$CPU_ARCH_L" = "armv7l" ]; then
+            repo_arch="armhf"
         elif [ "$CPU_ARCH_L" = "i386" ] || [ "$CPU_ARCH_L" = "i686" ]; then
             echoerror "repo.saltstack.com likely doesn't have 32-bit packages for Debian (yet?)"
             repo_arch="i386"
@@ -2790,6 +2798,8 @@ install_debian_8_deps() {
     if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then
         if [ "$CPU_ARCH_L" = "amd64" ] || [ "$CPU_ARCH_L" = "x86_64" ]; then
             repo_arch="amd64"
+        elif [ "$CPU_ARCH_L" = "armv7l" ]; then
+            repo_arch="armhf"
         elif [ "$CPU_ARCH_L" = "i386" ] || [ "$CPU_ARCH_L" = "i686" ]; then
             echoerror "repo.saltstack.com likely doesn't have 32-bit packages for Debian (yet?)"
             repo_arch="i386"
@@ -3113,7 +3123,13 @@ install_fedora_deps() {
         __install_saltstack_copr_salt_repository || return 1
     fi
 
-    __PACKAGES="yum-utils PyYAML libyaml m2crypto python-crypto python-jinja2 python-msgpack python-zmq python-requests"
+    __PACKAGES="yum-utils PyYAML libyaml m2crypto python-crypto python-jinja2 python-zmq"
+
+    if [ "$DISTRO_MAJOR_VERSION" -ge 23 ]; then
+        __PACKAGES="${__PACKAGES} python2-msgpack python2-requests"
+    else
+        __PACKAGES="${__PACKAGES} python-msgpack python-requests"
+    fi
 
     if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then
         __PACKAGES="${__PACKAGES} python-libcloud"
@@ -4082,6 +4098,14 @@ _eof
 }
 
 install_amazon_linux_ami_git_deps() {
+
+    # When installing from git, this variable might not be set yet for amazon linux. Set this
+    # to "latest" in order to set up the SaltStack repository and avoid a malformed baseurl
+    # and gpgkey reference in the install_amazon_linux_amI_deps function.
+    if [ "$STABLE_REV" = "" ]; then
+        STABLE_REV="latest"
+    fi
+
     install_amazon_linux_ami_deps || return 1
 
     if ! __check_command_exists git; then
@@ -4098,7 +4122,6 @@ install_amazon_linux_ami_git_deps() {
         fi
     fi
 
-
     # Let's trigger config_salt()
     if [ "$_TEMP_CONFIG_DIR" = "null" ]; then
         _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/"
@@ -4410,7 +4433,7 @@ __configure_freebsd_pkg_details() {
         echo "    url: \"${SALTPKGCONFURL}\","
         echo "    mirror_type: \"http\","
         echo "    enabled: true"
-        echo "    prioroity: 10"
+        echo "    priority: 10"
         echo "}"
     } > $salt_conf_file
     FROM_SALTSTACK="-r SaltStack"
@@ -5204,7 +5227,16 @@ install_opensuse_git_post() {
         [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue
 
         if [ -f /bin/systemctl ]; then
+            use_usr_lib=$BS_FALSE
             if [ "${DISTRO_MAJOR_VERSION}" -gt 13 ] || ([ "${DISTRO_MAJOR_VERSION}" -eq 13 ] && [ "${DISTRO_MINOR_VERSION}" -ge 2 ]); then
+                use_usr_lib=$BS_TRUE
+            fi
+
+            if [ "${DISTRO_MAJOR_VERSION}" -eq 12 ] && [ -d "/usr/lib/systemd/" ]; then
+                use_usr_lib=$BS_TRUE
+            fi
+
+            if [ "${use_usr_lib}" -eq $BS_TRUE ]; then
                 __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/usr/lib/systemd/system/salt-${fname}.service"
             else
                 __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service"
@@ -5371,7 +5403,7 @@ install_suse_12_stable_deps() {
 }
 
 install_suse_12_git_deps() {
-    install_suse_11_stable_deps || return 1
+    install_suse_12_stable_deps || return 1
 
     if ! __check_command_exists git; then
         __zypper_install git  || return 1

salt/modules/aptpkg.py

@@ -1332,18 +1332,30 @@ def upgrade_available(name):
     return latest_version(name) != ''
 
 
-def version_cmp(pkg1, pkg2):
+def version_cmp(pkg1, pkg2, ignore_epoch=False):
     '''
     Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if
     pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem
     making the comparison.
 
+    ignore_epoch : False
+        Set to ``True`` to ignore the epoch when comparing versions
+
+        .. versionadded:: 2015.8.10,2016.3.2
+
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' pkg.version_cmp '0.2.4-0ubuntu1' '0.2.4.1-0ubuntu1'
     '''
+    normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x)
+    # both apt_pkg.version_compare and _cmd_quote need string arguments.
+    pkg1 = normalize(pkg1)
+    pkg2 = normalize(pkg2)
+
+    # if we have apt_pkg, this will be quickier this way
+    # and also do not rely on shell.
     if HAS_APTPKG:
         try:
             # the apt_pkg module needs to be manually initialized