diff --git a/doc/ref/file_server/dynamic-modules.rst b/doc/ref/file_server/dynamic-modules.rst deleted file mode 100644 index 17bf149685fd..000000000000 --- a/doc/ref/file_server/dynamic-modules.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _module-sync: -.. _dynamic-module-distribution: - -=========================== -Dynamic Module Distribution -=========================== - -.. versionadded:: 0.9.5 - -Custom Salt execution, state, and other modules can be distributed to Salt -minions using the Salt file server. - -Under the root of any environment defined via the :conf_master:`file_roots` -option on the master server directories corresponding to the type of module can -be used. - -The directories are prepended with an underscore: - -- :file:`_beacons` -- :file:`_clouds` -- :file:`_engines` -- :file:`_grains` -- :file:`_modules` -- :file:`_output` -- :file:`_proxy` -- :file:`_renderers` -- :file:`_returners` -- :file:`_states` -- :file:`_tops` -- :file:`_utils` - -The contents of these directories need to be synced over to the minions after -Python modules have been created in them. There are a number of ways to sync -the modules. - -Sync Via States -=============== - -The minion configuration contains an option ``autoload_dynamic_modules`` -which defaults to ``True``. This option makes the state system refresh all -dynamic modules when states are run. To disable this behavior set -:conf_minion:`autoload_dynamic_modules` to ``False`` in the minion config. - -When dynamic modules are autoloaded via states, modules only pertinent to -the environments matched in the master's top file are downloaded. - -This is important to remember, because modules can be manually loaded from -any specific environment that environment specific modules will be loaded -when a state run is executed. - -Sync Via the saltutil Module -============================ - -The saltutil module has a number of functions that can be used to sync all -or specific dynamic modules. The saltutil module function ``saltutil.sync_all`` -will sync all module types over to a minion. For more information see: -:mod:`salt.modules.saltutil` diff --git a/doc/topics/beacons/index.rst b/doc/topics/beacons/index.rst index f2090388db9d..f8c197685e5e 100644 --- a/doc/topics/beacons/index.rst +++ b/doc/topics/beacons/index.rst @@ -290,6 +290,8 @@ All beacons are configured using a similar process of enabling the beacon, writing a reactor SLS (and state SLS if needed), and mapping a beacon event to the reactor SLS. +.. _writing-beacons: + Writing Beacon Plugins ====================== @@ -360,5 +362,5 @@ new execution modules and functions to back specific beacons. Distributing Custom Beacons --------------------------- -Custom beacons can be distributed to minions using ``saltutil``, see -:ref:`Dynamic Module Distribution `. +Custom beacons can be distributed to minions via the standard methods, see +:ref:`Modular Systems `. diff --git a/doc/topics/development/dunder_dictionaries.rst b/doc/topics/development/dunder_dictionaries.rst deleted file mode 100644 index f3ce3a8a86d4..000000000000 --- a/doc/topics/development/dunder_dictionaries.rst +++ /dev/null @@ -1,122 +0,0 @@ -.. _dunder-dictionaries: - -=================== -Dunder Dictionaries -=================== - -Salt provides several special "dunder" dictionaries as a convenience for Salt -development. These include ``__opts__``, ``__context__``, ``__salt__``, and -others. This document will describe each dictionary and detail where they exist -and what information and/or functionality they provide. - - -__opts__ --------- - -Available in -~~~~~~~~~~~~ - -- All loader modules - -The ``__opts__`` dictionary contains all of the options passed in the -configuration file for the master or minion. - -.. note:: - - In many places in salt, instead of pulling raw data from the __opts__ - dict, configuration data should be pulled from the salt `get` functions - such as config.get, aka - __salt__['config.get']('foo:bar') - The `get` functions also allow for dict traversal via the *:* delimiter. - Consider using get functions whenever using __opts__ or __pillar__ and - __grains__ (when using grains for configuration data) - -The configuration file data made available in the ``__opts__`` dictionary is the -configuration data relative to the running daemon. If the modules are loaded and -executed by the master, then the master configuration data is available, if the -modules are executed by the minion, then the minion configuration is -available. Any additional information passed into the respective configuration -files is made available - -__salt__ --------- - -Available in -~~~~~~~~~~~~ - -- Execution Modules -- State Modules -- Returners -- Runners -- SDB Modules - -``__salt__`` contains the execution module functions. This allows for all -functions to be called as they have been set up by the salt loader. - -.. code-block:: python - - __salt__['cmd.run']('fdisk -l') - __salt__['network.ip_addrs']() - -.. note:: - - When used in runners, ``__salt__`` references other runner modules, and not - execution modules. - -__grains__ ----------- - -Available in -~~~~~~~~~~~~ - -- Execution Modules -- State Modules -- Returners -- External Pillar - -The ``__grains__`` dictionary contains the grains data generated by the minion -that is currently being worked with. In execution modules, state modules and -returners this is the grains of the minion running the calls, when generating -the external pillar the ``__grains__`` is the grains data from the minion that -the pillar is being generated for. - -__pillar__ ------------ - -Available in -~~~~~~~~~~~~ - -- Execution Modules -- State Modules -- Returners - -The ``__pillar__`` dictionary contains the pillar for the respective minion. - -__context__ ------------ - -``__context__`` exists in state modules and execution modules. - -During a state run the ``__context__`` dictionary persists across all states -that are run and then is destroyed when the state ends. - -When running an execution module ``__context__`` persists across all module -executions until the modules are refreshed; such as when -:py:func:`saltutil.sync_all ` or -:py:func:`state.apply ` are executed. - -A great place to see how to use ``__context__`` is in the cp.py module in -salt/modules/cp.py. The fileclient authenticates with the master when it is -instantiated and then is used to copy files to the minion. Rather than create a -new fileclient for each file that is to be copied down, one instance of the -fileclient is instantiated in the ``__context__`` dictionary and is reused for -each file. Here is an example from salt/modules/cp.py: - -.. code-block:: python - - if not 'cp.fileclient' in __context__: - __context__['cp.fileclient'] = salt.fileclient.get_file_client(__opts__) - - -.. note:: Because __context__ may or may not have been destroyed, always be - sure to check for the existence of the key in __context__ and - generate the key before using it. diff --git a/doc/topics/development/index.rst b/doc/topics/development/index.rst index c4a341c0404b..65a7e6ee861f 100644 --- a/doc/topics/development/index.rst +++ b/doc/topics/development/index.rst @@ -7,6 +7,7 @@ Developing Salt :glob: * + modules/index extend/index tests/* raet/index diff --git a/doc/topics/development/modular_systems.rst b/doc/topics/development/modular_systems.rst deleted file mode 100644 index 6060ceea88cc..000000000000 --- a/doc/topics/development/modular_systems.rst +++ /dev/null @@ -1,170 +0,0 @@ -=============== -Modular Systems -=============== - -When first working with Salt, it is not always clear where all of the modular -components are and what they do. Salt comes loaded with more modular systems -than many users are aware of, making Salt very easy to extend in many places. - -The most commonly used modular systems are execution modules and states. But -the modular systems extend well beyond the more easily exposed components -and are often added to Salt to make the complete system more flexible. - -Execution Modules -================= - -Execution modules make up the core of the functionality used by Salt to -interact with client systems. The execution modules create the core system -management library used by all Salt systems, including states, which -interact with minion systems. - -Execution modules are completely open ended in their execution. They can -be used to do anything required on a minion, from installing packages to -detecting information about the system. The only restraint in execution -modules is that the defined functions always return a JSON serializable -object. - -For a list of all built in execution modules, click :ref:`here -` - -For information on writing execution modules, see :ref:`this page -`. - - -Interactive Debugging -===================== - -Sometimes debugging with ``print()`` and extra logs sprinkled everywhere is not -the best strategy. - -IPython is a helpful debug tool that has an interactive python environment -which can be embedded in python programs. - -First the system will require IPython to be installed. - -.. code-block:: bash - - # Debian - apt-get install ipython - - # Arch Linux - pacman -Syu ipython2 - - # RHEL/CentOS (via EPEL) - yum install python-ipython - - -Now, in the troubling python module, add the following line at a location where -the debugger should be started: - -.. code-block:: python - - test = 'test123' - import IPython; IPython.embed_kernel() - -After running a Salt command that hits that line, the following will show up in -the log file: - -.. code-block:: text - - [CRITICAL] To connect another client to this kernel, use: - [IPKernelApp] --existing kernel-31271.json - -Now on the system that invoked ``embed_kernel``, run the following command from -a shell: - -.. code-block:: bash - - # NOTE: use ipython2 instead of ipython for Arch Linux - ipython console --existing - -This provides a console that has access to all the vars and functions, and even -supports tab-completion. - -.. code-block:: python - - print(test) - test123 - -To exit IPython and continue running Salt, press ``Ctrl-d`` to logout. - - -State Modules -============= - -State modules are used to define the state interfaces used by Salt States. -These modules are restrictive in that they must follow a number of rules to -function properly. - -.. note:: - - State modules define the available routines in sls files. If calling - an execution module directly is desired, take a look at the `module` - state. - -Auth -==== - -The auth module system allows for external authentication routines to be easily -added into Salt. The `auth` function needs to be implemented to satisfy the -requirements of an auth module. Use the ``pam`` module as an example. - -Fileserver -========== - -The fileserver module system is used to create fileserver backends used by the -Salt Master. These modules need to implement the functions used in the -fileserver subsystem. Use the ``gitfs`` module as an example. - -Grains -====== - -Grain modules define extra routines to populate grains data. All defined -public functions will be executed and MUST return a Python dict object. The -dict keys will be added to the grains made available to the minion. - -Output -====== - -The output modules supply the outputter system with routines to display data -in the terminal. These modules are very simple and only require the `output` -function to execute. The default system outputter is the ``nested`` module. - -Pillar -====== - -Used to define optional external pillar systems. The pillar generated via -the filesystem pillar is passed into external pillars. This is commonly used -as a bridge to database data for pillar, but is also the backend to the libvirt -state used to generate and sign libvirt certificates on the fly. - -Renderers -========= - -Renderers are the system used to render sls files into salt highdata for the -state compiler. They can be as simple as the ``py`` renderer and as complex as -``stateconf`` and ``pydsl``. - -Returners -========= - -Returners are used to send data from minions to external sources, commonly -databases. A full returner will implement all routines to be supported as an -external job cache. Use the ``redis`` returner as an example. - -Runners -======= - -Runners are purely master-side execution sequences. - -Tops -==== - -Tops modules are used to convert external data sources into top file data for -the state system. - -Wheel -===== - -The wheel system is used to manage master side management routines. These -routines are primarily intended for the API to enable master configuration. diff --git a/doc/topics/development/modules/configuration.rst b/doc/topics/development/modules/configuration.rst new file mode 100644 index 000000000000..c794830f8027 --- /dev/null +++ b/doc/topics/development/modules/configuration.rst @@ -0,0 +1,25 @@ +===================== +Configuration Options +===================== + +A number of configuration options can affect the load process. This is a quick +list of them: + +* ``autoload_dynamic_modules`` (:conf_minion:`Minion `) +* ``cython_enable`` (:conf_minion:`Minion `, :conf_master:`Master `) +* ``disable_modules`` (:conf_minion:`Minion `) +* ``disable_returners`` (:conf_minion:`Minion `) +* ``enable_zip_modules`` (:conf_minion:`Minion `) +* ``extension_modules`` (:conf_master:`Master `) +* ``extmod_whitelist`` (:conf_minion:`Minion `, :conf_master:`Master `) +* ``extmod_blacklist`` (:conf_minion:`Minion `, :conf_master:`Master `) +* ``whitelist_modules`` (:conf_minion:`Minion `) +* ``grains_dirs`` (:conf_minion:`Minion `) +* ``module_dirs`` (:conf_minion:`Minion `, :conf_master:`Master `) +* ``outputter_dirs`` (:conf_minion:`Minion `, :conf_master:`Master `) +* ``providers`` (:conf_minion:`Minion `) +* ``render_dirs`` (:conf_minion:`Minion `) +* ``returner_dirs`` (:conf_minion:`Minion `) +* ``runner_dirs`` (:conf_master:`Master `) +* ``states_dirs`` (:conf_minion:`Minion `) +* ``utils_dirs`` (:conf_minion:`Minion `) diff --git a/doc/topics/development/modules/developing.rst b/doc/topics/development/modules/developing.rst new file mode 100644 index 000000000000..a01137d744a7 --- /dev/null +++ b/doc/topics/development/modules/developing.rst @@ -0,0 +1,237 @@ +====================== +Developing New Modules +====================== + +Interactive Debugging +===================== + +Sometimes debugging with ``print()`` and extra logs sprinkled everywhere is not +the best strategy. + +IPython is a helpful debug tool that has an interactive python environment +which can be embedded in python programs. + +First the system will require IPython to be installed. + +.. code-block:: bash + + # Debian + apt-get install ipython + + # Arch Linux + pacman -Syu ipython2 + + # RHEL/CentOS (via EPEL) + yum install python-ipython + + +Now, in the troubling python module, add the following line at a location where +the debugger should be started: + +.. code-block:: python + + test = 'test123' + import IPython; IPython.embed_kernel() + +After running a Salt command that hits that line, the following will show up in +the log file: + +.. code-block:: text + + [CRITICAL] To connect another client to this kernel, use: + [IPKernelApp] --existing kernel-31271.json + +Now on the system that invoked ``embed_kernel``, run the following command from +a shell: + +.. code-block:: bash + + # NOTE: use ipython2 instead of ipython for Arch Linux + ipython console --existing + +This provides a console that has access to all the vars and functions, and even +supports tab-completion. + +.. code-block:: python + + print(test) + test123 + +To exit IPython and continue running Salt, press ``Ctrl-d`` to logout. + +Special Module Contents +======================= + +These are things that may be defined by the module to influence various things. + +__virtual__ +----------- + +__virtual_aliases__ +------------------- + +__virtualname__ +--------------- + +__init__ +-------- + +Called before ``__virtual__()`` + +__proxyenabled__ +---------------- +grains and proxy modules + +__proxyenabled__ as a list containing the names of the proxy types that the module supports. + +__load__ +-------- + +__func_alias__ +-------------- + +__outputter__ +------------- + +.. _dunder-dictionaries: + +Dunder Dictionaries +=================== + +Salt provides several special "dunder" dictionaries as a convenience for Salt +development. These include ``__opts__``, ``__context__``, ``__salt__``, and +others. This document will describe each dictionary and detail where they exist +and what information and/or functionality they provide. + +The following dunder dictionaries are always defined, but may be empty + +* ``__context__`` +* ``__grains__`` +* ``__pillar__`` +* ``__opts__`` + + +__opts__ +-------- + +Defined in: All modules + +The ``__opts__`` dictionary contains all of the options passed in the +configuration file for the master or minion. + +.. note:: + + In many places in salt, instead of pulling raw data from the __opts__ + dict, configuration data should be pulled from the salt `get` functions + such as config.get, aka - ``__salt__['config.get']('foo:bar')`` + The `get` functions also allow for dict traversal via the *:* delimiter. + Consider using get functions whenever using ``__opts__`` or ``__pillar__`` + and ``__grains__`` (when using grains for configuration data) + +The configuration file data made available in the ``__opts__`` dictionary is the +configuration data relative to the running daemon. If the modules are loaded and +executed by the master, then the master configuration data is available, if the +modules are executed by the minion, then the minion configuration is +available. Any additional information passed into the respective configuration +files is made available + +__salt__ +-------- + +Defined in: Auth, Beacons, Engines, Execution, Executors, Outputters, Pillars, +Proxies, Renderers, Returners, Runners, SDB, SSH Wrappers, State, Thorium + +``__salt__`` contains the execution module functions. This allows for all +functions to be called as they have been set up by the salt loader. + +.. code-block:: python + + __salt__['cmd.run']('fdisk -l') + __salt__['network.ip_addrs']() + +.. note:: + + When used in runners or outputters, ``__salt__`` references other + runner/outputter modules, and not execution modules. + +__grains__ +---------- + +Filled in for: Execution, Pillar, Renderer, Returner, SSH Wrapper, State. + +The ``__grains__`` dictionary contains the grains data generated by the minion +that is currently being worked with. In execution modules, state modules and +returners this is the grains of the minion running the calls, when generating +the external pillar the ``__grains__`` is the grains data from the minion that +the pillar is being generated for. + +While ``__grains__`` is defined for every module, it's only filled in for some. + +__pillar__ +----------- + +Filled in for: Execution, Returner, SSH Wrapper, State + +The ``__pillar__`` dictionary contains the pillar for the respective minion. + +While ``__pillar__`` is defined for every module, it's only filled in for some. + +__context__ +----------- + +During a state run the ``__context__`` dictionary persists across all states +that are run and then is destroyed when the state ends. + +When running an execution module ``__context__`` persists across all module +executions until the modules are refreshed; such as when +:py:func:`saltutil.sync_all ` or +:py:func:`state.apply ` are executed. + +A great place to see how to use ``__context__`` is in the cp.py module in +salt/modules/cp.py. The fileclient authenticates with the master when it is +instantiated and then is used to copy files to the minion. Rather than create a +new fileclient for each file that is to be copied down, one instance of the +fileclient is instantiated in the ``__context__`` dictionary and is reused for +each file. Here is an example from salt/modules/cp.py: + +.. code-block:: python + + if not 'cp.fileclient' in __context__: + __context__['cp.fileclient'] = salt.fileclient.get_file_client(__opts__) + + +.. note:: Because __context__ may or may not have been destroyed, always be + sure to check for the existence of the key in __context__ and + generate the key before using it. + +__utils__ +--------- +Defined in: Cloud, Engine, Execution, File Server, Pillar, Proxy, Runner, SDB. + +__proxy__ +--------- +Defined in: Beacon, Engine, Execution, Executor, Proxy, Renderer, Returner, State, Util + +__runners__ +----------- +Defined in: Engine, Roster, Thorium + +__ret__ +------- +Defined in: Proxy, Search + +__thorium__ +----------- +Defined in: Thorium + +__states__ +---------- +Defined in: Renderers, State + +__serializers__ +--------------- +Defined in: State + +__sdb__ +------- +Defined in: SDB diff --git a/doc/topics/development/external_pillars.rst b/doc/topics/development/modules/external_pillars.rst similarity index 100% rename from doc/topics/development/external_pillars.rst rename to doc/topics/development/modules/external_pillars.rst diff --git a/doc/topics/development/modules/index.rst b/doc/topics/development/modules/index.rst new file mode 100644 index 000000000000..e421c91f3130 --- /dev/null +++ b/doc/topics/development/modules/index.rst @@ -0,0 +1,394 @@ +.. _modular-systems: + +=============== +Modular Systems +=============== + +When first working with Salt, it is not always clear where all of the modular +components are and what they do. Salt comes loaded with more modular systems +than many users are aware of, making Salt very easy to extend in many places. + +The most commonly used modular systems are execution modules and states. But +the modular systems extend well beyond the more easily exposed components +and are often added to Salt to make the complete system more flexible. + +.. toctree:: + :maxdepth: 2 + :glob: + + developing + configuration + + +Loading Modules +=============== + +Modules come primarily from several sources: + +* The Salt package itself +* The Salt File Server +* The extmods directory +* Secondary packages installed + +Using one source to override another is not supported. + +The Salt Package +---------------- + +Salt itself ships with a large number of modules. These are part of the Salt +package itself and don't require the user to do anything to use them. (Although +a number of them have additional dependencies and/or configuration.) + +The Salt File Server +-------------------- + +The user may add modules by simply placing them in special directories in their +:ref:`fileserver `. + +The name of the directory inside of the file server is the directory name +prepended by underscore, such as: + +- :file:`_grains` +- :file:`_modules` +- :file:`_states` + +Modules must be synced before they can be used. This can happen a few ways, +discussed below. + +.. note: + Using saltenvs besides ``base`` may not work in all contexts. + +Sync Via States +~~~~~~~~~~~~~~~ + +The minion configuration contains an option ``autoload_dynamic_modules`` +which defaults to ``True``. This option makes the state system refresh all +dynamic modules when states are run. To disable this behavior set +:conf_minion:`autoload_dynamic_modules` to ``False`` in the minion config. + +When dynamic modules are autoloaded via states, only the modules defined in the +same saltenvs as the states currently being run. + +Sync Via the saltutil Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The saltutil module has a number of functions that can be used to sync all +or specific dynamic modules. The ``saltutil.sync_*`` +:py:mod:`execution functions ` and +:py:mod:`runner functions ` can be used to sync modules +to minions and the master, respectively. + + +The extmods Directory +--------------------- + +Any files places in the directory set by ``extension_modules`` settings +(:conf_minion:`minion `, +:conf_master:`master `, default +``/var/cache/salt/*/extmods``) can also be loaded as modules. Note that these +directories are also used by the ``saltutil.sync_*`` functions (mentioned +above) and files may be overwritten. + +Secondary Packages +------------------ + +Third-party packages may also add modules to Salt if they are installed in the +same system and Python environment as the Salt Minion or Master. + +This is done via setuptools entry points: + +.. code-block:: python + + setup( + # ... + entry_points={ + 'salt.loader': [ + 'module_dirs=spirofs.loader:module', + ], + }, + # ... + ) + +Note that these are not synced from the Salt Master to the Minions. They must be +installed indepdendently on each Minion. + +Module Types +============ + +The specific names used by each loading method above are as follows. See sections below +for a short summary of each of these systems. + +.. _module-name-table: + +============ ================================================================ ========================= ===================== +Module Type Salt Package Name FS/Directory Name Entry Point +============ ================================================================ ========================= ===================== +Auth ``salt.auth`` (:ref:`index `) ``auth`` [#no-fs]_ ``auth_dirs`` +Beacon ``salt.beacons`` (:ref:`index `) ``beacons`` ``beacons_dirs`` +Cache ``salt.cache`` (:ref:`index `) ``cache`` ``cache_dirs`` +Cloud ``salt.cloud.clouds`` (:ref:`index `) ``clouds`` ``cloud_dirs`` +Engine ``salt.engines`` (:ref:`index `) ``engines`` ``engines_dirs`` +Execution ``salt.modules`` (:ref:`index `) ``modules`` ``module_dirs`` +Executor ``salt.executors`` (:ref:`index `) ``executors`` [#no-fs]_ ``executor_dirs`` +File Server ``salt.fileserver`` (:ref:`index `) ``fileserver`` [#no-fs]_ ``fileserver_dirs`` +Grain ``salt.grains`` (:ref:`index `) ``grains`` ``grains_dirs`` +Log Handler ``salt.log.handlers`` (:ref:`index `) ``log_handlers`` ``log_handlers_dirs`` +Net API ``salt.netapi`` (:ref:`index `) ``netapi`` [#no-fs]_ ``netapi_dirs`` +Outputter ``salt.output`` (:ref:`index `) ``output`` ``outputter_dirs`` +Pillar ``salt.pillar`` (:ref:`index `) ``pillar`` ``pillar_dirs`` +Proxy ``salt.proxy`` (:ref:`index `) ``proxy`` ``proxy_dirs`` +Queue ``salt.queues`` (:ref:`index `) ``queues`` ``queue_dirs`` +Renderer ``salt.renderers`` (:ref:`index `) ``renderers`` ``render_dirs`` +Returner ``salt.returners`` (:ref:`index `) ``returners`` ``returner_dirs`` +Roster ``salt.roster`` (:ref:`index `) ``roster`` ``roster_dirs`` +Runner ``salt.runners`` (:ref:`index `) ``runners`` ``runner_dirs`` +SDB ``salt.sdb`` (:ref:`index `) ``sdb`` ``sdb_dirs`` +Search ``salt.search`` ``search`` [#no-fs]_ ``search_dirs`` +Serializer ``salt.serializers`` (:ref:`index `) ``serializers`` [#no-fs]_ ``serializers_dirs`` +SPM pkgdb ``salt.spm.pkgdb`` ``pkgdb`` [#no-fs]_ ``pkgdb_dirs`` +SPM pkgfiles ``salt.spm.pkgfiles`` ``pkgfiles`` [#no-fs]_ ``pkgfiles_dirs`` +SSH Wrapper ``salt.client.ssh.wrapper`` ``wrapper`` [#no-fs]_ ``wrapper_dirs`` +State ``salt.states`` (:ref:`index `) ``states`` ``states_dirs`` +Thorium ``salt.thorium`` (:ref:`index `) ``thorium`` [#no-fs]_ ``thorium_dirs`` +Top ``salt.tops`` (:ref:`index `) ``tops`` ``top_dirs`` +Util ``salt.utils`` ``utils`` ``utils_dirs`` +Wheel ``salt.wheels`` (:ref:`index `) ``wheel`` ``wheel_dirs`` +============ ================================================================ ========================= ===================== + +.. [#no-fs] These modules cannot be loaded from the Salt File Server. + +.. note: + While it is possible to import modules directly with the import statement, + it is strongly recommended that the appropriate + :ref:`dunder dictionary ` is used to access them + instead. This is because a number of factors affect module names, module + selection, and module overloading. + +Auth +---- + +The auth module system allows for external authentication routines to be easily +added into Salt. The `auth` function needs to be implemented to satisfy the +requirements of an auth module. Use the ``pam`` module as an example. + +See :ref:`External Authentication System ` for more about +authentication in Salt. + +Beacon +------ + +* :ref:`Writing Beacons ` + +Beacons are polled by the Salt event loop to monitor non-salt processes. See +:ref:`Beacons ` for more information about the beacon system. + +Cache +----- + +The minion cache is used by the master to store various information about +minions. See :ref:`Minion Data Cache ` for more information. + +Cloud +----- + +Cloud modules are backend implementations used by :ref:`Salt Cloud `. + +Engine +------ + +Engines are open-ended services managed by the Salt daemon (both master and +minion). They may interact with event loop, call other modules, or a variety of +non-salt tasks. See :ref:`Salt Engines ` for complete details. + +Execution +--------- + +.. toctree:: + :maxdepth: 1 + :glob: + + /ref/modules/index + +Execution modules make up the core of the functionality used by Salt to +interact with client systems. The execution modules create the core system +management library used by all Salt systems, including states, which +interact with minion systems. + +Execution modules are completely open ended in their execution. They can +be used to do anything required on a minion, from installing packages to +detecting information about the system. The only restraint in execution +modules is that the defined functions always return a JSON serializable +object. + +Executor +-------- + +Executors control how execution modules get called. The default is to just call +them, but this can be customized. + +File Server +----------- + +The file server module system is used to create file server backends used by the +Salt Master. These modules need to implement the functions used in the +fileserver subsystem. Use the ``gitfs`` module as an example. + +See :ref:`File Server Backends ` for more information. + +Grains +------ + +* :ref:`writing-grains` + +Grain modules define extra routines to populate grains data. All defined +public functions will be executed and MUST return a Python dict object. The +dict keys will be added to the grains made available to the minion. + +See :ref:`Grains ` for more. + +Log Handler +----------- + +Log handlers allows the logs from salt (master or minion) to be sent to log +aggregation systems. + +Net API +------- + +Net API modules are the actual server implementation used by Salt API. + +Output +------ + +The output modules supply the outputter system with routines to display data +in the terminal. These modules are very simple and only require the `output` +function to execute. The default system outputter is the ``nested`` module. + +Pillar +------ + +.. toctree:: + :maxdepth: 1 + :glob: + + external_pillars + +Used to define optional external pillar systems. The pillar generated via +the filesystem pillar is passed into external pillars. This is commonly used +as a bridge to database data for pillar, but is also the backend to the libvirt +state used to generate and sign libvirt certificates on the fly. + +Proxy +----- + +:ref:`Proxy Minions ` are a way to manage devices that cannot run +a full minion directly. + +Renderers +--------- + +Renderers are the system used to render sls files into salt highdata for the +state compiler. They can be as simple as the ``py`` renderer and as complex as +``stateconf`` and ``pydsl``. + +Returners +--------- + +Returners are used to send data from minions to external sources, commonly +databases. A full returner will implement all routines to be supported as an +external job cache. Use the ``redis`` returner as an example. + +Roster +------ + +The :ref:`Roster system ` is used by Salt SSH to enumerate devices. + +Runners +------- + +.. toctree:: + :maxdepth: 1 + :glob: + + /ref/runners/index + +Runners are purely master-side execution sequences. + +SDB +--- + +* :ref:`Writing SDB Modules ` + +SDB is a way to store data that's not associated with a minion. See +:ref:`Storing Data in Other Databases `. + +Search +------ + +A system for indexing the file server and pillars. Removed in 2018.3. + +Serializer +---------- + +Primarily used with :py:func:`file.serialize `. + +State +----- + +.. toctree:: + :maxdepth: 1 + :glob: + + /ref/states/index + +State modules are used to define the state interfaces used by Salt States. +These modules are restrictive in that they must follow a number of rules to +function properly. + +.. note:: + + State modules define the available routines in sls files. If calling + an execution module directly is desired, take a look at the `module` + state. + +SPM pkgdb +--------- + +* :ref:`SPM Development Guide: Package Database ` + +pkgdb modules provides storage backends to the package database. + +SPM pkgfiles +------------ + +* :ref:`SPM Development Guide: Package Database ` + +pkgfiles modules handle the actual installation. + +SSH Wrapper +----------- + +Replacement execution modules for :ref:`Salt SSH `. + +Thorium +------- + +Modules for use in the :ref:`Thorium ` event reactor. + +Tops +---- + +Tops modules are used to convert external data sources into top file data for +the state system. + +Util +---- + +Just utility modules to use with other modules via ``__utils__`` (see +:ref:`Dunder Dictionaries `). + +Wheel +----- + +The wheel system is used to manage master side management routines. These +routines are primarily intended for the API to enable master configuration. diff --git a/doc/topics/sdb/index.rst b/doc/topics/sdb/index.rst index cdc66b21f60f..d772bf1f2ce3 100644 --- a/doc/topics/sdb/index.rst +++ b/doc/topics/sdb/index.rst @@ -144,6 +144,7 @@ When writing Salt modules, it is not recommended to call ``sdb.get`` directly, as it requires the user to provide values in SDB, using a specific URI. Use ``config.get`` instead. +.. _sdb-writing-modules: Writing SDB Modules =================== diff --git a/doc/topics/spm/dev.rst b/doc/topics/spm/dev.rst index ece953b4f06d..4a4d82129b26 100644 --- a/doc/topics/spm/dev.rst +++ b/doc/topics/spm/dev.rst @@ -20,6 +20,7 @@ marked as required are crucial to SPM's core functionality, while arguments that are marked as optional are provided as a benefit to the module, if it needs to use them. +.. _spm-development-pkgdb: Package Database ---------------- @@ -146,6 +147,8 @@ The only argument that is expected is ``db_``, which is the package database file. +.. _spm-development-pkgfiles: + Package Files ------------- By default, package files are installed using the ``local`` module. This module diff --git a/salt/client/ssh/ssh_py_shim.py b/salt/client/ssh/ssh_py_shim.py index 5e5dbdc55e86..55c4e7c854e8 100644 --- a/salt/client/ssh/ssh_py_shim.py +++ b/salt/client/ssh/ssh_py_shim.py @@ -303,19 +303,23 @@ def main(argv): # pylint: disable=W0613 if OPTIONS.cmd_umask is not None: old_umask = os.umask(OPTIONS.cmd_umask) # pylint: disable=blacklisted-function if OPTIONS.tty: + proc = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Returns bytes instead of string on python 3 - stdout, _ = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + stdout, _ = proc.communicate() sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace")) sys.stdout.flush() + retcode = proc.returncode if OPTIONS.wipe: shutil.rmtree(OPTIONS.saltdir) elif OPTIONS.wipe: - subprocess.call(salt_argv) + retcode = subprocess.call(salt_argv) shutil.rmtree(OPTIONS.saltdir) else: - subprocess.call(salt_argv) + retcode = subprocess.call(salt_argv) if OPTIONS.cmd_umask is not None: os.umask(old_umask) # pylint: disable=blacklisted-function + return retcode + if __name__ == '__main__': sys.exit(main(sys.argv)) diff --git a/salt/grains/core.py b/salt/grains/core.py index 5e03eb99fd10..c0af2744ec49 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1173,25 +1173,34 @@ def _windows_platform_data(): os_release = platform.release() kernel_version = platform.version() info = salt.utils.win_osinfo.get_os_version_info() - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - - # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function - # started reporting the Desktop version instead of the Server version on - # Server versions of Windows, so we need to look those up - # So, if you find a Server Platform that's a key in the server - # dictionary, then lookup the actual Server Release. - # (Product Type 1 is Desktop, Everything else is Server) - if info['ProductType'] > 1 and os_release in server: - os_release = server[os_release] service_pack = None if info['ServicePackMajor'] > 0: service_pack = ''.join(['SP', six.text_type(info['ServicePackMajor'])]) + # This creates the osrelease grain based on the Windows Operating + # System Product Name. As long as Microsoft maintains a similar format + # this should be future proof + version = 'Unknown' + release = '' + if 'Server' in osinfo.Caption: + for item in osinfo.Caption.split(' '): + # If it's all digits, then it's version + if re.match(r'\d+', item): + version = item + # If it starts with R and then numbers, it's the release + # ie: R2 + if re.match(r'^R\d+$', item): + release = item + os_release = '{0}Server{1}'.format(version, release) + else: + for item in osinfo.Caption.split(' '): + # If it's a number, decimal number, Thin or Vista, then it's the + # version + if re.match(r'^(\d+(\.\d+)?)|Thin|Vista$', item): + version = item + os_release = version + grains = { 'kernelrelease': _clean_value('kernelrelease', osinfo.Version), 'kernelversion': _clean_value('kernelversion', kernel_version), diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 9aac53640325..d1236f2d78e5 100644 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -332,7 +332,7 @@ def _connarg(name, key=None, get_opts=True): try: dbc = MySQLdb.connect(**connargs) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return None @@ -648,7 +648,7 @@ def query(database, query, **connection_args): try: affected = _execute(cur, query) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -773,7 +773,7 @@ def status(**connection_args): try: _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return {} @@ -804,7 +804,7 @@ def version(**connection_args): try: _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return '' @@ -837,7 +837,7 @@ def slave_lag(**connection_args): try: _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return -3 @@ -922,7 +922,7 @@ def db_list(**connection_args): try: _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return [] @@ -1011,7 +1011,7 @@ def db_tables(name, **connection_args): try: _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return [] @@ -1046,7 +1046,7 @@ def db_exists(name, **connection_args): try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1101,7 +1101,7 @@ def db_create(name, character_set=None, collate=None, **connection_args): log.info('DB \'%s\' created', name) return True except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1137,7 +1137,7 @@ def db_remove(name, **connection_args): try: _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1169,7 +1169,7 @@ def user_list(**connection_args): qry = 'SELECT User,Host FROM mysql.user' _execute(cur, qry) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return [] @@ -1252,7 +1252,7 @@ def user_exists(user, try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1284,7 +1284,7 @@ def user_info(user, host='localhost', **connection_args): try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1384,7 +1384,7 @@ def user_create(user, try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1499,7 +1499,7 @@ def user_chpass(user, try: result = _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1554,7 +1554,7 @@ def user_remove(user, try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1772,7 +1772,7 @@ def user_grants(user, try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1886,7 +1886,7 @@ def grant_add(grant, try: _execute(cur, qry['qry'], qry['args']) except (MySQLdb.OperationalError, MySQLdb.ProgrammingError) as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False @@ -1960,7 +1960,7 @@ def grant_revoke(grant, try: _execute(cur, qry, args) except MySQLdb.OperationalError as exc: - err = 'MySQL Error {0}: {1}'.format(*exc) + err = 'MySQL Error {0}: {1}'.format(*exc.args) __context__['mysql.error'] = err log.error(err) return False diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index ed332f099585..53c2b197ab39 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -502,6 +502,8 @@ def on_finish(self): ''' # timeout all the futures self.timeout_futures() + # clear local_client objects to disconnect event publisher's IOStream connections + del self.saltclients def on_connection_close(self): ''' @@ -931,14 +933,27 @@ def _disbatch_local(self, chunk): ''' Dispatch local client commands ''' - # Generate jid before triggering a job to subscribe all returns from minions - chunk['jid'] = salt.utils.jid.gen_jid(self.application.opts) + # Generate jid and find all minions before triggering a job to subscribe all returns from minions + chunk['jid'] = salt.utils.jid.gen_jid(self.application.opts) if not chunk.get('jid', None) else chunk['jid'] + minions = set(self.ckminions.check_minions(chunk['tgt'], chunk.get('tgt_type', 'glob'))) + + def subscribe_minion(minion): + salt_evt = self.application.event_listener.get_event( + self, + tag='salt/job/{}/ret/{}'.format(chunk['jid'], minion), + matcher=EventListener.exact_matcher) + syndic_evt = self.application.event_listener.get_event( + self, + tag='syndic/job/{}/ret/{}'.format(chunk['jid'], minion), + matcher=EventListener.exact_matcher) + return salt_evt, syndic_evt # start listening for the event before we fire the job to avoid races - events = [ - self.application.event_listener.get_event(self, tag='salt/job/'+chunk['jid']), - self.application.event_listener.get_event(self, tag='syndic/job/'+chunk['jid']), - ] + events = [] + for minion in minions: + salt_evt, syndic_evt = subscribe_minion(minion) + events.append(salt_evt) + events.append(syndic_evt) f_call = self._format_call_run_job_async(chunk) # fire a job off @@ -954,6 +969,12 @@ def _disbatch_local(self, chunk): pass raise tornado.gen.Return('No minions matched the target. No command was sent, no jid was assigned.') + # get_event for missing minion + for minion in list(set(pub_data['minions']) - set(minions)): + salt_evt, syndic_evt = subscribe_minion(minion) + events.append(salt_evt) + events.append(syndic_evt) + # Map of minion_id -> returned for all minions we think we need to wait on minions = {m: False for m in pub_data['minions']} @@ -1008,7 +1029,10 @@ def cancel_inflight_futures(): cancel_inflight_futures() raise tornado.gen.Return(chunk_ret) continue + f_result = f.result() + if f in events: + events.remove(f) # if this is a start, then we need to add it to the pile if f_result['tag'].endswith('/new'): for minion_id in f_result['data']['minions']: @@ -1018,7 +1042,6 @@ def cancel_inflight_futures(): chunk_ret[f_result['data']['id']] = f_result['data']['return'] # clear finished event future minions[f_result['data']['id']] = True - # if there are no more minions to wait for, then we are done if not more_todo() and min_wait_time.done(): cancel_inflight_futures() @@ -1027,11 +1050,6 @@ def cancel_inflight_futures(): except TimeoutException: pass - if f == events[0]: - events[0] = self.application.event_listener.get_event(self, tag='salt/job/'+chunk['jid']) - else: - events[1] = self.application.event_listener.get_event(self, tag='syndic/job/'+chunk['jid']) - @tornado.gen.coroutine def job_not_running(self, jid, tgt, tgt_type, minions, is_finished): ''' diff --git a/salt/pillar/pepa.py b/salt/pillar/pepa.py index ff270b37479c..f7ee83489be6 100644 --- a/salt/pillar/pepa.py +++ b/salt/pillar/pepa.py @@ -364,8 +364,8 @@ def key_value_to_tree(data): for flatkey, value in six.iteritems(data): t = tree keys = flatkey.split(__opts__['pepa_delimiter']) - for key in keys: - if key == keys[-1]: + for i, key in enumerate(keys, 1): + if i == len(keys): t[key] = value else: t = t.setdefault(key, {}) diff --git a/salt/renderers/py.py b/salt/renderers/py.py index 8effd35cbe26..f156a9bee3eb 100644 --- a/salt/renderers/py.py +++ b/salt/renderers/py.py @@ -47,8 +47,8 @@ ``/srv/salt/foo/bar/baz.sls``, then ``__sls__`` in that file will be ``foo.bar.baz``. -The global context ``data`` (same as context ``{{ data }}`` for states written -with Jinja + YAML). The following YAML + Jinja state declaration: +When writing a reactor SLS file the global context ``data`` (same as context ``{{ data }}`` +for states written with Jinja + YAML) is available. The following YAML + Jinja state declaration: .. code-block:: jinja diff --git a/salt/states/service.py b/salt/states/service.py index 0b9f9babf1af..a0e553ead9f4 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -422,9 +422,16 @@ def running(name, else: before_toggle_enable_status = True + unmask_ret = {'comment': ''} + if unmask: + unmask_ret = unmasked(name, unmask_runtime) + # See if the service is already running if before_toggle_status: - ret['comment'] = 'The service {0} is already running'.format(name) + ret['comment'] = '\n'.join( + [_f for _f in ['The service {0} is already running'.format(name), + unmask_ret['comment']] if _f] + ) if enable is True and not before_toggle_enable_status: ret.update(_enable(name, None, **kwargs)) elif enable is False and before_toggle_enable_status: @@ -434,7 +441,9 @@ def running(name, # Run the tests if __opts__['test']: ret['result'] = None - ret['comment'] = 'Service {0} is set to start'.format(name) + ret['comment'] = '\n'.join( + [_f for _f in ['Service {0} is set to start'.format(name), + unmask_ret['comment']] if _f]) return ret # Conditionally add systemd-specific args to call to service.start @@ -494,6 +503,9 @@ def running(name, .format(ret['comment'], init_delay) ) + if unmask: + ret['comment'] = '\n'.join([ret['comment'], unmask_ret['comment']]) + return ret diff --git a/salt/version.py b/salt/version.py index b8b9c024d49c..715a9eb43e66 100644 --- a/salt/version.py +++ b/salt/version.py @@ -645,26 +645,47 @@ def system_version(): else: return '' - version = system_version() - release = platform.release() if platform.win32_ver()[0]: + # Get the version and release info based on the Windows Operating + # System Product Name. As long as Microsoft maintains a similar format + # this should be future proof import win32api # pylint: disable=3rd-party-module-not-gated - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function - # started reporting the Desktop version instead of the Server version on - # Server versions of Windows, so we need to look those up - # So, if you find a Server Platform that's a key in the server - # dictionary, then lookup the actual Server Release. - # If this is a Server Platform then `GetVersionEx` will return a number - # greater than 1. - if win32api.GetVersionEx(1)[8] > 1 and release in server: - release = server[release] + import win32con # pylint: disable=3rd-party-module-not-gated + + # Get the product name from the registry + hkey = win32con.HKEY_LOCAL_MACHINE + key = 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion' + value_name = 'ProductName' + reg_handle = win32api.RegOpenKey(hkey, key) + + # Returns a tuple of (product_name, value_type) + product_name, _ = win32api.RegQueryValueEx(reg_handle, value_name) + + version = 'Unknown' + release = '' + if 'Server' in product_name: + for item in product_name.split(' '): + # If it's all digits, then it's version + if re.match(r'\d+', item): + version = item + # If it starts with R and then numbers, it's the release + # ie: R2 + if re.match(r'^R\d+$', item): + release = item + release = '{0}Server{1}'.format(version, release) + else: + for item in product_name.split(' '): + # If it's a number, decimal number, Thin or Vista, then it's the + # version + if re.match(r'^(\d+(\.\d+)?)|Thin|Vista$', item): + version = item + release = version + _, ver, sp, extra = platform.win32_ver() version = ' '.join([release, ver, sp, extra]) + else: + version = system_version() + release = platform.release() system = [ ('system', platform.system()), @@ -737,6 +758,7 @@ def msi_conformant_version(): commi = __saltstack_version__.noc return '{0}.{1}.{2}.{3}'.format(year2, month, minor, commi) + if __name__ == '__main__': if len(sys.argv) == 2 and sys.argv[1] == 'msi': # Building the msi requires an msi-conformant version diff --git a/tests/unit/pillar/test_pepa.py b/tests/unit/pillar/test_pepa.py new file mode 100644 index 000000000000..79dad1af8bbc --- /dev/null +++ b/tests/unit/pillar/test_pepa.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# Import python libs +from __future__ import absolute_import, print_function, unicode_literals +from collections import OrderedDict + +# Import Salt Testing libs +from tests.support.unit import TestCase + +# Import Salt Libs +import salt.pillar.pepa as pepa + + +class PepaPillarTestCase(TestCase): + def test_repeated_keys(self): + expected_result = { + "foo": { + "bar": { + "foo": True, + "baz": True, + }, + }, + } + data = OrderedDict([ + ('foo..bar..foo', True), + ('foo..bar..baz', True), + ]) + result = pepa.key_value_to_tree(data) + self.assertDictEqual(result, expected_result) diff --git a/tests/unit/states/test_service.py b/tests/unit/states/test_service.py index 805b85d2c1bf..9699f1e246fd 100644 --- a/tests/unit/states/test_service.py +++ b/tests/unit/states/test_service.py @@ -57,7 +57,10 @@ def test_running(self): 'name': 'salt', 'result': True}, {'changes': 'saltstack', 'comment': 'Service salt failed to start', 'name': 'salt', - 'result': False}] + 'result': False}, + {'changes': 'saltstack', + 'comment': 'Started Service salt\nService masking not available on this minion', + 'name': 'salt', 'result': True, 'warnings': ["The 'unmask' argument is not supported by this platform/action"]}] tmock = MagicMock(return_value=True) fmock = MagicMock(return_value=False) @@ -91,6 +94,13 @@ def test_running(self): with patch.object(service, '_enable', MagicMock(return_value={'changes': 'saltstack'})): self.assertDictEqual(service.running("salt", True), ret[4]) + with patch.dict(service.__salt__, {'service.status': MagicMock(side_effect=[False, True]), + 'service.enabled': MagicMock(side_effect=[False, True]), + 'service.unmask': MagicMock(side_effect=[False, True]), + 'service.start': MagicMock(return_value="stack")}): + with patch.object(service, '_enable', MagicMock(return_value={'changes': 'saltstack'})): + self.assertDictEqual(service.running("salt", True, unmask=True), ret[7]) + with patch.dict(service.__opts__, {'test': True}): with patch.dict(service.__salt__, {'service.status': tmock}): self.assertDictEqual(service.running("salt"), ret[5])