From 3d09c3b7a326d9e705e4714ea1b937e40407193d Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 24 Feb 2016 19:00:32 +0300 Subject: [PATCH 1/9] Set auth retry count to 0 if multimaster mode is failover. Backport of PR #31382 into 2015.5. --- salt/minion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/minion.py b/salt/minion.py index 8413c9103746..2c99279bee37 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -815,6 +815,7 @@ def eval_master(self, opts['master_active_list'] = opts['master'] if opts.get('master_shuffle'): shuffle(opts['master_list']) + opts['auth_tries'] = 0 elif isinstance(opts['master'], str): # We have a string, but a list was what was intended. Convert. # See issue 23611 for details From 5871e4d1e0ec46667dc24d735f534f636e54a03b Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 24 Feb 2016 15:38:37 -0700 Subject: [PATCH 2/9] Update contributing docs --- Contributing.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Contributing.rst b/Contributing.rst index 358d9aa80938..b52d2372cedb 100644 --- a/Contributing.rst +++ b/Contributing.rst @@ -13,9 +13,10 @@ Please read the following guidelines before you `report an issue`_ 1. **Use the GitHub issue search** — check if the issue has already been reported. If it has been, please comment on the existing issue. -2. **Check if the issue has been fixed** — the latest `develop` - branch may already contain a fix. Please try to reproduce the bug against - the latest git head or the latest release. +2. **Check if the issue has been fixed** — Various point-release branches, such + as ``2015.5``, ``2015.8``, ``2016.3``, or even ``develop``, may already contain + a fix. Please try to reproduce the bug against the latest git HEAD or the latest + release. 3. **Isolate the demonstrable problem** — make sure that the code in the project's repository is *definitely* responsible for the issue. @@ -23,7 +24,7 @@ Please read the following guidelines before you `report an issue`_ 4. **Include a reproducible example** — Provide the steps which led you to the problem. -Please try to be as detailed as possible in your report too. What is your +Please try to be as detailed as possible in your report, too. What is your environment? What steps will reproduce the issue? What Operating System? What would you expect to be the outcome? All these details will help people to assess and fix any potential bugs. @@ -38,16 +39,16 @@ Features Salt is always working to be more powerful. Feature additions and requests are welcomed. When requesting a feature it will be categorized for a release or -placed under "Approved for Future Release". +placed under the "Feature" label. If a new feature is desired, the fastest way to get it into Salt is to -contribute the code. Before starting on a new feature an issue should be filed +contribute the code. Before starting on a new feature, an issue should be filed for it. The one requesting the feature will be able to then discuss the feature with the Salt team and discover the best way to get the feature into Salt and if the feature makes sense. -It is extremely common that the desired feature has already been completed, -look for it in the docs, ask about it first in IRC, and on the mailing list +It is extremely common that the desired feature has already been completed. +Look for it in the docs, ask about it first in IRC, and on the mailing list before filing the request. It is also common that the problem which would be solved by the new feature can be easily solved another way, which is a great reason to ask first. @@ -55,9 +56,13 @@ reason to ask first. Fixing issues ============= -If you wish to help us fixing the issue you're reporting, `Salt's documentation`_ already includes +If you wish to help us fix the issue you're reporting, `Salt's documentation`_ already includes information to help you setup a development environment, under `Developing Salt`_. +`SaltStack's Contributing documentation`_ is also helpful, as it explains sending in pull requests, +keeping your salt branches in sync, and knowing `which branch`_ new features or bug fixes should be +submitted against. + Fix the issue you have in hands, if possible also add a test case to Salt's testing suite, create a `pull request`_, and **that's it**! @@ -69,5 +74,7 @@ salt's code. .. _`Salt's documentation`: http://docs.saltstack.com/en/latest/index.html .. _`Developing Salt`: http://docs.saltstack.com/en/latest/topics/development/hacking.html .. _`pull request`: http://docs.saltstack.com/en/latest/topics/development/contributing.html#sending-a-github-pull-request +.. _`SaltStack's Contributing documentation`: https://docs.saltstack.com/en/latest/topics/development/contributing.html +.. _`which branch`: https://docs.saltstack.com/en/latest/topics/development/contributing.html#which-salt-branch .. vim: fenc=utf-8 spell spl=en From 83e6480d20e0e72ac0c661293415595f5847f8dd Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 25 Feb 2016 14:05:38 -0700 Subject: [PATCH 3/9] Remove duplicate "timeout" definition in Roster docs --- doc/topics/ssh/roster.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/topics/ssh/roster.rst b/doc/topics/ssh/roster.rst index f36050e3150f..08a8f505d195 100644 --- a/doc/topics/ssh/roster.rst +++ b/doc/topics/ssh/roster.rst @@ -45,5 +45,4 @@ The information which can be stored in a roster ``target`` is the following: priv: # File path to ssh private key, defaults to salt-ssh.rsa timeout: # Number of seconds to wait for response when establishing # an SSH connection - timeout: # Number of seconds to wait for response minion_opts: # Dictionary of minion opts From c3f7a2f2e5c64c32ea8c663d8fdf1964144a1b74 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 26 Feb 2016 14:34:14 -0600 Subject: [PATCH 4/9] Add ability to specify an alternate base dir for file caching This commit extends the fileclient such that it is possible to override the base directory for file caching. The new "cachedir" param can be either a relative path or an absolute path. If it is relative, the path will be appended to __opts__['cachedir'], and if it is absolute then it will take the place of __opts__['cachedir']. --- salt/fileclient.py | 113 ++++++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index cfcbd431e435..9027761d569a 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -89,7 +89,7 @@ def _file_local_list(self, dest): return filelist @contextlib.contextmanager - def _cache_loc(self, path, saltenv='base', env=None): + def _cache_loc(self, path, saltenv='base', env=None, cachedir=None): ''' Return the local location to cache the file, cache dirs will be made ''' @@ -103,10 +103,12 @@ def _cache_loc(self, path, saltenv='base', env=None): # Backwards compatibility saltenv = env - dest = os.path.join(self.opts['cachedir'], - 'files', - saltenv, - path) + if cachedir is None: + cachedir = self.opts['cachedir'] + elif not os.path.isabs(cachedir): + cachedir = os.path.join(self.opts['cachedir'], cachedir) + + dest = os.path.join(cachedir, 'files', saltenv, path) destdir = os.path.dirname(dest) cumask = os.umask(63) if not os.path.isdir(destdir): @@ -124,7 +126,8 @@ def get_file(self, makedirs=False, saltenv='base', gzip=None, - env=None): + env=None, + cachedir=None): ''' Copies a file from the local files or master depending on implementation @@ -137,7 +140,7 @@ def file_list_emptydirs(self, saltenv='base', prefix='', env=None): ''' raise NotImplementedError - def cache_file(self, path, saltenv='base', env=None): + def cache_file(self, path, saltenv='base', env=None, cachedir=None): ''' Pull a file down from the file server and store it in the minion file cache @@ -152,9 +155,9 @@ def cache_file(self, path, saltenv='base', env=None): # Backwards compatibility saltenv = env - return self.get_url(path, '', True, saltenv) + return self.get_url(path, '', True, saltenv, cachedir=cachedir) - def cache_files(self, paths, saltenv='base', env=None): + def cache_files(self, paths, saltenv='base', env=None, cachedir=None): ''' Download a list of files stored on the master and put them in the minion file cache @@ -173,10 +176,10 @@ def cache_files(self, paths, saltenv='base', env=None): if isinstance(paths, str): paths = paths.split(',') for path in paths: - ret.append(self.cache_file(path, saltenv)) + ret.append(self.cache_file(path, saltenv, cachedir=cachedir)) return ret - def cache_master(self, saltenv='base', env=None): + def cache_master(self, saltenv='base', env=None, cachedir=None): ''' Download and cache all files on a master in a specified environment ''' @@ -192,11 +195,14 @@ def cache_master(self, saltenv='base', env=None): ret = [] for path in self.file_list(saltenv): - ret.append(self.cache_file('salt://{0}'.format(path), saltenv)) + ret.append( + self.cache_file( + 'salt://{0}'.format(path), saltenv, cachedir=cachedir) + ) return ret def cache_dir(self, path, saltenv='base', include_empty=False, - include_pat=None, exclude_pat=None, env=None): + include_pat=None, exclude_pat=None, env=None, cachedir=None): ''' Download all of the files in a subdir of the master ''' @@ -230,7 +236,8 @@ def cache_dir(self, path, saltenv='base', include_empty=False, if fn_.strip() and fn_.startswith(path): if salt.utils.check_include_exclude( fn_, include_pat, exclude_pat): - fn_ = self.cache_file('salt://' + fn_, saltenv) + fn_ = self.cache_file( + 'salt://' + fn_, saltenv, cachedir=cachedir) if fn_: ret.append(fn_) @@ -244,11 +251,12 @@ def cache_dir(self, path, saltenv='base', include_empty=False, # prefix = '' # else: # prefix = separated[0] - dest = salt.utils.path_join( - self.opts['cachedir'], - 'files', - saltenv - ) + if cachedir is None: + cachedir = self.opts['cachedir'] + elif not os.path.isabs(cachedir): + cachedir = os.path.join(self.opts['cachdir'], cachedir) + + dest = salt.utils.path_join(cachedir, 'files', saltenv) for fn_ in self.file_list_emptydirs(saltenv): if fn_.startswith(path): minion_dir = '{0}/{1}'.format(dest, fn_) @@ -340,7 +348,7 @@ def symlink_list(self, saltenv='base', prefix='', env=None): return {} - def is_cached(self, path, saltenv='base', env=None): + def is_cached(self, path, saltenv='base', env=None, cachedir=None): ''' Returns the full path to a file if it is cached locally on the minion otherwise returns a blank string @@ -367,12 +375,14 @@ def is_cached(self, path, saltenv='base', env=None): self.opts['cachedir'], 'localfiles', path.lstrip('|/')) filesdest = os.path.join( self.opts['cachedir'], 'files', saltenv, path.lstrip('|/')) - extrndest = self._extrn_path(path, saltenv) + extrndest = self._extrn_path(path, saltenv, cachedir=cachedir) if os.path.exists(filesdest): return salt.utils.url.escape(filesdest) if escaped else filesdest elif os.path.exists(localsfilesdest): - return salt.utils.url.escape(localsfilesdest) if escaped else localsfilesdest + return salt.utils.url.escape(localsfilesdest) \ + if escaped \ + else localsfilesdest elif os.path.exists(extrndest): return extrndest @@ -430,7 +440,7 @@ def list_states(self, saltenv): states.append(path.replace('/', '.')[:-4]) return states - def get_state(self, sls, saltenv): + def get_state(self, sls, saltenv, cachedir=None): ''' Get a state file from the master and store it in the local minion cache return the location of the file @@ -439,12 +449,13 @@ def get_state(self, sls, saltenv): sls = sls.replace('.', '/') for path in ['salt://{0}.sls'.format(sls), '/'.join(['salt:/', sls, 'init.sls'])]: - dest = self.cache_file(path, saltenv) + dest = self.cache_file(path, saltenv, cachedir=cachedir) if dest: return {'source': path, 'dest': dest} return {} - def get_dir(self, path, dest='', saltenv='base', gzip=None, env=None): + def get_dir(self, path, dest='', saltenv='base', gzip=None, env=None, + cachedir=None): ''' Get a directory recursively from the salt-master ''' @@ -487,15 +498,15 @@ def get_dir(self, path, dest='', saltenv='base', gzip=None, env=None): self.get_file( 'salt://{0}'.format(fn_), '{0}/{1}'.format(dest, minion_relpath), - True, saltenv, gzip + True, saltenv, gzip, cachedir=cachedir ) ) # Replicate empty dirs from master try: for fn_ in self.file_list_emptydirs(saltenv): if fn_.startswith(path): - # Prevent an empty dir "salt://foobar/" from matching a path of - # "salt://foo" + # Prevent an empty dir "salt://foobar/" from matching a + # path of "salt://foo" try: if fn_[len(path)] != '/': continue @@ -513,7 +524,8 @@ def get_dir(self, path, dest='', saltenv='base', gzip=None, env=None): ret.sort() return ret - def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False): + def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, + no_cache=False, cachedir=None): ''' Get a single file from a URL. ''' @@ -538,7 +550,8 @@ def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache= return url_data.path if url_data.scheme == 'salt': - return self.get_file(url, dest, makedirs, saltenv) + return self.get_file( + url, dest, makedirs, saltenv, cachedir=cachedir) if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): @@ -547,7 +560,7 @@ def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache= else: return '' elif not no_cache: - dest = self._extrn_path(url, saltenv) + dest = self._extrn_path(url, saltenv, cachedir=cachedir) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) @@ -575,7 +588,9 @@ def s3_opt(key, default=None): location=s3_opt('location')) return dest except Exception as exc: - raise MinionError('Could not fetch from {0}. Exception: {1}'.format(url, exc)) + raise MinionError( + 'Could not fetch from {0}. Exception: {1}'.format(url, exc) + ) if url_data.scheme == 'ftp': try: ftp = ftplib.FTP(url_data.hostname) @@ -652,6 +667,7 @@ def get_template( makedirs=False, saltenv='base', env=None, + cachedir=None, **kwargs): ''' Cache a file then process it as a template @@ -668,7 +684,7 @@ def get_template( kwargs['saltenv'] = saltenv url_data = urlparse(url) - sfn = self.cache_file(url, saltenv) + sfn = self.cache_file(url, saltenv, cachedir=cachedir) if not os.path.exists(sfn): return '' if template in salt.utils.templates.TEMPLATE_REGISTRY: @@ -690,7 +706,7 @@ def get_template( return '' if not dest: # No destination passed, set the dest as an extrn_files cache - dest = self._extrn_path(url, saltenv) + dest = self._extrn_path(url, saltenv, cachedir=cachedir) # If Salt generated the dest name, create any required dirs makedirs = True @@ -704,7 +720,7 @@ def get_template( shutil.move(data['data'], dest) return dest - def _extrn_path(self, url, saltenv): + def _extrn_path(self, url, saltenv, cachedir=None): ''' Return the extn_filepath for a given url ''' @@ -714,8 +730,13 @@ def _extrn_path(self, url, saltenv): else: netloc = url_data.netloc + if cachedir is None: + cachedir = self.opts['cachedir'] + elif not os.path.isabs(cachedir): + cachedir = os.path.join(self.opts['cachedir'], cachedir) + return salt.utils.path_join( - self.opts['cachedir'], + cachedir, 'extrn_files', saltenv, netloc, @@ -756,7 +777,8 @@ def get_file(self, makedirs=False, saltenv='base', gzip=None, - env=None): + env=None, + cachedir=None): ''' Copies a file from the local files directory into :param:`dest` gzip compression settings are ignored for local files @@ -973,14 +995,14 @@ def get_file(self, makedirs=False, saltenv='base', gzip=None, - env=None): + env=None, + cachedir=None): ''' Get a single file from the salt-master path must be a salt server location, aka, salt://path/to/file, if dest is omitted, then the downloaded file will be placed in the minion cache ''' - # Check if file exists on server, before creating files and # directories hash_server = self.hash_file(path, saltenv) @@ -1007,7 +1029,8 @@ def get_file(self, dest2check = dest if not dest2check: rel_path = self._check_proto(path) - with self._cache_loc(rel_path, saltenv) as cache_dest: + with self._cache_loc( + rel_path, saltenv, cachedir=cachedir) as cache_dest: dest2check = cache_dest if dest2check and os.path.isfile(dest2check): @@ -1056,7 +1079,10 @@ def get_file(self, if not data['data']: if not fn_ and data['dest']: # This is a 0 byte file on the master - with self._cache_loc(data['dest'], saltenv) as cache_dest: + with self._cache_loc( + data['dest'], + saltenv, + cachedir=cachedir) as cache_dest: dest = cache_dest with salt.utils.fopen(cache_dest, 'wb+') as ofile: ofile.write(data['data']) @@ -1071,7 +1097,10 @@ def get_file(self, continue break if not fn_: - with self._cache_loc(data['dest'], saltenv) as cache_dest: + with self._cache_loc( + data['dest'], + saltenv, + cachedir=cachedir) as cache_dest: dest = cache_dest # If a directory was formerly cached at this path, then # remove it to avoid a traceback trying to write the file From 65bdcb3afa2395c9b89215354ba09d28b96e2178 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 26 Feb 2016 14:41:12 -0600 Subject: [PATCH 5/9] Accept and pass through the alternate cachedir when prepping the thin tar --- salt/client/ssh/state.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/client/ssh/state.py b/salt/client/ssh/state.py index fe9b7e4dbaf4..56cfa65443dd 100644 --- a/salt/client/ssh/state.py +++ b/salt/client/ssh/state.py @@ -119,7 +119,7 @@ def salt_refs(data, ret=None): return ret -def prep_trans_tar(file_client, chunks, file_refs, pillar=None): +def prep_trans_tar(file_client, chunks, file_refs, pillar=None, id_=None): ''' Generate the execution package from the saltenv file refs and a low state data structure @@ -142,6 +142,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None): if pillar: with salt.utils.fopen(pillarfn, 'w+') as fp_: fp_.write(json.dumps(pillar)) + cachedir = os.path.join('salt-ssh', id_) for saltenv in file_refs: file_refs[saltenv].extend(sync_refs) env_root = os.path.join(gendir, saltenv) @@ -150,7 +151,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None): for ref in file_refs[saltenv]: for name in ref: short = name[7:] - path = file_client.cache_file(name, saltenv) + path = file_client.cache_file(name, saltenv, cachedir=cachedir) if path: tgt = os.path.join(env_root, short) tgt_dir = os.path.dirname(tgt) @@ -158,7 +159,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None): os.makedirs(tgt_dir) shutil.copy(path, tgt) continue - files = file_client.cache_dir(name, saltenv) + files = file_client.cache_dir(name, saltenv, cachedir=cachedir) if files: for filename in files: fn = filename[filename.find(short) + len(short):] From 0320494b1d172bab34f8e3892fd4fa8a530f42ff Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 26 Feb 2016 14:41:50 -0600 Subject: [PATCH 6/9] Update the SSH state module wrappers to pass an alternate cachedir --- salt/client/ssh/wrapper/state.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index b1135deeea10..0dce3cb580fa 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -94,11 +94,13 @@ def sls(mods, saltenv='base', test=None, exclude=None, env=None, **kwargs): __opts__.get('extra_filerefs', '') ) ) + # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, - __pillar__) + __pillar__, + id_=st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], @@ -161,11 +163,13 @@ def low(data, **kwargs): __opts__.get('extra_filerefs', '') ) ) + # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, - __pillar__) + __pillar__, + id_=st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( __opts__['thin_dir'], @@ -225,11 +229,13 @@ def high(data, **kwargs): __opts__.get('extra_filerefs', '') ) ) + # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, - __pillar__) + __pillar__, + id_=st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( __opts__['thin_dir'], @@ -317,11 +323,13 @@ def highstate(test=None, **kwargs): for chunk in chunks: if not isinstance(chunk, dict): return chunks + # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, - __pillar__) + __pillar__, + id_=st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], @@ -388,11 +396,13 @@ def top(topfn, test=None, **kwargs): __opts__.get('extra_filerefs', '') ) ) + # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, - __pillar__) + __pillar__, + id_=st_kwargs['id_']) trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__['thin_dir'], @@ -605,12 +615,13 @@ def single(fun, name, test=None, **kwargs): ) ) - # Create the tar containing the state pkg and relevant files + # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __context__['fileclient'], chunks, file_refs, - __pillar__) + __pillar__, + id_=st_kwargs['id_']) # Create a hash so we can verify the tar on the target system trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) From d9370a8041dea7673c4121934364553325444c7b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 26 Feb 2016 15:37:51 -0600 Subject: [PATCH 7/9] Update cp module salt-ssh wrapper to use new cachedir param --- salt/client/ssh/wrapper/cp.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/salt/client/ssh/wrapper/cp.py b/salt/client/ssh/wrapper/cp.py index 9cb76a01b999..dcb30396dd7d 100644 --- a/salt/client/ssh/wrapper/cp.py +++ b/salt/client/ssh/wrapper/cp.py @@ -7,6 +7,7 @@ # Import salt libs import salt.client.ssh import logging +import os from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -33,7 +34,10 @@ def get_file(path, if template is not None: (path, dest) = _render_filenames(path, dest, saltenv, template) - src = __context__['fileclient'].cache_file(path, saltenv) + src = __context__['fileclient'].cache_file( + path, + saltenv, + cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_'])) single = salt.client.ssh.Single( __opts__, '', @@ -46,7 +50,10 @@ def get_dir(path, dest, saltenv='base'): ''' Transfer a directory down ''' - src = __context__['fileclient'].cache_dir(path, saltenv) + src = __context__['fileclient'].cache_dir( + path, + saltenv, + cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_'])) src = ' '.join(src) single = salt.client.ssh.Single( __opts__, @@ -60,7 +67,10 @@ def get_url(path, dest, saltenv='base'): ''' retrive a URL ''' - src = __context__['fileclient'].get_url(path, saltenv) + src = __context__['fileclient'].get_url( + path, + saltenv, + cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_'])) single = salt.client.ssh.Single( __opts__, '', From 0d352bbc16ec476aa901b59f710a1b24b01f6eb9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 29 Feb 2016 00:53:38 -0600 Subject: [PATCH 8/9] Add fileclient tests --- tests/integration/fileclient_test.py | 333 +++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 tests/integration/fileclient_test.py diff --git a/tests/integration/fileclient_test.py b/tests/integration/fileclient_test.py new file mode 100644 index 000000000000..747acb6d3acc --- /dev/null +++ b/tests/integration/fileclient_test.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Erik Johnson ` +''' + +# Import Salt Testing libs +from salttesting import skipIf +from salttesting.helpers import ensure_in_syspath +from salttesting.mock import patch, NO_MOCK, NO_MOCK_REASON + +ensure_in_syspath('../..') + +# Import Python libs +import errno +import logging +import os +import shutil + +# Import salt libs +import integration +import salt.utils +from salt import fileclient +from salt.ext import six +from salttesting.helpers import ensure_in_syspath, destructiveTest +ensure_in_syspath('..') + +SALTENVS = ('base', 'dev') +FS_ROOT = os.path.join(integration.TMP, 'fileclient_fs_root') +CACHE_ROOT = os.path.join(integration.TMP, 'fileclient_cache_root') +SUBDIR = 'subdir' +SUBDIR_FILES = ('foo.txt', 'bar.txt', 'baz.txt') + + +def _get_file_roots(): + return dict( + [(x, [os.path.join(FS_ROOT, x)]) for x in SALTENVS] + ) + + +fileclient.__opts__ = {} +MOCKED_OPTS = { + 'file_roots': _get_file_roots(), + 'fileserver_backend': ['roots'], + 'cachedir': CACHE_ROOT, + 'file_client': 'local', +} + +log = logging.getLogger(__name__) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@destructiveTest +class FileclientTest(integration.ModuleCase): + ''' + Tests for the fileclient. The LocalClient is the only thing we can test as + it is the only way we can mock the fileclient (the tests run from the + minion process, so the master cannot be mocked from test code). + ''' + + def setUp(self): + ''' + No need to add a dummy foo.txt to muddy up the github repo, just make + our own fileserver root on-the-fly. + ''' + def _new_dir(path): + ''' + Add a new dir at ``path`` using os.makedirs. If the directory + already exists, remove it recursively and then try to create it + again. + ''' + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST: + # Just in case a previous test was interrupted, remove the + # directory and try adding it again. + shutil.rmtree(path) + os.makedirs(path) + else: + raise + + # Crete the FS_ROOT + for saltenv in SALTENVS: + saltenv_root = os.path.join(FS_ROOT, saltenv) + # Make sure we have a fresh root dir for this saltenv + _new_dir(saltenv_root) + + path = os.path.join(saltenv_root, 'foo.txt') + with salt.utils.fopen(path, 'w') as fp_: + fp_.write( + 'This is a test file in the \'{0}\' saltenv.\n' + .format(saltenv) + ) + + subdir_abspath = os.path.join(saltenv_root, SUBDIR) + os.makedirs(subdir_abspath) + for subdir_file in SUBDIR_FILES: + path = os.path.join(subdir_abspath, subdir_file) + with salt.utils.fopen(path, 'w') as fp_: + fp_.write( + 'This is file \'{0}\' in subdir \'{1} from saltenv ' + '\'{2}\''.format(subdir_file, SUBDIR, saltenv) + ) + + # Create the CACHE_ROOT + _new_dir(CACHE_ROOT) + + def tearDown(self): + ''' + Remove the directories created for these tests + ''' + shutil.rmtree(FS_ROOT) + shutil.rmtree(CACHE_ROOT) + + def test_cache_dir(self): + ''' + Ensure entire directory is cached to correct location + ''' + patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts)) + patched_opts.update(MOCKED_OPTS) + + with patch.dict(fileclient.__opts__, patched_opts): + client = fileclient.get_file_client(fileclient.__opts__, pillar=False) + for saltenv in SALTENVS: + self.assertTrue( + client.cache_dir( + 'salt://{0}'.format(SUBDIR), + saltenv, + cachedir=None + ) + ) + for subdir_file in SUBDIR_FILES: + cache_loc = os.path.join(fileclient.__opts__['cachedir'], + 'files', + saltenv, + SUBDIR, + subdir_file) + # Double check that the content of the cached file + # identifies it as being from the correct saltenv. The + # setUp function creates the file with the name of the + # saltenv mentioned in the file, so a simple 'in' check is + # sufficient here. If opening the file raises an exception, + # this is a problem, so we are not catching the exception + # and letting it be raised so that the test fails. + with salt.utils.fopen(cache_loc) as fp_: + content = fp_.read() + log.debug('cache_loc = %s', cache_loc) + log.debug('content = %s', content) + self.assertTrue(subdir_file in content) + self.assertTrue(SUBDIR in content) + self.assertTrue(saltenv in content) + + def test_cache_dir_with_alternate_cachedir_and_absolute_path(self): + ''' + Ensure entire directory is cached to correct location when an alternate + cachedir is specified and that cachedir is an absolute path + ''' + patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts)) + patched_opts.update(MOCKED_OPTS) + alt_cachedir = os.path.join(integration.TMP, 'abs_cachedir') + + with patch.dict(fileclient.__opts__, patched_opts): + client = fileclient.get_file_client(fileclient.__opts__, pillar=False) + for saltenv in SALTENVS: + self.assertTrue( + client.cache_dir( + 'salt://{0}'.format(SUBDIR), + saltenv, + cachedir=alt_cachedir + ) + ) + for subdir_file in SUBDIR_FILES: + cache_loc = os.path.join(alt_cachedir, + 'files', + saltenv, + SUBDIR, + subdir_file) + # Double check that the content of the cached file + # identifies it as being from the correct saltenv. The + # setUp function creates the file with the name of the + # saltenv mentioned in the file, so a simple 'in' check is + # sufficient here. If opening the file raises an exception, + # this is a problem, so we are not catching the exception + # and letting it be raised so that the test fails. + with salt.utils.fopen(cache_loc) as fp_: + content = fp_.read() + log.debug('cache_loc = %s', cache_loc) + log.debug('content = %s', content) + self.assertTrue(subdir_file in content) + self.assertTrue(SUBDIR in content) + self.assertTrue(saltenv in content) + + def test_cache_dir_with_alternate_cachedir_and_relative_path(self): + ''' + Ensure entire directory is cached to correct location when an alternate + cachedir is specified and that cachedir is a relative path + ''' + patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts)) + patched_opts.update(MOCKED_OPTS) + alt_cachedir = 'foo' + + with patch.dict(fileclient.__opts__, patched_opts): + client = fileclient.get_file_client(fileclient.__opts__, pillar=False) + for saltenv in SALTENVS: + self.assertTrue( + client.cache_dir( + 'salt://{0}'.format(SUBDIR), + saltenv, + cachedir=alt_cachedir + ) + ) + for subdir_file in SUBDIR_FILES: + cache_loc = os.path.join(fileclient.__opts__['cachedir'], + alt_cachedir, + 'files', + saltenv, + SUBDIR, + subdir_file) + # Double check that the content of the cached file + # identifies it as being from the correct saltenv. The + # setUp function creates the file with the name of the + # saltenv mentioned in the file, so a simple 'in' check is + # sufficient here. If opening the file raises an exception, + # this is a problem, so we are not catching the exception + # and letting it be raised so that the test fails. + with salt.utils.fopen(cache_loc) as fp_: + content = fp_.read() + log.debug('cache_loc = %s', cache_loc) + log.debug('content = %s', content) + self.assertTrue(subdir_file in content) + self.assertTrue(SUBDIR in content) + self.assertTrue(saltenv in content) + + def test_cache_file(self): + ''' + Ensure file is cached to correct location + ''' + patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts)) + patched_opts.update(MOCKED_OPTS) + + with patch.dict(fileclient.__opts__, patched_opts): + client = fileclient.get_file_client(fileclient.__opts__, pillar=False) + for saltenv in SALTENVS: + self.assertTrue( + client.cache_file('salt://foo.txt', saltenv, cachedir=None) + ) + cache_loc = os.path.join( + fileclient.__opts__['cachedir'], 'files', saltenv, 'foo.txt') + # Double check that the content of the cached file identifies + # it as being from the correct saltenv. The setUp function + # creates the file with the name of the saltenv mentioned in + # the file, so a simple 'in' check is sufficient here. If + # opening the file raises an exception, this is a problem, so + # we are not catching the exception and letting it be raised so + # that the test fails. + with salt.utils.fopen(cache_loc) as fp_: + content = fp_.read() + log.debug('cache_loc = %s', cache_loc) + log.debug('content = %s', content) + self.assertTrue(saltenv in content) + + def test_cache_file_with_alternate_cachedir_and_absolute_path(self): + ''' + Ensure file is cached to correct location when an alternate cachedir is + specified and that cachedir is an absolute path + ''' + patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts)) + patched_opts.update(MOCKED_OPTS) + alt_cachedir = os.path.join(integration.TMP, 'abs_cachedir') + + with patch.dict(fileclient.__opts__, patched_opts): + client = fileclient.get_file_client(fileclient.__opts__, pillar=False) + for saltenv in SALTENVS: + self.assertTrue( + client.cache_file('salt://foo.txt', + saltenv, + cachedir=alt_cachedir) + ) + cache_loc = os.path.join(alt_cachedir, + 'files', + saltenv, + 'foo.txt') + # Double check that the content of the cached file identifies + # it as being from the correct saltenv. The setUp function + # creates the file with the name of the saltenv mentioned in + # the file, so a simple 'in' check is sufficient here. If + # opening the file raises an exception, this is a problem, so + # we are not catching the exception and letting it be raised so + # that the test fails. + with salt.utils.fopen(cache_loc) as fp_: + content = fp_.read() + log.debug('cache_loc = %s', cache_loc) + log.debug('content = %s', content) + self.assertTrue(saltenv in content) + + def test_cache_file_with_alternate_cachedir_and_relative_path(self): + ''' + Ensure file is cached to correct location when an alternate cachedir is + specified and that cachedir is a relative path + ''' + patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts)) + patched_opts.update(MOCKED_OPTS) + alt_cachedir = 'foo' + + with patch.dict(fileclient.__opts__, patched_opts): + client = fileclient.get_file_client(fileclient.__opts__, pillar=False) + for saltenv in SALTENVS: + self.assertTrue( + client.cache_file('salt://foo.txt', + saltenv, + cachedir=alt_cachedir) + ) + cache_loc = os.path.join(fileclient.__opts__['cachedir'], + alt_cachedir, + 'files', + saltenv, + 'foo.txt') + # Double check that the content of the cached file identifies + # it as being from the correct saltenv. The setUp function + # creates the file with the name of the saltenv mentioned in + # the file, so a simple 'in' check is sufficient here. If + # opening the file raises an exception, this is a problem, so + # we are not catching the exception and letting it be raised so + # that the test fails. + with salt.utils.fopen(cache_loc) as fp_: + content = fp_.read() + log.debug('cache_loc = %s', cache_loc) + log.debug('content = %s', content) + self.assertTrue(saltenv in content) + + +if __name__ == '__main__': + integration.run_tests(FileclientTest) From b683df9b823f07b9089bc8a01559696c3048c0e7 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 29 Feb 2016 11:12:55 -0700 Subject: [PATCH 9/9] Pylint fix --- tests/integration/fileclient_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/fileclient_test.py b/tests/integration/fileclient_test.py index 747acb6d3acc..7f2bb488c2f8 100644 --- a/tests/integration/fileclient_test.py +++ b/tests/integration/fileclient_test.py @@ -2,6 +2,7 @@ ''' :codeauthor: :email:`Erik Johnson ` ''' +from __future__ import absolute_import # Import Salt Testing libs from salttesting import skipIf